From d2501c4f4dc54bf3a3db67290e0944d870f376f7 Mon Sep 17 00:00:00 2001 From: Arjun <6592213+arkml@users.noreply.github.com> Date: Sat, 4 Jul 2026 00:25:55 +0530 Subject: [PATCH] video mode popout over the app --- apps/x/VIDEO_MODE.md | 18 ++- apps/x/apps/main/src/ipc.ts | 37 +++-- apps/x/apps/renderer/src/App.tsx | 87 ++++++----- .../renderer/src/components/video-popout.tsx | 142 ++++++++++++------ apps/x/packages/shared/src/ipc.ts | 23 ++- 5 files changed, 208 insertions(+), 99 deletions(-) diff --git a/apps/x/VIDEO_MODE.md b/apps/x/VIDEO_MODE.md index fff8f6f4..5b04b770 100644 --- a/apps/x/VIDEO_MODE.md +++ b/apps/x/VIDEO_MODE.md @@ -26,10 +26,17 @@ On top of any mode: no webcam frames are captured; tiles show a silhouette avatar. - **Mascot dismissal** (meeting view): swaps the animated mascot for a Meet-style letter avatar ("R"). -- **Popout**: while screen sharing, if the app window loses focus (the user - switched to the app they're sharing), a small always-on-top frameless - window pops out with the user + mascot mini-tiles; refocusing dismisses it. - Its expand button focuses the main window (`video:focusMain`). +- **Popout**: starting a screen share immediately pops the mini-call out into + a small always-on-top frameless window (user + mascot tiles, live caption, + control bar) that floats over every app — including Rowboat itself, since + the shared screen may be Rowboat. It stays for the whole share (surviving + app switches) and disappears when sharing stops or video mode ends. While + sharing it is THE call surface: the in-app PiP hides, and presenting from + the full-screen meeting collapses the meeting into it (restored when the + share ends). Control-bar actions (camera toggle / stop share / end call) + round-trip `video:popoutAction` → main → `video:popout-action` → app + window, which owns the mic/camera/capture. The expand button focuses the + main window (`video:focusMain`). `call`/`meeting` options are disabled unless both voice input (Deepgram) and voice output (TTS) are configured. Entering a call saves the user's TTS @@ -95,7 +102,8 @@ disabled while a call owns the mic. ## Popout window -- Renderer asks `video:setPopout {show}` (main handler: +- Shown iff `videoChatMode !== 'off' && screenState === 'live'` (effect in + `App.tsx`). Renderer asks `video:setPopout {show}` (main handler: `apps/main/src/ipc.ts:1742`); main creates a frameless, `alwaysOnTop` ('floating'), all-workspaces BrowserWindow at the top-right of the primary display, loading the renderer bundle with `#video-popout` diff --git a/apps/x/apps/main/src/ipc.ts b/apps/x/apps/main/src/ipc.ts index 62e5b8fb..846fa7fa 100644 --- a/apps/x/apps/main/src/ipc.ts +++ b/apps/x/apps/main/src/ipc.ts @@ -382,16 +382,29 @@ type InvokeHandlers = { [K in InvokeChannels]: InvokeHandler; }; -// Video-mode popout window (shown while screen sharing when the app loses -// focus) and the last call state pushed by the main window — replayed to the -// popout when it finishes loading. +// Video-mode popout window (shown for the whole duration of a screen share, +// floating over every app including Rowboat itself) and the last call state +// pushed by the main window — replayed to the popout when it finishes loading. let videoPopoutWin: BrowserWindow | null = null; let lastVideoPopoutState: { ttsState: 'idle' | 'synthesizing' | 'speaking'; status: 'listening' | 'thinking' | 'speaking' | null; cameraOn: boolean; + interimText: string | null; } | null = null; +// Match only real app windows — getAllWindows() can also contain the popout +// itself and hidden utility windows (e.g. PDF-export renderers), which must +// not be shown, focused, or sent app events. +function findMainAppWindow(): BrowserWindow | undefined { + return BrowserWindow.getAllWindows().find((w) => { + if (w === videoPopoutWin || w.isDestroyed()) return false; + const url = w.webContents.getURL(); + const isAppWindow = url.startsWith('app://') || url.startsWith('http://localhost'); + return isAppWindow && !url.includes('#video-popout'); + }); +} + /** * Register all IPC handlers with type safety and runtime validation * @@ -1749,7 +1762,7 @@ export function setupIpcHandlers() { const workArea = screen.getPrimaryDisplay().workArea; const width = 340; - const height = 148; + const height = 184; const ipcDir = path.dirname(fileURLToPath(import.meta.url)); const preloadPath = app.isPackaged ? path.join(ipcDir, '../preload/dist/preload.js') @@ -1804,15 +1817,7 @@ export function setupIpcHandlers() { return {}; }, 'video:focusMain': async () => { - // Match only real app windows — getAllWindows() can also contain the - // popout itself and hidden utility windows (e.g. PDF-export renderers), - // which must not be shown or focused. - const main = BrowserWindow.getAllWindows().find((w) => { - if (w === videoPopoutWin || w.isDestroyed()) return false; - const url = w.webContents.getURL(); - const isAppWindow = url.startsWith('app://') || url.startsWith('http://localhost'); - return isAppWindow && !url.includes('#video-popout'); - }); + const main = findMainAppWindow(); if (main) { if (main.isMinimized()) main.restore(); main.show(); @@ -1820,6 +1825,12 @@ export function setupIpcHandlers() { } return {}; }, + 'video:popoutAction': async (_event, args) => { + // Relay a popout control-bar action to the app window, which owns the + // call (mic, camera, screen capture) and executes it there. + findMainAppWindow()?.webContents.send('video:popout-action', args); + return {}; + }, // Live-note handlers 'live-note:run': async (_event, args) => { const result = await runLiveNoteAgent(args.filePath, 'manual', args.context); diff --git a/apps/x/apps/renderer/src/App.tsx b/apps/x/apps/renderer/src/App.tsx index d6d69d74..cfd82a1d 100644 --- a/apps/x/apps/renderer/src/App.tsx +++ b/apps/x/apps/renderer/src/App.tsx @@ -1221,15 +1221,35 @@ function App() { voiceRef.current.setPaused(activeIsProcessing || tts.state !== 'idle') }, [videoChatMode, activeIsProcessing, tts.state]) + // Presenting from the full-screen call collapses it into the floating + // popout (the user needs their screen — possibly Rowboat itself — free to + // navigate while sharing). When the share ends, return to full screen. + const returnToMeetingAfterShareRef = useRef(false) + // Screen sharing (any video mode): frames of the shared screen ride along // with each message next to the webcam frames. const handleToggleScreenShare = useCallback(async () => { if (video.screenState === 'live') { video.stopScreenShare() } else { - await video.startScreenShare() + const ok = await video.startScreenShare() + if (ok && videoChatModeRef.current === 'meeting') { + returnToMeetingAfterShareRef.current = true + void handleVideoModeChange('call') + } } - }, [video]) + }, [video, handleVideoModeChange]) + + // Share ended (from the popout, the PiP, or the OS): restore the + // full-screen call if that's where the share was started from. + useEffect(() => { + if (video.screenState === 'live') return + if (!returnToMeetingAfterShareRef.current) return + returnToMeetingAfterShareRef.current = false + if (videoChatModeRef.current === 'call') { + void handleVideoModeChange('meeting') + } + }, [video.screenState, handleVideoModeChange]) // Meet-style camera mute: video mode (and any screen share) stays on, but // no webcam frames are captured while the camera is off. @@ -1247,40 +1267,39 @@ function App() { : 'listening' : null - // Meet-style popout: while screen sharing, losing app focus (the user - // switched to the app they're sharing) pops the mini-call out into an - // always-on-top window; refocusing Rowboat dismisses it. The short show - // delay keeps a quick cmd-tab pass-through from flashing a window. - const [appFocused, setAppFocused] = useState(true) + // Meet-style popout: the floating always-on-top mini-call appears the + // moment screen sharing starts and stays for the whole share — including + // over Rowboat itself (the shared screen may BE Rowboat), and across app + // switches. It disappears when sharing stops or video mode ends. useEffect(() => { - const onFocus = () => setAppFocused(true) - const onBlur = () => setAppFocused(false) - window.addEventListener('focus', onFocus) - window.addEventListener('blur', onBlur) - return () => { - window.removeEventListener('focus', onFocus) - window.removeEventListener('blur', onBlur) - } - }, []) - useEffect(() => { - const shouldShow = videoChatMode !== 'off' && video.screenState === 'live' && !appFocused - if (shouldShow) { - const timer = setTimeout(() => { - void window.ipc.invoke('video:setPopout', { show: true }).catch(() => {}) - }, 300) - return () => clearTimeout(timer) - } - void window.ipc.invoke('video:setPopout', { show: false }).catch(() => {}) - }, [videoChatMode, video.screenState, appFocused]) + const shouldShow = videoChatMode !== 'off' && video.screenState === 'live' + void window.ipc.invoke('video:setPopout', { show: shouldShow }).catch(() => {}) + }, [videoChatMode, video.screenState]) - // Keep the popout's mascot/status/camera mirror of the call fresh. The main - // process caches the latest state and replays it when the popout loads. + // Keep the popout's mascot/status/camera/caption mirror of the call fresh. + // The main process caches the latest state and replays it when the popout + // loads. useEffect(() => { if (videoChatMode === 'off') return void window.ipc - .invoke('video:popoutState', { ttsState: tts.state, status: videoCallStatus, cameraOn: video.cameraOn }) + .invoke('video:popoutState', { + ttsState: tts.state, + status: videoCallStatus, + cameraOn: video.cameraOn, + interimText: videoCallStatus ? voice.interimText || null : null, + }) .catch(() => {}) - }, [videoChatMode, tts.state, videoCallStatus, video.cameraOn]) + }, [videoChatMode, tts.state, videoCallStatus, video.cameraOn, voice.interimText]) + + // Execute popout control-bar actions (the popout window has no access to + // the call's mic/camera/capture — they live here). + useEffect(() => { + return window.ipc.on('video:popout-action', ({ action }) => { + if (action === 'toggle-camera') handleToggleCamera() + else if (action === 'stop-share') video.stopScreenShare() + else if (action === 'end-call') void handleVideoModeChange('off') + }) + }, [handleToggleCamera, video, handleVideoModeChange]) // Enter to submit voice input, Escape to cancel useEffect(() => { @@ -6644,9 +6663,10 @@ function App() { onComposioConnected={handleComposioConnected} /> )} - {/* Webcam PiP preview while video chat mode is on (the full-screen - meeting view replaces it) */} - {(videoChatMode === 'chat' || videoChatMode === 'call') && ( + {/* Webcam PiP preview while video chat mode is on. Hidden while + screen sharing (the floating popout is the call surface then) + and in the full-screen meeting view (which replaces it). */} + {(videoChatMode === 'chat' || videoChatMode === 'call') && video.screenState !== 'live' && ( handleVideoModeChange('off')} @@ -6660,7 +6680,6 @@ function App() { : undefined } interimText={videoChatMode === 'call' ? voice.interimText : undefined} - isScreenSharing={video.screenState === 'live'} onToggleScreenShare={handleToggleScreenShare} cameraOn={video.cameraOn} onToggleCamera={handleToggleCamera} diff --git a/apps/x/apps/renderer/src/components/video-popout.tsx b/apps/x/apps/renderer/src/components/video-popout.tsx index ea97f5d2..8e70a56f 100644 --- a/apps/x/apps/renderer/src/components/video-popout.tsx +++ b/apps/x/apps/renderer/src/components/video-popout.tsx @@ -1,5 +1,5 @@ import { useCallback, useEffect, useRef, useState } from 'react' -import { Maximize2, User } from 'lucide-react' +import { Maximize2, MonitorUp, PhoneOff, User, Video, VideoOff } from 'lucide-react' import { TalkingHead } from '@/components/talking-head' @@ -7,6 +7,7 @@ type PopoutState = { ttsState: 'idle' | 'synthesizing' | 'speaking' status: 'listening' | 'thinking' | 'speaking' | null cameraOn: boolean + interimText: string | null } const STATUS_DISPLAY: Record, { label: string; dotClass: string }> = { @@ -19,14 +20,17 @@ const dragRegion = { WebkitAppRegion: 'drag' } as React.CSSProperties const noDragRegion = { WebkitAppRegion: 'no-drag' } as React.CSSProperties /** - * Content of the always-on-top popout window shown while screen sharing when - * the main app window loses focus (Meet-style floating mini-call). Rendered - * in its own BrowserWindow (see `video:setPopout` in the main process); call - * state streams in over the `video:popout-state` push channel. Captures its - * own webcam feed — MediaStreams can't cross windows. + * Content of the always-on-top popout window shown for the whole duration of + * a screen share (Meet-style floating mini-call) — it floats over every app, + * including Rowboat itself, and is the call's control surface while sharing: + * camera toggle, stop-share, end-call. Rendered in its own BrowserWindow + * (see `video:setPopout` in the main process); call state streams in over + * the `video:popout-state` push channel and control actions round-trip back + * through `video:popoutAction`. Captures its own webcam feed — MediaStreams + * can't cross windows. */ export function VideoPopout() { - const [state, setState] = useState({ ttsState: 'idle', status: null, cameraOn: true }) + const [state, setState] = useState({ ttsState: 'idle', status: null, cameraOn: true, interimText: null }) const videoRef = useRef(null) useEffect(() => { @@ -63,55 +67,103 @@ export function VideoPopout() { // so the mascot still animates while the assistant speaks in the main window. const getLevel = useCallback(() => 0.45 + 0.35 * Math.sin(performance.now() / 90), []) + const sendAction = useCallback((action: 'toggle-camera' | 'stop-share' | 'end-call') => { + void window.ipc.invoke('video:popoutAction', { action }).catch(() => {}) + }, []) + const statusDisplay = state.status ? STATUS_DISPLAY[state.status] : null return (
-
- {state.cameraOn ? ( -