video mode popout over the app

apps/x/VIDEO_MODE.md

@@ -26,10 +26,17 @@ On top of any mode:
   no webcam frames are captured; tiles show a silhouette avatar.
 - **Mascot dismissal** (meeting view): swaps the animated mascot for a
   Meet-style letter avatar ("R").
-- **Popout**: while screen sharing, if the app window loses focus (the user
-  switched to the app they're sharing), a small always-on-top frameless
-  window pops out with the user + mascot mini-tiles; refocusing dismisses it.
-  Its expand button focuses the main window (`video:focusMain`).
+- **Popout**: starting a screen share immediately pops the mini-call out into
+  a small always-on-top frameless window (user + mascot tiles, live caption,
+  control bar) that floats over every app — including Rowboat itself, since
+  the shared screen may be Rowboat. It stays for the whole share (surviving
+  app switches) and disappears when sharing stops or video mode ends. While
+  sharing it is THE call surface: the in-app PiP hides, and presenting from
+  the full-screen meeting collapses the meeting into it (restored when the
+  share ends). Control-bar actions (camera toggle / stop share / end call)
+  round-trip `video:popoutAction` → main → `video:popout-action` → app
+  window, which owns the mic/camera/capture. The expand button focuses the
+  main window (`video:focusMain`).
 
 `call`/`meeting` options are disabled unless both voice input (Deepgram) and
 voice output (TTS) are configured. Entering a call saves the user's TTS
@@ -95,7 +102,8 @@ disabled while a call owns the mic.
 
 ## Popout window
 
-- Renderer asks `video:setPopout {show}` (main handler:
+- Shown iff `videoChatMode !== 'off' && screenState === 'live'` (effect in
+  `App.tsx`). Renderer asks `video:setPopout {show}` (main handler:
   `apps/main/src/ipc.ts:1742`); main creates a frameless, `alwaysOnTop`
   ('floating'), all-workspaces BrowserWindow at the top-right of the primary
   display, loading the renderer bundle with `#video-popout`

apps/x/apps/main/src/ipc.ts

@@ -382,16 +382,29 @@ type InvokeHandlers = {
   [K in InvokeChannels]: InvokeHandler<K>;
 };
 
-// Video-mode popout window (shown while screen sharing when the app loses
-// focus) and the last call state pushed by the main window — replayed to the
-// popout when it finishes loading.
+// Video-mode popout window (shown for the whole duration of a screen share,
+// floating over every app including Rowboat itself) and the last call state
+// pushed by the main window — replayed to the popout when it finishes loading.
 let videoPopoutWin: BrowserWindow | null = null;
 let lastVideoPopoutState: {
   ttsState: 'idle' | 'synthesizing' | 'speaking';
   status: 'listening' | 'thinking' | 'speaking' | null;
   cameraOn: boolean;
+  interimText: string | null;
 } | null = null;
 
+// Match only real app windows — getAllWindows() can also contain the popout
+// itself and hidden utility windows (e.g. PDF-export renderers), which must
+// not be shown, focused, or sent app events.
+function findMainAppWindow(): BrowserWindow | undefined {
+  return BrowserWindow.getAllWindows().find((w) => {
+    if (w === videoPopoutWin || w.isDestroyed()) return false;
+    const url = w.webContents.getURL();
+    const isAppWindow = url.startsWith('app://') || url.startsWith('http://localhost');
+    return isAppWindow && !url.includes('#video-popout');
+  });
+}
+
 /**
  * Register all IPC handlers with type safety and runtime validation
  *
@@ -1749,7 +1762,7 @@ export function setupIpcHandlers() {
 
       const workArea = screen.getPrimaryDisplay().workArea;
       const width = 340;
-      const height = 148;
+      const height = 184;
       const ipcDir = path.dirname(fileURLToPath(import.meta.url));
       const preloadPath = app.isPackaged
         ? path.join(ipcDir, '../preload/dist/preload.js')
@@ -1804,15 +1817,7 @@ export function setupIpcHandlers() {
       return {};
     },
     'video:focusMain': async () => {
-      // Match only real app windows — getAllWindows() can also contain the
-      // popout itself and hidden utility windows (e.g. PDF-export renderers),
-      // which must not be shown or focused.
-      const main = BrowserWindow.getAllWindows().find((w) => {
-        if (w === videoPopoutWin || w.isDestroyed()) return false;
-        const url = w.webContents.getURL();
-        const isAppWindow = url.startsWith('app://') || url.startsWith('http://localhost');
-        return isAppWindow && !url.includes('#video-popout');
-      });
+      const main = findMainAppWindow();
       if (main) {
         if (main.isMinimized()) main.restore();
         main.show();
@@ -1820,6 +1825,12 @@ export function setupIpcHandlers() {
       }
       return {};
     },
+    'video:popoutAction': async (_event, args) => {
+      // Relay a popout control-bar action to the app window, which owns the
+      // call (mic, camera, screen capture) and executes it there.
+      findMainAppWindow()?.webContents.send('video:popout-action', args);
+      return {};
+    },
     // Live-note handlers
     'live-note:run': async (_event, args) => {
       const result = await runLiveNoteAgent(args.filePath, 'manual', args.context);

apps/x/apps/renderer/src/App.tsx

@@ -1221,15 +1221,35 @@ function App() {
     voiceRef.current.setPaused(activeIsProcessing || tts.state !== 'idle')
   }, [videoChatMode, activeIsProcessing, tts.state])
 
+  // Presenting from the full-screen call collapses it into the floating
+  // popout (the user needs their screen — possibly Rowboat itself — free to
+  // navigate while sharing). When the share ends, return to full screen.
+  const returnToMeetingAfterShareRef = useRef(false)
+
   // Screen sharing (any video mode): frames of the shared screen ride along
   // with each message next to the webcam frames.
   const handleToggleScreenShare = useCallback(async () => {
     if (video.screenState === 'live') {
       video.stopScreenShare()
     } else {
-      await video.startScreenShare()
+      const ok = await video.startScreenShare()
+      if (ok && videoChatModeRef.current === 'meeting') {
+        returnToMeetingAfterShareRef.current = true
+        void handleVideoModeChange('call')
+      }
     }
-  }, [video])
+  }, [video, handleVideoModeChange])
+
+  // Share ended (from the popout, the PiP, or the OS): restore the
+  // full-screen call if that's where the share was started from.
+  useEffect(() => {
+    if (video.screenState === 'live') return
+    if (!returnToMeetingAfterShareRef.current) return
+    returnToMeetingAfterShareRef.current = false
+    if (videoChatModeRef.current === 'call') {
+      void handleVideoModeChange('meeting')
+    }
+  }, [video.screenState, handleVideoModeChange])
 
   // Meet-style camera mute: video mode (and any screen share) stays on, but
   // no webcam frames are captured while the camera is off.
@@ -1247,40 +1267,39 @@ function App() {
           : 'listening'
       : null
 
-  // Meet-style popout: while screen sharing, losing app focus (the user
-  // switched to the app they're sharing) pops the mini-call out into an
-  // always-on-top window; refocusing Rowboat dismisses it. The short show
-  // delay keeps a quick cmd-tab pass-through from flashing a window.
-  const [appFocused, setAppFocused] = useState(true)
+  // Meet-style popout: the floating always-on-top mini-call appears the
+  // moment screen sharing starts and stays for the whole share — including
+  // over Rowboat itself (the shared screen may BE Rowboat), and across app
+  // switches. It disappears when sharing stops or video mode ends.
   useEffect(() => {
-    const onFocus = () => setAppFocused(true)
-    const onBlur = () => setAppFocused(false)
-    window.addEventListener('focus', onFocus)
-    window.addEventListener('blur', onBlur)
-    return () => {
-      window.removeEventListener('focus', onFocus)
-      window.removeEventListener('blur', onBlur)
-    }
-  }, [])
-  useEffect(() => {
-    const shouldShow = videoChatMode !== 'off' && video.screenState === 'live' && !appFocused
-    if (shouldShow) {
-      const timer = setTimeout(() => {
-        void window.ipc.invoke('video:setPopout', { show: true }).catch(() => {})
-      }, 300)
-      return () => clearTimeout(timer)
-    }
-    void window.ipc.invoke('video:setPopout', { show: false }).catch(() => {})
-  }, [videoChatMode, video.screenState, appFocused])
+    const shouldShow = videoChatMode !== 'off' && video.screenState === 'live'
+    void window.ipc.invoke('video:setPopout', { show: shouldShow }).catch(() => {})
+  }, [videoChatMode, video.screenState])
 
-  // Keep the popout's mascot/status/camera mirror of the call fresh. The main
-  // process caches the latest state and replays it when the popout loads.
+  // Keep the popout's mascot/status/camera/caption mirror of the call fresh.
+  // The main process caches the latest state and replays it when the popout
+  // loads.
   useEffect(() => {
     if (videoChatMode === 'off') return
     void window.ipc
-      .invoke('video:popoutState', { ttsState: tts.state, status: videoCallStatus, cameraOn: video.cameraOn })
+      .invoke('video:popoutState', {
+        ttsState: tts.state,
+        status: videoCallStatus,
+        cameraOn: video.cameraOn,
+        interimText: videoCallStatus ? voice.interimText || null : null,
+      })
       .catch(() => {})
-  }, [videoChatMode, tts.state, videoCallStatus, video.cameraOn])
+  }, [videoChatMode, tts.state, videoCallStatus, video.cameraOn, voice.interimText])
+
+  // Execute popout control-bar actions (the popout window has no access to
+  // the call's mic/camera/capture — they live here).
+  useEffect(() => {
+    return window.ipc.on('video:popout-action', ({ action }) => {
+      if (action === 'toggle-camera') handleToggleCamera()
+      else if (action === 'stop-share') video.stopScreenShare()
+      else if (action === 'end-call') void handleVideoModeChange('off')
+    })
+  }, [handleToggleCamera, video, handleVideoModeChange])
 
   // Enter to submit voice input, Escape to cancel
   useEffect(() => {
@@ -6644,9 +6663,10 @@ function App() {
                 onComposioConnected={handleComposioConnected}
               />
             )}
-            {/* Webcam PiP preview while video chat mode is on (the full-screen
-                meeting view replaces it) */}
-            {(videoChatMode === 'chat' || videoChatMode === 'call') && (
+            {/* Webcam PiP preview while video chat mode is on. Hidden while
+                screen sharing (the floating popout is the call surface then)
+                and in the full-screen meeting view (which replaces it). */}
+            {(videoChatMode === 'chat' || videoChatMode === 'call') && video.screenState !== 'live' && (
               <VideoPreviewOverlay
                 streamRef={video.streamRef}
                 onTurnOff={() => handleVideoModeChange('off')}
@@ -6660,7 +6680,6 @@ function App() {
                     : undefined
                 }
                 interimText={videoChatMode === 'call' ? voice.interimText : undefined}
-                isScreenSharing={video.screenState === 'live'}
                 onToggleScreenShare={handleToggleScreenShare}
                 cameraOn={video.cameraOn}
                 onToggleCamera={handleToggleCamera}

apps/x/apps/renderer/src/components/video-popout.tsx

@@ -1,5 +1,5 @@
 import { useCallback, useEffect, useRef, useState } from 'react'
-import { Maximize2, User } from 'lucide-react'
+import { Maximize2, MonitorUp, PhoneOff, User, Video, VideoOff } from 'lucide-react'
 
 import { TalkingHead } from '@/components/talking-head'
 
@@ -7,6 +7,7 @@ type PopoutState = {
   ttsState: 'idle' | 'synthesizing' | 'speaking'
   status: 'listening' | 'thinking' | 'speaking' | null
   cameraOn: boolean
+  interimText: string | null
 }
 
 const STATUS_DISPLAY: Record<NonNullable<PopoutState['status']>, { label: string; dotClass: string }> = {
@@ -19,14 +20,17 @@ const dragRegion = { WebkitAppRegion: 'drag' } as React.CSSProperties
 const noDragRegion = { WebkitAppRegion: 'no-drag' } as React.CSSProperties
 
 /**
- * Content of the always-on-top popout window shown while screen sharing when
- * the main app window loses focus (Meet-style floating mini-call). Rendered
- * in its own BrowserWindow (see `video:setPopout` in the main process); call
- * state streams in over the `video:popout-state` push channel. Captures its
- * own webcam feed — MediaStreams can't cross windows.
+ * Content of the always-on-top popout window shown for the whole duration of
+ * a screen share (Meet-style floating mini-call) — it floats over every app,
+ * including Rowboat itself, and is the call's control surface while sharing:
+ * camera toggle, stop-share, end-call. Rendered in its own BrowserWindow
+ * (see `video:setPopout` in the main process); call state streams in over
+ * the `video:popout-state` push channel and control actions round-trip back
+ * through `video:popoutAction`. Captures its own webcam feed — MediaStreams
+ * can't cross windows.
  */
 export function VideoPopout() {
-  const [state, setState] = useState<PopoutState>({ ttsState: 'idle', status: null, cameraOn: true })
+  const [state, setState] = useState<PopoutState>({ ttsState: 'idle', status: null, cameraOn: true, interimText: null })
   const videoRef = useRef<HTMLVideoElement | null>(null)
 
   useEffect(() => {
@@ -63,55 +67,103 @@ export function VideoPopout() {
   // so the mascot still animates while the assistant speaks in the main window.
   const getLevel = useCallback(() => 0.45 + 0.35 * Math.sin(performance.now() / 90), [])
 
+  const sendAction = useCallback((action: 'toggle-camera' | 'stop-share' | 'end-call') => {
+    void window.ipc.invoke('video:popoutAction', { action }).catch(() => {})
+  }, [])
+
   const statusDisplay = state.status ? STATUS_DISPLAY[state.status] : null
 
   return (
     <div
-      className="flex h-screen w-screen select-none gap-1.5 bg-neutral-900 p-1.5"
+      className="relative flex h-screen w-screen select-none flex-col gap-1.5 bg-neutral-900 p-1.5"
       style={dragRegion}
     >
-      <div className="relative flex-1 overflow-hidden rounded-lg bg-neutral-800">
-        {state.cameraOn ? (
-          <video
-            ref={videoRef}
-            muted
-            playsInline
-            className="h-full w-full object-cover"
-            style={{ transform: 'scaleX(-1)' }}
-          />
-        ) : (
-          <div className="flex h-full w-full items-center justify-center">
-            <span className="flex h-14 w-14 items-center justify-center rounded-full bg-neutral-700 text-neutral-400">
-              <User className="h-7 w-7" />
+      <div className="flex min-h-0 flex-1 gap-1.5">
+        <div className="relative flex-1 overflow-hidden rounded-lg bg-neutral-800">
+          {state.cameraOn ? (
+            <video
+              ref={videoRef}
+              muted
+              playsInline
+              className="h-full w-full object-cover"
+              style={{ transform: 'scaleX(-1)' }}
+            />
+          ) : (
+            <div className="flex h-full w-full items-center justify-center">
+              <span className="flex h-12 w-12 items-center justify-center rounded-full bg-neutral-700 text-neutral-400">
+                <User className="h-6 w-6" />
+              </span>
+            </div>
+          )}
+          <span className="absolute bottom-1 left-1.5 rounded bg-black/50 px-1 py-px text-[10px] text-white">
+            You
+          </span>
+        </div>
+        <div className="relative flex flex-1 items-center justify-center overflow-hidden rounded-lg bg-neutral-800">
+          <TalkingHead ttsState={state.ttsState} getLevel={getLevel} size={84} />
+          <span className="absolute bottom-1 left-1.5 rounded bg-black/50 px-1 py-px text-[10px] text-white">
+            Rowboat
+          </span>
+          {statusDisplay && (
+            <span className="absolute right-1.5 top-1.5 flex items-center gap-1 rounded-full bg-black/50 px-1.5 py-0.5 text-[10px] font-medium text-white">
+              <span className={`block h-1.5 w-1.5 rounded-full ${statusDisplay.dotClass}`} />
+              {statusDisplay.label}
+            </span>
+          )}
+        </div>
+        {/* Live caption of the in-progress utterance, floating over the tiles */}
+        {state.interimText && (
+          <div className="pointer-events-none absolute inset-x-1.5 bottom-9 flex justify-center">
+            <span className="max-w-full truncate rounded bg-black/70 px-1.5 py-0.5 text-[10px] text-white/90">
+              {state.interimText}
             </span>
           </div>
         )}
-        <span className="absolute bottom-1 left-1.5 rounded bg-black/50 px-1 py-px text-[10px] text-white">
-          You
-        </span>
       </div>
-      <div className="relative flex flex-1 items-center justify-center overflow-hidden rounded-lg bg-neutral-800">
-        <TalkingHead ttsState={state.ttsState} getLevel={getLevel} size={92} />
-        <span className="absolute bottom-1 left-1.5 rounded bg-black/50 px-1 py-px text-[10px] text-white">
-          Rowboat
-        </span>
-        {statusDisplay && (
-          <span className="absolute right-1.5 top-1.5 flex items-center gap-1 rounded-full bg-black/50 px-1.5 py-0.5 text-[10px] font-medium text-white">
-            <span className={`block h-1.5 w-1.5 rounded-full ${statusDisplay.dotClass}`} />
-            {statusDisplay.label}
-          </span>
-        )}
+
+      {/* Control bar — actions execute in the main app window */}
+      <div className="flex h-7 shrink-0 items-center justify-center gap-2" style={noDragRegion}>
+        <button
+          type="button"
+          onClick={() => sendAction('toggle-camera')}
+          className={`flex h-6 w-6 items-center justify-center rounded-full transition-colors ${
+            state.cameraOn
+              ? 'bg-neutral-700 text-white/90 hover:bg-neutral-600'
+              : 'bg-red-600 text-white hover:bg-red-500'
+          }`}
+          aria-label={state.cameraOn ? 'Turn off camera' : 'Turn on camera'}
+          title={state.cameraOn ? 'Turn off camera' : 'Turn on camera'}
+        >
+          {state.cameraOn ? <Video className="h-3.5 w-3.5" /> : <VideoOff className="h-3.5 w-3.5" />}
+        </button>
+        <button
+          type="button"
+          onClick={() => sendAction('stop-share')}
+          className="flex h-6 w-6 items-center justify-center rounded-full bg-sky-600 text-white transition-colors hover:bg-sky-500"
+          aria-label="Stop sharing screen"
+          title="Stop sharing screen"
+        >
+          <MonitorUp className="h-3.5 w-3.5" />
+        </button>
+        <button
+          type="button"
+          onClick={() => sendAction('end-call')}
+          className="flex h-6 w-8 items-center justify-center rounded-full bg-red-600 text-white transition-colors hover:bg-red-500"
+          aria-label="End video mode"
+          title="End video mode"
+        >
+          <PhoneOff className="h-3.5 w-3.5" />
+        </button>
+        <button
+          type="button"
+          onClick={() => void window.ipc.invoke('video:focusMain', null)}
+          className="flex h-6 w-6 items-center justify-center rounded-full bg-neutral-700 text-white/90 transition-colors hover:bg-neutral-600"
+          aria-label="Back to Rowboat"
+          title="Back to Rowboat"
+        >
+          <Maximize2 className="h-3.5 w-3.5" />
+        </button>
       </div>
-      <button
-        type="button"
-        onClick={() => void window.ipc.invoke('video:focusMain', null)}
-        className="absolute left-1.5 top-1.5 flex h-5 w-5 items-center justify-center rounded bg-black/50 text-white/80 hover:text-white"
-        style={noDragRegion}
-        aria-label="Back to Rowboat"
-        title="Back to Rowboat"
-      >
-        <Maximize2 className="h-3 w-3" />
-      </button>
     </div>
   )
 }

apps/x/packages/shared/src/ipc.ts

@@ -1380,8 +1380,9 @@ const ipcSchemas = {
       granted: z.boolean(),
     }),
   },
-  // Video-mode popout: show/hide a small always-on-top window (user + mascot
-  // tiles) while screen sharing when the main window loses focus, Meet-style.
+  // Video-mode popout: show/hide the small always-on-top window (user +
+  // mascot tiles) that floats over everything for the duration of a screen
+  // share, Meet-style.
   'video:setPopout': {
     req: z.object({ show: z.boolean() }),
     res: z.object({}),
@@ -1393,6 +1394,8 @@ const ipcSchemas = {
       ttsState: z.enum(['idle', 'synthesizing', 'speaking']),
       status: z.enum(['listening', 'thinking', 'speaking']).nullable(),
       cameraOn: z.boolean(),
+      // Live transcript of the in-progress utterance (hands-free call only).
+      interimText: z.string().nullable(),
     }),
     res: z.object({}),
   },
@@ -1401,12 +1404,28 @@ const ipcSchemas = {
     req: z.null(),
     res: z.object({}),
   },
+  // Popout control bar → main process → relayed to the app window, which
+  // executes the action on the live call.
+  'video:popoutAction': {
+    req: z.object({
+      action: z.enum(['toggle-camera', 'stop-share', 'end-call']),
+    }),
+    res: z.object({}),
+  },
   // Push channel: main → popout window with the latest call state.
   'video:popout-state': {
     req: z.object({
       ttsState: z.enum(['idle', 'synthesizing', 'speaking']),
       status: z.enum(['listening', 'thinking', 'speaking']).nullable(),
       cameraOn: z.boolean(),
+      interimText: z.string().nullable(),
+    }),
+    res: z.null(),
+  },
+  // Push channel: main → app window with a popout control-bar action.
+  'video:popout-action': {
+    req: z.object({
+      action: z.enum(['toggle-camera', 'stop-share', 'end-call']),
     }),
     res: z.null(),
   },