full video call mode

apps/x/apps/renderer/src/App.tsx

@@ -12,7 +12,7 @@ import { ChatSidebar } from './components/chat-sidebar';
 import { useSessionChat } from '@/hooks/useSessionChat';
 import { ChatHeader } from './components/chat-header';
 import { ChatEmptyState } from './components/chat-empty-state';
-import { ChatInputWithMentions, type PermissionMode, type StagedAttachment } from './components/chat-input-with-mentions';
+import { ChatInputWithMentions, type PermissionMode, type StagedAttachment, type VideoChatMode } from './components/chat-input-with-mentions';
 import { ChatMessageAttachments } from '@/components/chat-message-attachments'
 import { GraphView, type GraphEdge, type GraphNode } from '@/components/graph-view';
 import { BasesView, type BaseConfig, DEFAULT_BASE_CONFIG } from '@/components/bases-view';
@@ -122,6 +122,7 @@ import { useVideoMode } from '@/hooks/useVideoMode'
 import { useVoiceTTS } from '@/hooks/useVoiceTTS'
 import { TalkingHeadOverlay } from '@/components/talking-head'
 import { VideoPreviewOverlay } from '@/components/video-preview-overlay'
+import { VideoCallView } from '@/components/video-call-view'
 import { ProductTour, type TourNavTarget } from '@/components/product-tour'
 import { useMeetingTranscription, type CalendarEventMeta } from '@/hooks/useMeetingTranscription'
 import { useAnalyticsIdentity } from '@/hooks/useAnalyticsIdentity'
@@ -986,6 +987,13 @@ function App() {
   const ttsRef = useRef(tts)
   ttsRef.current = tts
 
+  // Latest assistant line handed to TTS — shown as the caption in the
+  // full-screen call view while the assistant is speaking.
+  const [assistantCaption, setAssistantCaption] = useState('')
+  useEffect(() => {
+    if (tts.state === 'idle') setAssistantCaption('')
+  }, [tts.state])
+
   // Speak newly completed <voice> blocks from the new runtime's live stream
   // (parity with the legacy text-delta voice extraction below). The store
   // accumulates completed blocks in chatState.voiceSegments; we speak only
@@ -1004,6 +1012,7 @@ function App() {
       spokenVoiceRef.current.count += 1
       if (ttsEnabledRef.current) {
         ttsRef.current.speak(segment)
+        setAssistantCaption(segment)
       }
     }
   }, [voiceSegments, runId])
@@ -1018,8 +1027,8 @@ function App() {
   // auto-submitted utterances, spoken responses (handler defined below, after
   // the voice/submit plumbing it drives).
   const video = useVideoMode()
-  const [videoChatMode, setVideoChatMode] = useState<'off' | 'chat' | 'call'>('off')
+  const [videoChatMode, setVideoChatMode] = useState<VideoChatMode>('off')
-  const videoChatModeRef = useRef<'off' | 'chat' | 'call'>('off')
+  const videoChatModeRef = useRef<VideoChatMode>('off')
   // TTS settings to restore when a hands-free call ends (a call forces
   // full-read-aloud TTS for its duration).
   const preCallTtsRef = useRef<{ enabled: boolean; mode: 'summary' | 'full' } | null>(null)
@@ -1083,7 +1092,7 @@ function App() {
 
   const handleStartRecording = useCallback(() => {
     // Hands-free call mode owns the mic — ignore push-to-talk while it's on.
-    if (videoChatModeRef.current === 'call') return
+    if (videoChatModeRef.current === 'call' || videoChatModeRef.current === 'meeting') return
     setIsRecording(true)
     isRecordingRef.current = true
     voice.start()
@@ -1145,15 +1154,18 @@ function App() {
     isRecordingRef.current = false
   }, [voice])
 
-  // Video chat mode transitions. 'chat' just runs the camera; 'call' also
+  // Video chat mode transitions. 'chat' just runs the camera; 'call' and
-  // listens continuously (auto-submitting each utterance as a voice message)
+  // 'meeting' also listen continuously (auto-submitting each utterance as a
-  // and forces full read-aloud TTS so the assistant answers out loud.
+  // voice message) and force full read-aloud TTS so the assistant answers out
-  const handleVideoModeChange = useCallback(async (mode: 'off' | 'chat' | 'call') => {
+  // loud. 'meeting' is the same pipeline presented as a full-screen call.
+  const handleVideoModeChange = useCallback(async (mode: VideoChatMode) => {
     const prev = videoChatModeRef.current
     if (mode === prev) return
+    const wasHandsFree = prev === 'call' || prev === 'meeting'
+    const isHandsFree = mode === 'call' || mode === 'meeting'
 
-    // Leaving a call: release the mic and restore the pre-call TTS settings.
+    // Leaving hands-free: release the mic and restore the pre-call TTS settings.
-    if (prev === 'call') {
+    if (wasHandsFree && !isHandsFree) {
       voiceRef.current.cancel()
       const saved = preCallTtsRef.current
       preCallTtsRef.current = null
@@ -1179,7 +1191,8 @@ function App() {
       if (!ok) return // camera denied/unavailable — stay off
     }
 
-    if (mode === 'call') {
+    // Entering hands-free (call ↔ meeting switches keep the mic/TTS as-is).
+    if (isHandsFree && !wasHandsFree) {
       // A manual push-to-talk recording can't coexist with the call's mic.
       if (isRecordingRef.current) {
         voiceRef.current.cancel()
@@ -1204,7 +1217,7 @@ function App() {
   // During a call, mute the mic while the assistant is thinking or speaking
   // so its own TTS (or a half-turn) never gets transcribed back at it.
   useEffect(() => {
-    if (videoChatMode !== 'call') return
+    if (videoChatMode !== 'call' && videoChatMode !== 'meeting') return
     voiceRef.current.setPaused(activeIsProcessing || tts.state !== 'idle')
   }, [videoChatMode, activeIsProcessing, tts.state])
 
@@ -2332,6 +2345,7 @@ function App() {
               console.log('[voice] extracted voice tag:', voiceContent)
               if (voiceContent && ttsEnabledRef.current) {
                 ttsRef.current.speak(voiceContent)
+                setAssistantCaption(voiceContent)
               }
               spokenIndexRef.current += voiceMatch.index + voiceMatch[0].length
             }
@@ -6569,8 +6583,9 @@ function App() {
                 onComposioConnected={handleComposioConnected}
               />
             )}
-            {/* Webcam PiP preview while video chat mode is on */}
+            {/* Webcam PiP preview while video chat mode is on (the full-screen
-            {videoChatMode !== 'off' && (
+                meeting view replaces it) */}
+            {(videoChatMode === 'chat' || videoChatMode === 'call') && (
               <VideoPreviewOverlay
                 streamRef={video.streamRef}
                 onTurnOff={() => handleVideoModeChange('off')}
@@ -6586,6 +6601,24 @@ function App() {
                 interimText={videoChatMode === 'call' ? voice.interimText : undefined}
               />
             )}
+            {/* Full-screen Meet-style call: user tile + animated mascot tile */}
+            {videoChatMode === 'meeting' && (
+              <VideoCallView
+                streamRef={video.streamRef}
+                ttsState={tts.state}
+                getTtsLevel={tts.getLevel}
+                status={
+                  tts.state === 'speaking'
+                    ? 'speaking'
+                    : tts.state === 'synthesizing' || activeIsProcessing
+                      ? 'thinking'
+                      : 'listening'
+                }
+                interimText={voice.interimText}
+                assistantCaption={assistantCaption}
+                onLeave={() => handleVideoModeChange('off')}
+              />
+            )}
             {/* Talking head hovers over the active view while avatar voice mode is
                 on (hidden during the tour, which shows its own mascot) */}
             {ttsAvatarEnabled && !tourActive && (

apps/x/apps/renderer/src/components/chat-input-with-mentions.tsx

@@ -212,6 +212,8 @@ function compactWorkDirPath(path: string) {
   return path.replace(/^\/Users\/[^/]+/, '~')
 }
 
+export type VideoChatMode = 'off' | 'chat' | 'call' | 'meeting'
+
 interface ChatInputInnerProps {
   onSubmit: (message: PromptInputMessage, mentions?: FileMention[], attachments?: StagedAttachment[], searchEnabled?: boolean, codeMode?: 'claude' | 'codex', permissionMode?: PermissionMode) => void
   onStop?: () => void
@@ -239,10 +241,11 @@ interface ChatInputInnerProps {
   onTtsModeChange?: (mode: 'summary' | 'full') => void
   ttsAvatarEnabled?: boolean
   onToggleTtsAvatar?: () => void
-  /** Video chat mode: 'chat' attaches webcam frames to messages; 'call' is
+  /** Video chat mode: 'chat' attaches webcam frames to messages; 'call' and
-   *  fully hands-free — continuous listening, spoken responses. */
+   *  'meeting' are fully hands-free — continuous listening, spoken responses
-  videoChatMode?: 'off' | 'chat' | 'call'
+   *  ('meeting' additionally takes over the whole screen, Meet-style). */
-  onVideoModeChange?: (mode: 'off' | 'chat' | 'call') => void
+  videoChatMode?: VideoChatMode
+  onVideoModeChange?: (mode: VideoChatMode) => void
   /** Hands-free call needs both voice input (STT) and voice output (TTS). */
   videoCallAvailable?: boolean
   /** Fired when the user picks a different model in the dropdown (only when no run exists yet). */
@@ -1404,12 +1407,15 @@ function ChatInputInner({
                 <DropdownMenuContent align="end">
                   <DropdownMenuRadioGroup
                     value={videoChatMode}
-                    onValueChange={(v) => onVideoModeChange(v as 'chat' | 'call')}
+                    onValueChange={(v) => onVideoModeChange(v as VideoChatMode)}
                   >
                     <DropdownMenuRadioItem value="chat">Video + chat</DropdownMenuRadioItem>
                     <DropdownMenuRadioItem value="call" disabled={!videoCallAvailable}>
                       Video call (hands-free)
                     </DropdownMenuRadioItem>
+                    <DropdownMenuRadioItem value="meeting" disabled={!videoCallAvailable}>
+                      Video call (full screen)
+                    </DropdownMenuRadioItem>
                   </DropdownMenuRadioGroup>
                 </DropdownMenuContent>
               </DropdownMenu>
@@ -1592,8 +1598,8 @@ export interface ChatInputWithMentionsProps {
   onTtsModeChange?: (mode: 'summary' | 'full') => void
   ttsAvatarEnabled?: boolean
   onToggleTtsAvatar?: () => void
-  videoChatMode?: 'off' | 'chat' | 'call'
+  videoChatMode?: VideoChatMode
-  onVideoModeChange?: (mode: 'off' | 'chat' | 'call') => void
+  onVideoModeChange?: (mode: VideoChatMode) => void
   videoCallAvailable?: boolean
   onSelectedModelChange?: (model: SelectedModel | null) => void
   workDir?: string | null

apps/x/apps/renderer/src/components/chat-sidebar.tsx

@@ -37,7 +37,7 @@ import { MarkdownPreOverride } from '@/components/ai-elements/markdown-code-over
 import { defaultRemarkPlugins } from 'streamdown'
 import remarkBreaks from 'remark-breaks'
 import { type ChatTab } from '@/components/tab-bar'
-import { ChatInputWithMentions, type PermissionMode, type StagedAttachment, type SelectedModel } from '@/components/chat-input-with-mentions'
+import { ChatInputWithMentions, type PermissionMode, type StagedAttachment, type SelectedModel, type VideoChatMode } from '@/components/chat-input-with-mentions'
 import { ChatMessageAttachments } from '@/components/chat-message-attachments'
 import { useSidebar } from '@/components/ui/sidebar'
 import { wikiLabel } from '@/lib/wiki-links'
@@ -194,8 +194,8 @@ interface ChatSidebarProps {
   onTtsModeChange?: (mode: 'summary' | 'full') => void
   ttsAvatarEnabled?: boolean
   onToggleTtsAvatar?: () => void
-  videoChatMode?: 'off' | 'chat' | 'call'
+  videoChatMode?: VideoChatMode
-  onVideoModeChange?: (mode: 'off' | 'chat' | 'call') => void
+  onVideoModeChange?: (mode: VideoChatMode) => void
   videoCallAvailable?: boolean
   onComposioConnected?: (toolkitSlug: string) => void
 }

apps/x/apps/renderer/src/components/video-call-view.tsx

@@ -0,0 +1,162 @@
+import { useEffect, useRef, useState } from 'react'
+import { PhoneOff } from 'lucide-react'
+
+import { MascotFaceIcon, TalkingHead } from '@/components/talking-head'
+import type { TTSState } from '@/hooks/useVoiceTTS'
+import { cn } from '@/lib/utils'
+
+export type VideoCallStatus = 'listening' | 'thinking' | 'speaking'
+
+interface VideoCallViewProps {
+  /** Live camera stream from useVideoMode — attached to the user's tile. */
+  streamRef: React.MutableRefObject<MediaStream | null>
+  ttsState: TTSState
+  /** Live TTS output level — drives the mascot's mouth animation. */
+  getTtsLevel: () => number
+  status: VideoCallStatus
+  /** Live transcript of the user's in-progress utterance. */
+  interimText?: string
+  /** The assistant line currently being spoken aloud. */
+  assistantCaption?: string
+  onLeave: () => void
+}
+
+const STATUS_DISPLAY: Record<VideoCallStatus, { label: string; dotClass: string }> = {
+  listening: { label: 'Listening', dotClass: 'bg-green-500 animate-pulse' },
+  thinking: { label: 'Thinking…', dotClass: 'bg-amber-400' },
+  speaking: { label: 'Speaking', dotClass: 'bg-sky-400 animate-pulse' },
+}
+
+/**
+ * Full-screen hands-free call: a Meet-style two-tile layout with the user's
+ * webcam on one side and the mascot as the other participant. The mascot
+ * animates with the assistant's speech; dismissing it swaps in a Meet-style
+ * letter avatar ("R"). Live captions run along the bottom.
+ */
+export function VideoCallView({
+  streamRef,
+  ttsState,
+  getTtsLevel,
+  status,
+  interimText,
+  assistantCaption,
+  onLeave,
+}: VideoCallViewProps) {
+  const videoRef = useRef<HTMLVideoElement | null>(null)
+  const [mascotVisible, setMascotVisible] = useState(true)
+
+  useEffect(() => {
+    const videoEl = videoRef.current
+    if (!videoEl) return
+    videoEl.srcObject = streamRef.current
+    videoEl.play().catch(() => {})
+    return () => {
+      videoEl.srcObject = null
+    }
+  }, [streamRef])
+
+  const userSpeaking = status === 'listening' && Boolean(interimText)
+  const assistantSpeaking = ttsState === 'speaking'
+
+  const caption = assistantSpeaking && assistantCaption
+    ? { who: 'Rowboat', text: assistantCaption }
+    : interimText
+      ? { who: 'You', text: interimText }
+      : null
+
+  return (
+    <div className="fixed inset-0 z-[100] flex flex-col bg-neutral-950">
+      {/* Participant tiles */}
+      <div className="grid min-h-0 flex-1 grid-cols-1 gap-3 p-4 pb-2 md:grid-cols-2">
+        {/* User */}
+        <div
+          className={cn(
+            'relative flex items-center justify-center overflow-hidden rounded-2xl bg-neutral-900 transition-shadow',
+            userSpeaking && 'ring-2 ring-green-500/80'
+          )}
+        >
+          <video
+            ref={videoRef}
+            muted
+            playsInline
+            className="h-full w-full object-cover"
+            style={{ transform: 'scaleX(-1)' }}
+          />
+          <span className="absolute bottom-3 left-3 rounded-md bg-black/50 px-2 py-0.5 text-sm text-white">
+            You
+          </span>
+        </div>
+
+        {/* Assistant */}
+        <div
+          className={cn(
+            'group relative flex items-center justify-center overflow-hidden rounded-2xl bg-neutral-900 transition-shadow',
+            assistantSpeaking && 'ring-2 ring-sky-400/80'
+          )}
+        >
+          {mascotVisible ? (
+            <TalkingHead ttsState={ttsState} getLevel={getTtsLevel} size={220} />
+          ) : (
+            <span
+              className="flex h-40 w-40 items-center justify-center rounded-full bg-sky-600 text-7xl font-medium text-white"
+              aria-label="Rowboat"
+            >
+              R
+            </span>
+          )}
+          <span className="absolute bottom-3 left-3 rounded-md bg-black/50 px-2 py-0.5 text-sm text-white">
+            Rowboat
+          </span>
+          <button
+            type="button"
+            onClick={() => setMascotVisible((v) => !v)}
+            className="absolute right-3 top-3 rounded-md bg-black/50 px-2 py-1 text-xs text-white/80 opacity-0 transition-opacity hover:text-white group-hover:opacity-100"
+          >
+            {mascotVisible ? 'Hide mascot' : 'Show mascot'}
+          </button>
+        </div>
+      </div>
+
+      {/* Captions */}
+      <div className="flex h-14 items-center justify-center px-6">
+        {caption && (
+          <div className="max-w-3xl truncate rounded-lg bg-black/60 px-4 py-2 text-sm text-white/90">
+            <span className="mr-2 font-semibold text-white">{caption.who}:</span>
+            {caption.text}
+          </div>
+        )}
+      </div>
+
+      {/* Control bar */}
+      <div className="flex items-center justify-center gap-4 pb-5">
+        <span className="flex items-center gap-2 rounded-full bg-neutral-800 px-3 py-1.5 text-xs font-medium text-white/90">
+          <span className={cn('block h-2 w-2 rounded-full', STATUS_DISPLAY[status].dotClass)} />
+          {STATUS_DISPLAY[status].label}
+        </span>
+        <button
+          type="button"
+          onClick={() => setMascotVisible((v) => !v)}
+          className={cn(
+            'relative flex h-10 w-10 items-center justify-center rounded-full bg-neutral-800 text-white/90 transition-colors hover:bg-neutral-700',
+          )}
+          aria-label={mascotVisible ? 'Hide mascot' : 'Show mascot'}
+        >
+          <MascotFaceIcon />
+          {!mascotVisible && (
+            <span className="absolute inset-0 flex items-center justify-center pointer-events-none">
+              <span className="block h-[1.5px] w-6 -rotate-45 rounded-full bg-white/80" />
+            </span>
+          )}
+        </button>
+        <button
+          type="button"
+          onClick={onLeave}
+          className="flex h-10 w-14 items-center justify-center rounded-full bg-red-600 text-white transition-colors hover:bg-red-500"
+          aria-label="Leave call"
+        >
+          <PhoneOff className="h-5 w-5" />
+        </button>
+      </div>
+    </div>
+  )
+}