From 42353c63f4d22c7ec070cc40dd8cf00c4b992cb0 Mon Sep 17 00:00:00 2001 From: Arjun <6592213+arkml@users.noreply.github.com> Date: Fri, 3 Jul 2026 23:32:14 +0530 Subject: [PATCH] full video call mode --- apps/x/apps/renderer/src/App.tsx | 61 +++++-- .../components/chat-input-with-mentions.tsx | 20 ++- .../renderer/src/components/chat-sidebar.tsx | 6 +- .../src/components/video-call-view.tsx | 162 ++++++++++++++++++ 4 files changed, 225 insertions(+), 24 deletions(-) create mode 100644 apps/x/apps/renderer/src/components/video-call-view.tsx diff --git a/apps/x/apps/renderer/src/App.tsx b/apps/x/apps/renderer/src/App.tsx index ddb3483a..c7cc4c80 100644 --- a/apps/x/apps/renderer/src/App.tsx +++ b/apps/x/apps/renderer/src/App.tsx @@ -12,7 +12,7 @@ import { ChatSidebar } from './components/chat-sidebar'; import { useSessionChat } from '@/hooks/useSessionChat'; import { ChatHeader } from './components/chat-header'; import { ChatEmptyState } from './components/chat-empty-state'; -import { ChatInputWithMentions, type PermissionMode, type StagedAttachment } from './components/chat-input-with-mentions'; +import { ChatInputWithMentions, type PermissionMode, type StagedAttachment, type VideoChatMode } from './components/chat-input-with-mentions'; import { ChatMessageAttachments } from '@/components/chat-message-attachments' import { GraphView, type GraphEdge, type GraphNode } from '@/components/graph-view'; import { BasesView, type BaseConfig, DEFAULT_BASE_CONFIG } from '@/components/bases-view'; @@ -122,6 +122,7 @@ import { useVideoMode } from '@/hooks/useVideoMode' import { useVoiceTTS } from '@/hooks/useVoiceTTS' import { TalkingHeadOverlay } from '@/components/talking-head' import { VideoPreviewOverlay } from '@/components/video-preview-overlay' +import { VideoCallView } from '@/components/video-call-view' import { ProductTour, type TourNavTarget } from '@/components/product-tour' import { useMeetingTranscription, type CalendarEventMeta } from '@/hooks/useMeetingTranscription' import { useAnalyticsIdentity } from '@/hooks/useAnalyticsIdentity' @@ -986,6 +987,13 @@ function App() { const ttsRef = useRef(tts) ttsRef.current = tts + // Latest assistant line handed to TTS — shown as the caption in the + // full-screen call view while the assistant is speaking. + const [assistantCaption, setAssistantCaption] = useState('') + useEffect(() => { + if (tts.state === 'idle') setAssistantCaption('') + }, [tts.state]) + // Speak newly completed blocks from the new runtime's live stream // (parity with the legacy text-delta voice extraction below). The store // accumulates completed blocks in chatState.voiceSegments; we speak only @@ -1004,6 +1012,7 @@ function App() { spokenVoiceRef.current.count += 1 if (ttsEnabledRef.current) { ttsRef.current.speak(segment) + setAssistantCaption(segment) } } }, [voiceSegments, runId]) @@ -1018,8 +1027,8 @@ function App() { // auto-submitted utterances, spoken responses (handler defined below, after // the voice/submit plumbing it drives). const video = useVideoMode() - const [videoChatMode, setVideoChatMode] = useState<'off' | 'chat' | 'call'>('off') - const videoChatModeRef = useRef<'off' | 'chat' | 'call'>('off') + const [videoChatMode, setVideoChatMode] = useState('off') + const videoChatModeRef = useRef('off') // TTS settings to restore when a hands-free call ends (a call forces // full-read-aloud TTS for its duration). const preCallTtsRef = useRef<{ enabled: boolean; mode: 'summary' | 'full' } | null>(null) @@ -1083,7 +1092,7 @@ function App() { const handleStartRecording = useCallback(() => { // Hands-free call mode owns the mic — ignore push-to-talk while it's on. - if (videoChatModeRef.current === 'call') return + if (videoChatModeRef.current === 'call' || videoChatModeRef.current === 'meeting') return setIsRecording(true) isRecordingRef.current = true voice.start() @@ -1145,15 +1154,18 @@ function App() { isRecordingRef.current = false }, [voice]) - // Video chat mode transitions. 'chat' just runs the camera; 'call' also - // listens continuously (auto-submitting each utterance as a voice message) - // and forces full read-aloud TTS so the assistant answers out loud. - const handleVideoModeChange = useCallback(async (mode: 'off' | 'chat' | 'call') => { + // Video chat mode transitions. 'chat' just runs the camera; 'call' and + // 'meeting' also listen continuously (auto-submitting each utterance as a + // voice message) and force full read-aloud TTS so the assistant answers out + // loud. 'meeting' is the same pipeline presented as a full-screen call. + const handleVideoModeChange = useCallback(async (mode: VideoChatMode) => { const prev = videoChatModeRef.current if (mode === prev) return + const wasHandsFree = prev === 'call' || prev === 'meeting' + const isHandsFree = mode === 'call' || mode === 'meeting' - // Leaving a call: release the mic and restore the pre-call TTS settings. - if (prev === 'call') { + // Leaving hands-free: release the mic and restore the pre-call TTS settings. + if (wasHandsFree && !isHandsFree) { voiceRef.current.cancel() const saved = preCallTtsRef.current preCallTtsRef.current = null @@ -1179,7 +1191,8 @@ function App() { if (!ok) return // camera denied/unavailable — stay off } - if (mode === 'call') { + // Entering hands-free (call ↔ meeting switches keep the mic/TTS as-is). + if (isHandsFree && !wasHandsFree) { // A manual push-to-talk recording can't coexist with the call's mic. if (isRecordingRef.current) { voiceRef.current.cancel() @@ -1204,7 +1217,7 @@ function App() { // During a call, mute the mic while the assistant is thinking or speaking // so its own TTS (or a half-turn) never gets transcribed back at it. useEffect(() => { - if (videoChatMode !== 'call') return + if (videoChatMode !== 'call' && videoChatMode !== 'meeting') return voiceRef.current.setPaused(activeIsProcessing || tts.state !== 'idle') }, [videoChatMode, activeIsProcessing, tts.state]) @@ -2332,6 +2345,7 @@ function App() { console.log('[voice] extracted voice tag:', voiceContent) if (voiceContent && ttsEnabledRef.current) { ttsRef.current.speak(voiceContent) + setAssistantCaption(voiceContent) } spokenIndexRef.current += voiceMatch.index + voiceMatch[0].length } @@ -6569,8 +6583,9 @@ function App() { onComposioConnected={handleComposioConnected} /> )} - {/* Webcam PiP preview while video chat mode is on */} - {videoChatMode !== 'off' && ( + {/* Webcam PiP preview while video chat mode is on (the full-screen + meeting view replaces it) */} + {(videoChatMode === 'chat' || videoChatMode === 'call') && ( handleVideoModeChange('off')} @@ -6586,6 +6601,24 @@ function App() { interimText={videoChatMode === 'call' ? voice.interimText : undefined} /> )} + {/* Full-screen Meet-style call: user tile + animated mascot tile */} + {videoChatMode === 'meeting' && ( + handleVideoModeChange('off')} + /> + )} {/* Talking head hovers over the active view while avatar voice mode is on (hidden during the tour, which shows its own mascot) */} {ttsAvatarEnabled && !tourActive && ( diff --git a/apps/x/apps/renderer/src/components/chat-input-with-mentions.tsx b/apps/x/apps/renderer/src/components/chat-input-with-mentions.tsx index 7dd73aed..a97365d5 100644 --- a/apps/x/apps/renderer/src/components/chat-input-with-mentions.tsx +++ b/apps/x/apps/renderer/src/components/chat-input-with-mentions.tsx @@ -212,6 +212,8 @@ function compactWorkDirPath(path: string) { return path.replace(/^\/Users\/[^/]+/, '~') } +export type VideoChatMode = 'off' | 'chat' | 'call' | 'meeting' + interface ChatInputInnerProps { onSubmit: (message: PromptInputMessage, mentions?: FileMention[], attachments?: StagedAttachment[], searchEnabled?: boolean, codeMode?: 'claude' | 'codex', permissionMode?: PermissionMode) => void onStop?: () => void @@ -239,10 +241,11 @@ interface ChatInputInnerProps { onTtsModeChange?: (mode: 'summary' | 'full') => void ttsAvatarEnabled?: boolean onToggleTtsAvatar?: () => void - /** Video chat mode: 'chat' attaches webcam frames to messages; 'call' is - * fully hands-free — continuous listening, spoken responses. */ - videoChatMode?: 'off' | 'chat' | 'call' - onVideoModeChange?: (mode: 'off' | 'chat' | 'call') => void + /** Video chat mode: 'chat' attaches webcam frames to messages; 'call' and + * 'meeting' are fully hands-free — continuous listening, spoken responses + * ('meeting' additionally takes over the whole screen, Meet-style). */ + videoChatMode?: VideoChatMode + onVideoModeChange?: (mode: VideoChatMode) => void /** Hands-free call needs both voice input (STT) and voice output (TTS). */ videoCallAvailable?: boolean /** Fired when the user picks a different model in the dropdown (only when no run exists yet). */ @@ -1404,12 +1407,15 @@ function ChatInputInner({ onVideoModeChange(v as 'chat' | 'call')} + onValueChange={(v) => onVideoModeChange(v as VideoChatMode)} > Video + chat Video call (hands-free) + + Video call (full screen) + @@ -1592,8 +1598,8 @@ export interface ChatInputWithMentionsProps { onTtsModeChange?: (mode: 'summary' | 'full') => void ttsAvatarEnabled?: boolean onToggleTtsAvatar?: () => void - videoChatMode?: 'off' | 'chat' | 'call' - onVideoModeChange?: (mode: 'off' | 'chat' | 'call') => void + videoChatMode?: VideoChatMode + onVideoModeChange?: (mode: VideoChatMode) => void videoCallAvailable?: boolean onSelectedModelChange?: (model: SelectedModel | null) => void workDir?: string | null diff --git a/apps/x/apps/renderer/src/components/chat-sidebar.tsx b/apps/x/apps/renderer/src/components/chat-sidebar.tsx index baa198fe..c32bac18 100644 --- a/apps/x/apps/renderer/src/components/chat-sidebar.tsx +++ b/apps/x/apps/renderer/src/components/chat-sidebar.tsx @@ -37,7 +37,7 @@ import { MarkdownPreOverride } from '@/components/ai-elements/markdown-code-over import { defaultRemarkPlugins } from 'streamdown' import remarkBreaks from 'remark-breaks' import { type ChatTab } from '@/components/tab-bar' -import { ChatInputWithMentions, type PermissionMode, type StagedAttachment, type SelectedModel } from '@/components/chat-input-with-mentions' +import { ChatInputWithMentions, type PermissionMode, type StagedAttachment, type SelectedModel, type VideoChatMode } from '@/components/chat-input-with-mentions' import { ChatMessageAttachments } from '@/components/chat-message-attachments' import { useSidebar } from '@/components/ui/sidebar' import { wikiLabel } from '@/lib/wiki-links' @@ -194,8 +194,8 @@ interface ChatSidebarProps { onTtsModeChange?: (mode: 'summary' | 'full') => void ttsAvatarEnabled?: boolean onToggleTtsAvatar?: () => void - videoChatMode?: 'off' | 'chat' | 'call' - onVideoModeChange?: (mode: 'off' | 'chat' | 'call') => void + videoChatMode?: VideoChatMode + onVideoModeChange?: (mode: VideoChatMode) => void videoCallAvailable?: boolean onComposioConnected?: (toolkitSlug: string) => void } diff --git a/apps/x/apps/renderer/src/components/video-call-view.tsx b/apps/x/apps/renderer/src/components/video-call-view.tsx new file mode 100644 index 00000000..487e9c07 --- /dev/null +++ b/apps/x/apps/renderer/src/components/video-call-view.tsx @@ -0,0 +1,162 @@ +import { useEffect, useRef, useState } from 'react' +import { PhoneOff } from 'lucide-react' + +import { MascotFaceIcon, TalkingHead } from '@/components/talking-head' +import type { TTSState } from '@/hooks/useVoiceTTS' +import { cn } from '@/lib/utils' + +export type VideoCallStatus = 'listening' | 'thinking' | 'speaking' + +interface VideoCallViewProps { + /** Live camera stream from useVideoMode — attached to the user's tile. */ + streamRef: React.MutableRefObject + ttsState: TTSState + /** Live TTS output level — drives the mascot's mouth animation. */ + getTtsLevel: () => number + status: VideoCallStatus + /** Live transcript of the user's in-progress utterance. */ + interimText?: string + /** The assistant line currently being spoken aloud. */ + assistantCaption?: string + onLeave: () => void +} + +const STATUS_DISPLAY: Record = { + listening: { label: 'Listening', dotClass: 'bg-green-500 animate-pulse' }, + thinking: { label: 'Thinking…', dotClass: 'bg-amber-400' }, + speaking: { label: 'Speaking', dotClass: 'bg-sky-400 animate-pulse' }, +} + +/** + * Full-screen hands-free call: a Meet-style two-tile layout with the user's + * webcam on one side and the mascot as the other participant. The mascot + * animates with the assistant's speech; dismissing it swaps in a Meet-style + * letter avatar ("R"). Live captions run along the bottom. + */ +export function VideoCallView({ + streamRef, + ttsState, + getTtsLevel, + status, + interimText, + assistantCaption, + onLeave, +}: VideoCallViewProps) { + const videoRef = useRef(null) + const [mascotVisible, setMascotVisible] = useState(true) + + useEffect(() => { + const videoEl = videoRef.current + if (!videoEl) return + videoEl.srcObject = streamRef.current + videoEl.play().catch(() => {}) + return () => { + videoEl.srcObject = null + } + }, [streamRef]) + + const userSpeaking = status === 'listening' && Boolean(interimText) + const assistantSpeaking = ttsState === 'speaking' + + const caption = assistantSpeaking && assistantCaption + ? { who: 'Rowboat', text: assistantCaption } + : interimText + ? { who: 'You', text: interimText } + : null + + return ( +
+ {/* Participant tiles */} +
+ {/* User */} +
+
+ + {/* Assistant */} +
+ {mascotVisible ? ( + + ) : ( + + R + + )} + + Rowboat + + +
+
+ + {/* Captions */} +
+ {caption && ( +
+ {caption.who}: + {caption.text} +
+ )} +
+ + {/* Control bar */} +
+ + + {STATUS_DISPLAY[status].label} + + + +
+
+ ) +}