mirror of
https://github.com/rowboatlabs/rowboat.git
synced 2026-07-12 21:02:17 +02:00
full video call mode
This commit is contained in:
Arjun
parent
ad24699661
commit
42353c63f4
4 changed files with 225 additions and 24 deletions
|
|
@ -12,7 +12,7 @@ import { ChatSidebar } from './components/chat-sidebar';
|
|||
import { useSessionChat } from '@/hooks/useSessionChat';
|
||||
import { ChatHeader } from './components/chat-header';
|
||||
import { ChatEmptyState } from './components/chat-empty-state';
|
||||
import { ChatInputWithMentions, type PermissionMode, type StagedAttachment } from './components/chat-input-with-mentions';
|
||||
import { ChatInputWithMentions, type PermissionMode, type StagedAttachment, type VideoChatMode } from './components/chat-input-with-mentions';
|
||||
import { ChatMessageAttachments } from '@/components/chat-message-attachments'
|
||||
import { GraphView, type GraphEdge, type GraphNode } from '@/components/graph-view';
|
||||
import { BasesView, type BaseConfig, DEFAULT_BASE_CONFIG } from '@/components/bases-view';
|
||||
|
|
@ -122,6 +122,7 @@ import { useVideoMode } from '@/hooks/useVideoMode'
|
|||
import { useVoiceTTS } from '@/hooks/useVoiceTTS'
|
||||
import { TalkingHeadOverlay } from '@/components/talking-head'
|
||||
import { VideoPreviewOverlay } from '@/components/video-preview-overlay'
|
||||
import { VideoCallView } from '@/components/video-call-view'
|
||||
import { ProductTour, type TourNavTarget } from '@/components/product-tour'
|
||||
import { useMeetingTranscription, type CalendarEventMeta } from '@/hooks/useMeetingTranscription'
|
||||
import { useAnalyticsIdentity } from '@/hooks/useAnalyticsIdentity'
|
||||
|
|
@ -986,6 +987,13 @@ function App() {
|
|||
const ttsRef = useRef(tts)
|
||||
ttsRef.current = tts
|
||||
|
||||
// Latest assistant line handed to TTS — shown as the caption in the
|
||||
// full-screen call view while the assistant is speaking.
|
||||
const [assistantCaption, setAssistantCaption] = useState('')
|
||||
useEffect(() => {
|
||||
if (tts.state === 'idle') setAssistantCaption('')
|
||||
}, [tts.state])
|
||||
|
||||
// Speak newly completed <voice> blocks from the new runtime's live stream
|
||||
// (parity with the legacy text-delta voice extraction below). The store
|
||||
// accumulates completed blocks in chatState.voiceSegments; we speak only
|
||||
|
|
@ -1004,6 +1012,7 @@ function App() {
|
|||
spokenVoiceRef.current.count += 1
|
||||
if (ttsEnabledRef.current) {
|
||||
ttsRef.current.speak(segment)
|
||||
setAssistantCaption(segment)
|
||||
}
|
||||
}
|
||||
}, [voiceSegments, runId])
|
||||
|
|
@ -1018,8 +1027,8 @@ function App() {
|
|||
// auto-submitted utterances, spoken responses (handler defined below, after
|
||||
// the voice/submit plumbing it drives).
|
||||
const video = useVideoMode()
|
||||
const [videoChatMode, setVideoChatMode] = useState<'off' | 'chat' | 'call'>('off')
|
||||
const videoChatModeRef = useRef<'off' | 'chat' | 'call'>('off')
|
||||
const [videoChatMode, setVideoChatMode] = useState<VideoChatMode>('off')
|
||||
const videoChatModeRef = useRef<VideoChatMode>('off')
|
||||
// TTS settings to restore when a hands-free call ends (a call forces
|
||||
// full-read-aloud TTS for its duration).
|
||||
const preCallTtsRef = useRef<{ enabled: boolean; mode: 'summary' | 'full' } | null>(null)
|
||||
|
|
@ -1083,7 +1092,7 @@ function App() {
|
|||
|
||||
const handleStartRecording = useCallback(() => {
|
||||
// Hands-free call mode owns the mic — ignore push-to-talk while it's on.
|
||||
if (videoChatModeRef.current === 'call') return
|
||||
if (videoChatModeRef.current === 'call' || videoChatModeRef.current === 'meeting') return
|
||||
setIsRecording(true)
|
||||
isRecordingRef.current = true
|
||||
voice.start()
|
||||
|
|
@ -1145,15 +1154,18 @@ function App() {
|
|||
isRecordingRef.current = false
|
||||
}, [voice])
|
||||
|
||||
// Video chat mode transitions. 'chat' just runs the camera; 'call' also
|
||||
// listens continuously (auto-submitting each utterance as a voice message)
|
||||
// and forces full read-aloud TTS so the assistant answers out loud.
|
||||
const handleVideoModeChange = useCallback(async (mode: 'off' | 'chat' | 'call') => {
|
||||
// Video chat mode transitions. 'chat' just runs the camera; 'call' and
|
||||
// 'meeting' also listen continuously (auto-submitting each utterance as a
|
||||
// voice message) and force full read-aloud TTS so the assistant answers out
|
||||
// loud. 'meeting' is the same pipeline presented as a full-screen call.
|
||||
const handleVideoModeChange = useCallback(async (mode: VideoChatMode) => {
|
||||
const prev = videoChatModeRef.current
|
||||
if (mode === prev) return
|
||||
const wasHandsFree = prev === 'call' || prev === 'meeting'
|
||||
const isHandsFree = mode === 'call' || mode === 'meeting'
|
||||
|
||||
// Leaving a call: release the mic and restore the pre-call TTS settings.
|
||||
if (prev === 'call') {
|
||||
// Leaving hands-free: release the mic and restore the pre-call TTS settings.
|
||||
if (wasHandsFree && !isHandsFree) {
|
||||
voiceRef.current.cancel()
|
||||
const saved = preCallTtsRef.current
|
||||
preCallTtsRef.current = null
|
||||
|
|
@ -1179,7 +1191,8 @@ function App() {
|
|||
if (!ok) return // camera denied/unavailable — stay off
|
||||
}
|
||||
|
||||
if (mode === 'call') {
|
||||
// Entering hands-free (call ↔ meeting switches keep the mic/TTS as-is).
|
||||
if (isHandsFree && !wasHandsFree) {
|
||||
// A manual push-to-talk recording can't coexist with the call's mic.
|
||||
if (isRecordingRef.current) {
|
||||
voiceRef.current.cancel()
|
||||
|
|
@ -1204,7 +1217,7 @@ function App() {
|
|||
// During a call, mute the mic while the assistant is thinking or speaking
|
||||
// so its own TTS (or a half-turn) never gets transcribed back at it.
|
||||
useEffect(() => {
|
||||
if (videoChatMode !== 'call') return
|
||||
if (videoChatMode !== 'call' && videoChatMode !== 'meeting') return
|
||||
voiceRef.current.setPaused(activeIsProcessing || tts.state !== 'idle')
|
||||
}, [videoChatMode, activeIsProcessing, tts.state])
|
||||
|
||||
|
|
@ -2332,6 +2345,7 @@ function App() {
|
|||
console.log('[voice] extracted voice tag:', voiceContent)
|
||||
if (voiceContent && ttsEnabledRef.current) {
|
||||
ttsRef.current.speak(voiceContent)
|
||||
setAssistantCaption(voiceContent)
|
||||
}
|
||||
spokenIndexRef.current += voiceMatch.index + voiceMatch[0].length
|
||||
}
|
||||
|
|
@ -6569,8 +6583,9 @@ function App() {
|
|||
onComposioConnected={handleComposioConnected}
|
||||
/>
|
||||
)}
|
||||
{/* Webcam PiP preview while video chat mode is on */}
|
||||
{videoChatMode !== 'off' && (
|
||||
{/* Webcam PiP preview while video chat mode is on (the full-screen
|
||||
meeting view replaces it) */}
|
||||
{(videoChatMode === 'chat' || videoChatMode === 'call') && (
|
||||
<VideoPreviewOverlay
|
||||
streamRef={video.streamRef}
|
||||
onTurnOff={() => handleVideoModeChange('off')}
|
||||
|
|
@ -6586,6 +6601,24 @@ function App() {
|
|||
interimText={videoChatMode === 'call' ? voice.interimText : undefined}
|
||||
/>
|
||||
)}
|
||||
{/* Full-screen Meet-style call: user tile + animated mascot tile */}
|
||||
{videoChatMode === 'meeting' && (
|
||||
<VideoCallView
|
||||
streamRef={video.streamRef}
|
||||
ttsState={tts.state}
|
||||
getTtsLevel={tts.getLevel}
|
||||
status={
|
||||
tts.state === 'speaking'
|
||||
? 'speaking'
|
||||
: tts.state === 'synthesizing' || activeIsProcessing
|
||||
? 'thinking'
|
||||
: 'listening'
|
||||
}
|
||||
interimText={voice.interimText}
|
||||
assistantCaption={assistantCaption}
|
||||
onLeave={() => handleVideoModeChange('off')}
|
||||
/>
|
||||
)}
|
||||
{/* Talking head hovers over the active view while avatar voice mode is
|
||||
on (hidden during the tour, which shows its own mascot) */}
|
||||
{ttsAvatarEnabled && !tourActive && (
|
||||
|
|
|
|||
|
|
@ -212,6 +212,8 @@ function compactWorkDirPath(path: string) {
|
|||
return path.replace(/^\/Users\/[^/]+/, '~')
|
||||
}
|
||||
|
||||
export type VideoChatMode = 'off' | 'chat' | 'call' | 'meeting'
|
||||
|
||||
interface ChatInputInnerProps {
|
||||
onSubmit: (message: PromptInputMessage, mentions?: FileMention[], attachments?: StagedAttachment[], searchEnabled?: boolean, codeMode?: 'claude' | 'codex', permissionMode?: PermissionMode) => void
|
||||
onStop?: () => void
|
||||
|
|
@ -239,10 +241,11 @@ interface ChatInputInnerProps {
|
|||
onTtsModeChange?: (mode: 'summary' | 'full') => void
|
||||
ttsAvatarEnabled?: boolean
|
||||
onToggleTtsAvatar?: () => void
|
||||
/** Video chat mode: 'chat' attaches webcam frames to messages; 'call' is
|
||||
* fully hands-free — continuous listening, spoken responses. */
|
||||
videoChatMode?: 'off' | 'chat' | 'call'
|
||||
onVideoModeChange?: (mode: 'off' | 'chat' | 'call') => void
|
||||
/** Video chat mode: 'chat' attaches webcam frames to messages; 'call' and
|
||||
* 'meeting' are fully hands-free — continuous listening, spoken responses
|
||||
* ('meeting' additionally takes over the whole screen, Meet-style). */
|
||||
videoChatMode?: VideoChatMode
|
||||
onVideoModeChange?: (mode: VideoChatMode) => void
|
||||
/** Hands-free call needs both voice input (STT) and voice output (TTS). */
|
||||
videoCallAvailable?: boolean
|
||||
/** Fired when the user picks a different model in the dropdown (only when no run exists yet). */
|
||||
|
|
@ -1404,12 +1407,15 @@ function ChatInputInner({
|
|||
<DropdownMenuContent align="end">
|
||||
<DropdownMenuRadioGroup
|
||||
value={videoChatMode}
|
||||
onValueChange={(v) => onVideoModeChange(v as 'chat' | 'call')}
|
||||
onValueChange={(v) => onVideoModeChange(v as VideoChatMode)}
|
||||
>
|
||||
<DropdownMenuRadioItem value="chat">Video + chat</DropdownMenuRadioItem>
|
||||
<DropdownMenuRadioItem value="call" disabled={!videoCallAvailable}>
|
||||
Video call (hands-free)
|
||||
</DropdownMenuRadioItem>
|
||||
<DropdownMenuRadioItem value="meeting" disabled={!videoCallAvailable}>
|
||||
Video call (full screen)
|
||||
</DropdownMenuRadioItem>
|
||||
</DropdownMenuRadioGroup>
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
|
|
@ -1592,8 +1598,8 @@ export interface ChatInputWithMentionsProps {
|
|||
onTtsModeChange?: (mode: 'summary' | 'full') => void
|
||||
ttsAvatarEnabled?: boolean
|
||||
onToggleTtsAvatar?: () => void
|
||||
videoChatMode?: 'off' | 'chat' | 'call'
|
||||
onVideoModeChange?: (mode: 'off' | 'chat' | 'call') => void
|
||||
videoChatMode?: VideoChatMode
|
||||
onVideoModeChange?: (mode: VideoChatMode) => void
|
||||
videoCallAvailable?: boolean
|
||||
onSelectedModelChange?: (model: SelectedModel | null) => void
|
||||
workDir?: string | null
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ import { MarkdownPreOverride } from '@/components/ai-elements/markdown-code-over
|
|||
import { defaultRemarkPlugins } from 'streamdown'
|
||||
import remarkBreaks from 'remark-breaks'
|
||||
import { type ChatTab } from '@/components/tab-bar'
|
||||
import { ChatInputWithMentions, type PermissionMode, type StagedAttachment, type SelectedModel } from '@/components/chat-input-with-mentions'
|
||||
import { ChatInputWithMentions, type PermissionMode, type StagedAttachment, type SelectedModel, type VideoChatMode } from '@/components/chat-input-with-mentions'
|
||||
import { ChatMessageAttachments } from '@/components/chat-message-attachments'
|
||||
import { useSidebar } from '@/components/ui/sidebar'
|
||||
import { wikiLabel } from '@/lib/wiki-links'
|
||||
|
|
@ -194,8 +194,8 @@ interface ChatSidebarProps {
|
|||
onTtsModeChange?: (mode: 'summary' | 'full') => void
|
||||
ttsAvatarEnabled?: boolean
|
||||
onToggleTtsAvatar?: () => void
|
||||
videoChatMode?: 'off' | 'chat' | 'call'
|
||||
onVideoModeChange?: (mode: 'off' | 'chat' | 'call') => void
|
||||
videoChatMode?: VideoChatMode
|
||||
onVideoModeChange?: (mode: VideoChatMode) => void
|
||||
videoCallAvailable?: boolean
|
||||
onComposioConnected?: (toolkitSlug: string) => void
|
||||
}
|
||||
|
|
|
|||
162
apps/x/apps/renderer/src/components/video-call-view.tsx
Normal file
162
apps/x/apps/renderer/src/components/video-call-view.tsx
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
import { useEffect, useRef, useState } from 'react'
|
||||
import { PhoneOff } from 'lucide-react'
|
||||
|
||||
import { MascotFaceIcon, TalkingHead } from '@/components/talking-head'
|
||||
import type { TTSState } from '@/hooks/useVoiceTTS'
|
||||
import { cn } from '@/lib/utils'
|
||||
|
||||
export type VideoCallStatus = 'listening' | 'thinking' | 'speaking'
|
||||
|
||||
interface VideoCallViewProps {
|
||||
/** Live camera stream from useVideoMode — attached to the user's tile. */
|
||||
streamRef: React.MutableRefObject<MediaStream | null>
|
||||
ttsState: TTSState
|
||||
/** Live TTS output level — drives the mascot's mouth animation. */
|
||||
getTtsLevel: () => number
|
||||
status: VideoCallStatus
|
||||
/** Live transcript of the user's in-progress utterance. */
|
||||
interimText?: string
|
||||
/** The assistant line currently being spoken aloud. */
|
||||
assistantCaption?: string
|
||||
onLeave: () => void
|
||||
}
|
||||
|
||||
const STATUS_DISPLAY: Record<VideoCallStatus, { label: string; dotClass: string }> = {
|
||||
listening: { label: 'Listening', dotClass: 'bg-green-500 animate-pulse' },
|
||||
thinking: { label: 'Thinking…', dotClass: 'bg-amber-400' },
|
||||
speaking: { label: 'Speaking', dotClass: 'bg-sky-400 animate-pulse' },
|
||||
}
|
||||
|
||||
/**
|
||||
* Full-screen hands-free call: a Meet-style two-tile layout with the user's
|
||||
* webcam on one side and the mascot as the other participant. The mascot
|
||||
* animates with the assistant's speech; dismissing it swaps in a Meet-style
|
||||
* letter avatar ("R"). Live captions run along the bottom.
|
||||
*/
|
||||
export function VideoCallView({
|
||||
streamRef,
|
||||
ttsState,
|
||||
getTtsLevel,
|
||||
status,
|
||||
interimText,
|
||||
assistantCaption,
|
||||
onLeave,
|
||||
}: VideoCallViewProps) {
|
||||
const videoRef = useRef<HTMLVideoElement | null>(null)
|
||||
const [mascotVisible, setMascotVisible] = useState(true)
|
||||
|
||||
useEffect(() => {
|
||||
const videoEl = videoRef.current
|
||||
if (!videoEl) return
|
||||
videoEl.srcObject = streamRef.current
|
||||
videoEl.play().catch(() => {})
|
||||
return () => {
|
||||
videoEl.srcObject = null
|
||||
}
|
||||
}, [streamRef])
|
||||
|
||||
const userSpeaking = status === 'listening' && Boolean(interimText)
|
||||
const assistantSpeaking = ttsState === 'speaking'
|
||||
|
||||
const caption = assistantSpeaking && assistantCaption
|
||||
? { who: 'Rowboat', text: assistantCaption }
|
||||
: interimText
|
||||
? { who: 'You', text: interimText }
|
||||
: null
|
||||
|
||||
return (
|
||||
<div className="fixed inset-0 z-[100] flex flex-col bg-neutral-950">
|
||||
{/* Participant tiles */}
|
||||
<div className="grid min-h-0 flex-1 grid-cols-1 gap-3 p-4 pb-2 md:grid-cols-2">
|
||||
{/* User */}
|
||||
<div
|
||||
className={cn(
|
||||
'relative flex items-center justify-center overflow-hidden rounded-2xl bg-neutral-900 transition-shadow',
|
||||
userSpeaking && 'ring-2 ring-green-500/80'
|
||||
)}
|
||||
>
|
||||
<video
|
||||
ref={videoRef}
|
||||
muted
|
||||
playsInline
|
||||
className="h-full w-full object-cover"
|
||||
style={{ transform: 'scaleX(-1)' }}
|
||||
/>
|
||||
<span className="absolute bottom-3 left-3 rounded-md bg-black/50 px-2 py-0.5 text-sm text-white">
|
||||
You
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{/* Assistant */}
|
||||
<div
|
||||
className={cn(
|
||||
'group relative flex items-center justify-center overflow-hidden rounded-2xl bg-neutral-900 transition-shadow',
|
||||
assistantSpeaking && 'ring-2 ring-sky-400/80'
|
||||
)}
|
||||
>
|
||||
{mascotVisible ? (
|
||||
<TalkingHead ttsState={ttsState} getLevel={getTtsLevel} size={220} />
|
||||
) : (
|
||||
<span
|
||||
className="flex h-40 w-40 items-center justify-center rounded-full bg-sky-600 text-7xl font-medium text-white"
|
||||
aria-label="Rowboat"
|
||||
>
|
||||
R
|
||||
</span>
|
||||
)}
|
||||
<span className="absolute bottom-3 left-3 rounded-md bg-black/50 px-2 py-0.5 text-sm text-white">
|
||||
Rowboat
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setMascotVisible((v) => !v)}
|
||||
className="absolute right-3 top-3 rounded-md bg-black/50 px-2 py-1 text-xs text-white/80 opacity-0 transition-opacity hover:text-white group-hover:opacity-100"
|
||||
>
|
||||
{mascotVisible ? 'Hide mascot' : 'Show mascot'}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Captions */}
|
||||
<div className="flex h-14 items-center justify-center px-6">
|
||||
{caption && (
|
||||
<div className="max-w-3xl truncate rounded-lg bg-black/60 px-4 py-2 text-sm text-white/90">
|
||||
<span className="mr-2 font-semibold text-white">{caption.who}:</span>
|
||||
{caption.text}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Control bar */}
|
||||
<div className="flex items-center justify-center gap-4 pb-5">
|
||||
<span className="flex items-center gap-2 rounded-full bg-neutral-800 px-3 py-1.5 text-xs font-medium text-white/90">
|
||||
<span className={cn('block h-2 w-2 rounded-full', STATUS_DISPLAY[status].dotClass)} />
|
||||
{STATUS_DISPLAY[status].label}
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setMascotVisible((v) => !v)}
|
||||
className={cn(
|
||||
'relative flex h-10 w-10 items-center justify-center rounded-full bg-neutral-800 text-white/90 transition-colors hover:bg-neutral-700',
|
||||
)}
|
||||
aria-label={mascotVisible ? 'Hide mascot' : 'Show mascot'}
|
||||
>
|
||||
<MascotFaceIcon />
|
||||
{!mascotVisible && (
|
||||
<span className="absolute inset-0 flex items-center justify-center pointer-events-none">
|
||||
<span className="block h-[1.5px] w-6 -rotate-45 rounded-full bg-white/80" />
|
||||
</span>
|
||||
)}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onLeave}
|
||||
className="flex h-10 w-14 items-center justify-center rounded-full bg-red-600 text-white transition-colors hover:bg-red-500"
|
||||
aria-label="Leave call"
|
||||
>
|
||||
<PhoneOff className="h-5 w-5" />
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue