From 7152885694b1dc3117aadf52541544c37a4b1497 Mon Sep 17 00:00:00 2001 From: Arjun <6592213+arkml@users.noreply.github.com> Date: Thu, 2 Jul 2026 01:37:05 +0530 Subject: [PATCH] add talking mascot --- apps/x/.pnpm-store/v11/index.db | Bin 0 -> 8192 bytes apps/x/apps/renderer/src/App.tsx | 29 ++ .../components/chat-input-with-mentions.tsx | 38 ++ .../renderer/src/components/chat-sidebar.tsx | 6 + .../renderer/src/components/talking-head.tsx | 361 ++++++++++++++++++ apps/x/apps/renderer/src/hooks/useVoiceTTS.ts | 60 ++- 6 files changed, 490 insertions(+), 4 deletions(-) create mode 100644 apps/x/.pnpm-store/v11/index.db create mode 100644 apps/x/apps/renderer/src/components/talking-head.tsx diff --git a/apps/x/.pnpm-store/v11/index.db b/apps/x/.pnpm-store/v11/index.db new file mode 100644 index 0000000000000000000000000000000000000000..7f9770bd25137067815560f9ef52a33e713f2b53 GIT binary patch literal 8192 zcmeIuzpBD86bA4#2p0s=&GDX11#$5OY&BppTCFMSqD0LV@%|C%po4=C;1j#tR!Xsx zd-*<+oFpepe$$EEhlalXPCq)NHmfksS%-)*#*-P9XRK%~B>T9;=Xc?(b$^tiS5|q+ zqJcmF0uX=z1Rwwb2tWV=5P$##awu^7v_7h}nsvK|di`yVdUMb_v)cb|%{g=6U0>Kr zkg^>qDAS^Pkk?+mi kUUFZI)hjuq$Cn@g0SG_<0uX=z1Rwwb2tWV=5J(070T$gf00000 literal 0 HcmV?d00001 diff --git a/apps/x/apps/renderer/src/App.tsx b/apps/x/apps/renderer/src/App.tsx index 5ef631e2..977c28fa 100644 --- a/apps/x/apps/renderer/src/App.tsx +++ b/apps/x/apps/renderer/src/App.tsx @@ -119,6 +119,7 @@ import { AgentScheduleState } from '@x/shared/dist/agent-schedule-state.js' import { toast } from "sonner" import { useVoiceMode } from '@/hooks/useVoiceMode' import { useVoiceTTS } from '@/hooks/useVoiceTTS' +import { TalkingHeadOverlay } from '@/components/talking-head' import { useMeetingTranscription, type CalendarEventMeta } from '@/hooks/useMeetingTranscription' import { useAnalyticsIdentity } from '@/hooks/useAnalyticsIdentity' import * as analytics from '@/lib/analytics' @@ -971,6 +972,7 @@ function App() { const ttsEnabledRef = useRef(false) const [ttsMode, setTtsMode] = useState<'summary' | 'full'>('summary') const ttsModeRef = useRef<'summary' | 'full'>('summary') + const [ttsAvatarEnabled, setTtsAvatarEnabled] = useState(false) const [isRecording, setIsRecording] = useState(false) const voiceTextBufferRef = useRef('') const spokenIndexRef = useRef(0) @@ -1092,6 +1094,21 @@ function App() { setTtsEnabled(prev => { const next = !prev ttsEnabledRef.current = next + if (!next) { + ttsRef.current.cancel() + setTtsAvatarEnabled(false) + } + return next + }) + }, []) + + // Talking-head mode implies voice output: enabling it turns TTS on, + // disabling it turns both off. + const handleToggleTtsAvatar = useCallback(() => { + setTtsAvatarEnabled(prev => { + const next = !prev + setTtsEnabled(next) + ttsEnabledRef.current = next if (!next) { ttsRef.current.cancel() } @@ -6288,6 +6305,8 @@ function App() { ttsMode={ttsMode} onToggleTts={isActive ? handleToggleTts : undefined} onTtsModeChange={isActive ? handleTtsModeChange : undefined} + ttsAvatarEnabled={ttsAvatarEnabled} + onToggleTtsAvatar={isActive ? handleToggleTtsAvatar : undefined} /> ) @@ -6399,9 +6418,19 @@ function App() { ttsMode={ttsMode} onToggleTts={handleToggleTts} onTtsModeChange={handleTtsModeChange} + ttsAvatarEnabled={ttsAvatarEnabled} + onToggleTtsAvatar={handleToggleTtsAvatar} onComposioConnected={handleComposioConnected} /> )} + {/* Talking head hovers over the active view while avatar voice mode is on */} + {ttsAvatarEnabled && ( + + )} {/* Rendered last so its no-drag region paints over the sidebar drag region */} void onTtsModeChange?: (mode: 'summary' | 'full') => void + ttsAvatarEnabled?: boolean + onToggleTtsAvatar?: () => void /** Fired when the user picks a different model in the dropdown (only when no run exists yet). */ onSelectedModelChange?: (model: SelectedModel | null) => void /** Work directory for this chat (per-chat). Null when none is set. */ @@ -273,6 +276,8 @@ function ChatInputInner({ ttsMode, onToggleTts, onTtsModeChange, + ttsAvatarEnabled, + onToggleTtsAvatar, onSelectedModelChange, workDir = null, onWorkDirChange, @@ -1318,6 +1323,33 @@ function ChatInputInner({ )} )} + {onToggleTtsAvatar && ttsAvailable && ( + + + + + + {ttsAvatarEnabled ? 'Talking head on' : 'Talking head off'} + + + )} {voiceAvailable && onStartRecording && ( + )} + + + ) +} + +/** Small static mascot face used as the toolbar toggle icon. */ +export function MascotFaceIcon({ className }: { className?: string }) { + return ( + + ) +} diff --git a/apps/x/apps/renderer/src/hooks/useVoiceTTS.ts b/apps/x/apps/renderer/src/hooks/useVoiceTTS.ts index c0737bf8..7278e983 100644 --- a/apps/x/apps/renderer/src/hooks/useVoiceTTS.ts +++ b/apps/x/apps/renderer/src/hooks/useVoiceTTS.ts @@ -14,10 +14,15 @@ function synthesize(text: string): Promise { ); } -function playAudio(dataUrl: string, audioRef: React.MutableRefObject): Promise { +function playAudio( + dataUrl: string, + audioRef: React.MutableRefObject, + onAudioElement?: (audio: HTMLAudioElement) => void +): Promise { return new Promise((resolve, reject) => { const audio = new Audio(dataUrl); audioRef.current = audio; + onAudioElement?.(audio); audio.onended = () => { console.log('[tts] audio ended'); resolve(); @@ -42,6 +47,53 @@ export function useVoiceTTS() { const processingRef = useRef(false); // Pre-fetched audio ready to play immediately const prefetchedRef = useRef | null>(null); + // Web Audio analyser tap for lip-sync (talking head) + const audioCtxRef = useRef(null); + const analyserRef = useRef(null); + const levelBufferRef = useRef(null); + + // Route playback through an AnalyserNode so consumers can read the live + // output level. If Web Audio wiring fails, the element still plays directly. + const connectAnalyser = useCallback((audio: HTMLAudioElement) => { + try { + let ctx = audioCtxRef.current; + if (!ctx) { + ctx = new AudioContext(); + audioCtxRef.current = ctx; + const analyser = ctx.createAnalyser(); + analyser.fftSize = 512; + analyser.smoothingTimeConstant = 0.5; + analyser.connect(ctx.destination); + analyserRef.current = analyser; + } + if (ctx.state === 'suspended') { + void ctx.resume(); + } + const source = ctx.createMediaElementSource(audio); + source.connect(analyserRef.current!); + } catch (err) { + console.error('[tts] analyser hookup failed:', err); + } + }, []); + + // Current output level, 0..1. Safe to call every animation frame. + const getLevel = useCallback((): number => { + const analyser = analyserRef.current; + if (!analyser) return 0; + let buffer = levelBufferRef.current; + if (!buffer || buffer.length !== analyser.fftSize) { + buffer = new Uint8Array(analyser.fftSize); + levelBufferRef.current = buffer; + } + analyser.getByteTimeDomainData(buffer); + let sum = 0; + for (let i = 0; i < buffer.length; i++) { + const d = (buffer[i] - 128) / 128; + sum += d * d; + } + const rms = Math.sqrt(sum / buffer.length); + return Math.min(1, rms * 4); + }, []); const processQueue = useCallback(async () => { if (processingRef.current) return; @@ -74,7 +126,7 @@ export function useVoiceTTS() { prefetchedRef.current = synthesize(nextText); } - await playAudio(audio.dataUrl, audioRef); + await playAudio(audio.dataUrl, audioRef, connectAnalyser); } catch (err) { console.error('[tts] error:', err); prefetchedRef.current = null; @@ -85,7 +137,7 @@ export function useVoiceTTS() { prefetchedRef.current = null; processingRef.current = false; setState('idle'); - }, []); + }, [connectAnalyser]); const speak = useCallback((text: string) => { console.log('[tts] speak() called:', text.substring(0, 80)); @@ -104,5 +156,5 @@ export function useVoiceTTS() { setState('idle'); }, []); - return { state, speak, cancel }; + return { state, speak, cancel, getLevel }; }