apunkt/rowboat
1
0
Fork 0
mirror of https://github.com/rowboatlabs/rowboat.git synced 2026-07-12 21:02:17 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

add talking mascot

Browse source
This commit is contained in:
Arjun 2026-07-02 01:37:05 +05:30
parent 5f81f82159
commit 7152885694
6 changed files with 490 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

BIN
apps/x/.pnpm-store/v11/index.db Normal file
View file

Binary file not shown.

29
apps/x/apps/renderer/src/App.tsx
View file

@ -119,6 +119,7 @@ import { AgentScheduleState } from '@x/shared/dist/agent-schedule-state.js'
import { toast } from "sonner"
import { useVoiceMode } from '@/hooks/useVoiceMode'
import { useVoiceTTS } from '@/hooks/useVoiceTTS'
import { TalkingHeadOverlay } from '@/components/talking-head'
import { useMeetingTranscription, type CalendarEventMeta } from '@/hooks/useMeetingTranscription'
import { useAnalyticsIdentity } from '@/hooks/useAnalyticsIdentity'
import * as analytics from '@/lib/analytics'
@ -971,6 +972,7 @@ function App() {
const ttsEnabledRef = useRef(false)
const [ttsMode, setTtsMode] = useState<'summary' | 'full'>('summary')
const ttsModeRef = useRef<'summary' | 'full'>('summary')
const [ttsAvatarEnabled, setTtsAvatarEnabled] = useState(false)
const [isRecording, setIsRecording] = useState(false)
const voiceTextBufferRef = useRef('')
const spokenIndexRef = useRef(0)
@ -1092,6 +1094,21 @@ function App() {
setTtsEnabled(prev => {
const next = !prev
ttsEnabledRef.current = next
if (!next) {
ttsRef.current.cancel()
setTtsAvatarEnabled(false)
}
return next
})
}, [])
// Talking-head mode implies voice output: enabling it turns TTS on,
// disabling it turns both off.
const handleToggleTtsAvatar = useCallback(() => {
setTtsAvatarEnabled(prev => {
const next = !prev
setTtsEnabled(next)
ttsEnabledRef.current = next
if (!next) {
ttsRef.current.cancel()
}
@ -6288,6 +6305,8 @@ function App() {
ttsMode={ttsMode}
onToggleTts={isActive ? handleToggleTts : undefined}
onTtsModeChange={isActive ? handleTtsModeChange : undefined}
ttsAvatarEnabled={ttsAvatarEnabled}
onToggleTtsAvatar={isActive ? handleToggleTtsAvatar : undefined}
/>
</div>
)
@ -6399,9 +6418,19 @@ function App() {
ttsMode={ttsMode}
onToggleTts={handleToggleTts}
onTtsModeChange={handleTtsModeChange}
ttsAvatarEnabled={ttsAvatarEnabled}
onToggleTtsAvatar={handleToggleTtsAvatar}
onComposioConnected={handleComposioConnected}
/>
)}
{/* Talking head hovers over the active view while avatar voice mode is on */}
{ttsAvatarEnabled && (
<TalkingHeadOverlay
ttsState={tts.state}
getLevel={tts.getLevel}
onDismiss={handleToggleTtsAvatar}
/>
)}
{/* Rendered last so its no-drag region paints over the sidebar drag region */}
<FixedSidebarToggle
leftInsetPx={isMac ? MACOS_TRAFFIC_LIGHTS_RESERVED_PX : 0}

38
apps/x/apps/renderer/src/components/chat-input-with-mentions.tsx
View file

@ -50,6 +50,7 @@ import {
} from '@/lib/attachment-presentation'
import { getExtension, getFileDisplayName, getMimeFromExtension, isImageMime } from '@/lib/file-utils'
import { cn } from '@/lib/utils'
import { MascotFaceIcon } from '@/components/talking-head'
import {
type FileMention,
type PromptInputMessage,
@ -235,6 +236,8 @@ interface ChatInputInnerProps {
ttsMode?: 'summary' | 'full'
onToggleTts?: () => void
onTtsModeChange?: (mode: 'summary' | 'full') => void
ttsAvatarEnabled?: boolean
onToggleTtsAvatar?: () => void
/** Fired when the user picks a different model in the dropdown (only when no run exists yet). */
onSelectedModelChange?: (model: SelectedModel | null) => void
/** Work directory for this chat (per-chat). Null when none is set. */
@ -273,6 +276,8 @@ function ChatInputInner({
ttsMode,
onToggleTts,
onTtsModeChange,
ttsAvatarEnabled,
onToggleTtsAvatar,
onSelectedModelChange,
workDir = null,
onWorkDirChange,
@ -1318,6 +1323,33 @@ function ChatInputInner({
)}
</div>
)}
{onToggleTtsAvatar && ttsAvailable && (
<Tooltip delayDuration={CHAT_INPUT_TOOLTIP_DELAY_MS}>
<TooltipTrigger asChild>
<button
type="button"
onClick={onToggleTtsAvatar}
className={cn(
'relative flex h-7 w-7 shrink-0 items-center justify-center rounded-full transition-colors',
ttsAvatarEnabled
? 'text-foreground hover:bg-muted'
: 'text-muted-foreground hover:bg-muted hover:text-foreground'
)}
aria-label={ttsAvatarEnabled ? 'Disable talking head' : 'Enable talking head'}
>
<MascotFaceIcon />
{!ttsAvatarEnabled && (
<span className="absolute inset-0 flex items-center justify-center pointer-events-none">
<span className="block h-[1.5px] w-5 -rotate-45 rounded-full bg-muted-foreground" />
</span>
)}
</button>
</TooltipTrigger>
<TooltipContent side="top">
{ttsAvatarEnabled ? 'Talking head on' : 'Talking head off'}
</TooltipContent>
</Tooltip>
)}
{voiceAvailable && onStartRecording && (
<button
type="button"
@ -1492,6 +1524,8 @@ export interface ChatInputWithMentionsProps {
ttsMode?: 'summary' | 'full'
onToggleTts?: () => void
onTtsModeChange?: (mode: 'summary' | 'full') => void
ttsAvatarEnabled?: boolean
onToggleTtsAvatar?: () => void
onSelectedModelChange?: (model: SelectedModel | null) => void
workDir?: string | null
onWorkDirChange?: (value: string | null) => void
@ -1526,6 +1560,8 @@ export function ChatInputWithMentions({
ttsMode,
onToggleTts,
onTtsModeChange,
ttsAvatarEnabled,
onToggleTtsAvatar,
onSelectedModelChange,
workDir,
onWorkDirChange,
@ -1557,6 +1593,8 @@ export function ChatInputWithMentions({
ttsMode={ttsMode}
onToggleTts={onToggleTts}
onTtsModeChange={onTtsModeChange}
ttsAvatarEnabled={ttsAvatarEnabled}
onToggleTtsAvatar={onToggleTtsAvatar}
onSelectedModelChange={onSelectedModelChange}
workDir={workDir}
onWorkDirChange={onWorkDirChange}

6
apps/x/apps/renderer/src/components/chat-sidebar.tsx
View file

@ -192,6 +192,8 @@ interface ChatSidebarProps {
ttsMode?: 'summary' | 'full'
onToggleTts?: () => void
onTtsModeChange?: (mode: 'summary' | 'full') => void
ttsAvatarEnabled?: boolean
onToggleTtsAvatar?: () => void
onComposioConnected?: (toolkitSlug: string) => void
}
@ -256,6 +258,8 @@ export function ChatSidebar({
ttsMode,
onToggleTts,
onTtsModeChange,
ttsAvatarEnabled,
onToggleTtsAvatar,
onComposioConnected,
}: ChatSidebarProps) {
const { state: sidebarState } = useSidebar()
@ -830,6 +834,8 @@ export function ChatSidebar({
ttsMode={ttsMode}
onToggleTts={isActive ? onToggleTts : undefined}
onTtsModeChange={isActive ? onTtsModeChange : undefined}
ttsAvatarEnabled={ttsAvatarEnabled}
onToggleTtsAvatar={isActive ? onToggleTtsAvatar : undefined}
/>
</div>
)

361
apps/x/apps/renderer/src/components/talking-head.tsx Normal file
View file

@ -0,0 +1,361 @@
import { useCallback, useEffect, useRef, useState } from 'react'
import { X } from 'lucide-react'
import type { TTSState } from '@/hooks/useVoiceTTS'
import { cn } from '@/lib/utils'
const POSITION_STORAGE_KEY = 'talking-head-position'
// Palette pulled from the mascot artwork: pale lavender body, dark walnut boat.
const BODY_FILL = '#E8E9F5'
const BODY_STROKE = '#17171B'
const CHEEK_FILL = '#F2B8BE'
const BOAT_DARK = '#3E2E24'
const BOAT_MID = '#54402F'
const BOAT_LIGHT = '#6B5138'
const MOUTH_FILL = '#2A1E19'
type TalkingHeadProps = {
ttsState: TTSState
getLevel: () => number
size?: number
}
/**
* The Rowboat mascot as an animated inline SVG: a round pale character sitting
* in a wooden rowboat holding an oar. The mouth is driven every animation
* frame from the live TTS audio level; eyes blink on a randomized timer.
*/
export function TalkingHead({ ttsState, getLevel, size = 160 }: TalkingHeadProps) {
const mouthOpenRef = useRef<SVGEllipseElement>(null)
const mouthSmileRef = useRef<SVGPathElement>(null)
const oarRef = useRef<SVGGElement>(null)
const smoothedRef = useRef(0)
const [blinking, setBlinking] = useState(false)
const speaking = ttsState === 'speaking'
const thinking = ttsState === 'synthesizing'
// Lip sync + oar paddle loop. Writes SVG attributes directly to avoid
// re-rendering React at 60fps.
useEffect(() => {
let raf = 0
let t = 0
const tick = () => {
const target = speaking ? getLevel() : 0
const prev = smoothedRef.current
// Fast attack, slower decay reads as natural mouth movement
const smoothed = target > prev ? prev + (target - prev) * 0.5 : prev + (target - prev) * 0.2
smoothedRef.current = smoothed
const open = Math.min(1, smoothed * 1.6)
const mouthOpen = mouthOpenRef.current
const mouthSmile = mouthSmileRef.current
if (mouthOpen && mouthSmile) {
if (open > 0.06) {
mouthOpen.setAttribute('rx', String(6.5 + open * 4))
mouthOpen.setAttribute('ry', String(1.5 + open * 9))
mouthOpen.style.opacity = '1'
mouthSmile.style.opacity = '0'
} else {
mouthOpen.style.opacity = '0'
mouthSmile.style.opacity = '1'
}
}
const oar = oarRef.current
if (oar) {
if (speaking) {
t += 0.045
const angle = Math.sin(t) * 7
oar.setAttribute('transform', `rotate(${angle.toFixed(2)} 128 118)`)
} else {
oar.setAttribute('transform', 'rotate(0 128 118)')
}
}
raf = requestAnimationFrame(tick)
}
raf = requestAnimationFrame(tick)
return () => cancelAnimationFrame(raf)
}, [speaking, getLevel])
// Randomized blinking
useEffect(() => {
let timeout: ReturnType<typeof setTimeout>
let cancelled = false
const scheduleBlink = () => {
timeout = setTimeout(() => {
if (cancelled) return
setBlinking(true)
setTimeout(() => {
if (cancelled) return
setBlinking(false)
scheduleBlink()
}, 140)
}, 2400 + Math.random() * 2600)
}
scheduleBlink()
return () => {
cancelled = true
clearTimeout(timeout)
}
}, [])
return (
<div
className="talking-head-bob relative select-none"
style={{
width: size,
height: size,
animationDuration: speaking ? '1.6s' : '3.2s',
}}
>
<style>{`
@keyframes talking-head-bob {
0%, 100% { transform: translateY(0) rotate(-1.6deg); }
50% { transform: translateY(-4px) rotate(1.6deg); }
}
.talking-head-bob {
animation-name: talking-head-bob;
animation-timing-function: ease-in-out;
animation-iteration-count: infinite;
}
@keyframes talking-head-ripple {
0% { transform: scale(0.6); opacity: 0.5; }
100% { transform: scale(1.25); opacity: 0; }
}
.talking-head-ripple {
transform-origin: center;
transform-box: fill-box;
animation: talking-head-ripple 2.6s ease-out infinite;
}
@keyframes talking-head-bubble {
0%, 100% { opacity: 0.25; transform: translateY(0); }
50% { opacity: 1; transform: translateY(-2px); }
}
.talking-head-bubble {
animation: talking-head-bubble 1.2s ease-in-out infinite;
}
`}</style>
<svg viewBox="0 0 200 190" width={size} height={size} aria-hidden="true">
{/* water ripples under the boat */}
<g>
<ellipse className="talking-head-ripple" cx="100" cy="168" rx="62" ry="9" fill="none" stroke="#8FB6D9" strokeWidth="2" style={{ animationDelay: '0s' }} />
<ellipse className="talking-head-ripple" cx="100" cy="168" rx="62" ry="9" fill="none" stroke="#8FB6D9" strokeWidth="2" style={{ animationDelay: '1.3s' }} />
<ellipse cx="100" cy="168" rx="52" ry="7" fill="#8FB6D9" opacity="0.18" />
</g>
{/* thinking bubbles while synthesizing */}
{thinking && (
<g fill={BODY_STROKE} opacity="0.75">
<circle className="talking-head-bubble" cx="146" cy="34" r="3" style={{ animationDelay: '0s' }} />
<circle className="talking-head-bubble" cx="157" cy="26" r="4.2" style={{ animationDelay: '0.2s' }} />
<circle className="talking-head-bubble" cx="170" cy="16" r="5.4" style={{ animationDelay: '0.4s' }} />
</g>
)}
{/* character: head + body blob */}
<g>
<path
d="M 100 22
C 129 22 148 43 148 68
C 148 82 141 93 131 100
C 141 107 147 117 148 128
L 52 128
C 53 115 60 105 69 99
C 59 92 52 81 52 68
C 52 43 71 22 100 22 Z"
fill={BODY_FILL}
stroke={BODY_STROKE}
strokeWidth="5"
strokeLinejoin="round"
/>
{/* eyes */}
<g style={{ transform: thinking ? 'translateY(-2.5px)' : undefined, transition: 'transform 0.3s' }}>
<ellipse
cx="84" cy="64" rx="5" ry={blinking ? 0.8 : 7}
fill={BODY_STROKE}
style={{ transition: 'ry 0.06s' }}
/>
<ellipse
cx="116" cy="64" rx="5" ry={blinking ? 0.8 : 7}
fill={BODY_STROKE}
style={{ transition: 'ry 0.06s' }}
/>
<circle cx="86" cy="61" r="1.6" fill="#FFFFFF" opacity={blinking ? 0 : 0.9} />
<circle cx="118" cy="61" r="1.6" fill="#FFFFFF" opacity={blinking ? 0 : 0.9} />
</g>
{/* cheeks */}
<ellipse cx="72" cy="76" rx="6.5" ry="4" fill={CHEEK_FILL} opacity="0.85" />
<ellipse cx="128" cy="76" rx="6.5" ry="4" fill={CHEEK_FILL} opacity="0.85" />
{/* mouth: smile when quiet, open ellipse driven by audio level */}
<path
ref={mouthSmileRef}
d="M 91 80 Q 100 88 109 80"
fill="none"
stroke={BODY_STROKE}
strokeWidth="4"
strokeLinecap="round"
/>
<ellipse
ref={mouthOpenRef}
cx="100" cy="84" rx="7" ry="2"
fill={MOUTH_FILL}
stroke={BODY_STROKE}
strokeWidth="3"
style={{ opacity: 0 }}
/>
</g>
{/* oar (rotates while speaking) */}
<g ref={oarRef}>
<line x1="158" y1="88" x2="88" y2="152" stroke={BODY_STROKE} strokeWidth="12" strokeLinecap="round" />
<line x1="158" y1="88" x2="88" y2="152" stroke={BOAT_MID} strokeWidth="7" strokeLinecap="round" />
<path
d="M 84 148 L 56 170 C 52 173 52 178 57 178 L 90 165 Z"
fill={BOAT_DARK}
stroke={BODY_STROKE}
strokeWidth="4"
strokeLinejoin="round"
/>
</g>
{/* hand resting over the oar */}
<ellipse cx="121" cy="120" rx="10" ry="8" fill={BODY_FILL} stroke={BODY_STROKE} strokeWidth="4" />
{/* boat hull (drawn last so it overlaps the body) */}
<g>
<path
d="M 30 120
C 50 132 150 132 170 120
C 168 142 152 160 100 160
C 48 160 32 142 30 120 Z"
fill={BOAT_MID}
stroke={BODY_STROKE}
strokeWidth="5"
strokeLinejoin="round"
/>
{/* plank lines */}
<path d="M 36 133 C 60 143 140 143 164 133" fill="none" stroke={BOAT_DARK} strokeWidth="3" strokeLinecap="round" />
<path d="M 44 145 C 66 153 134 153 156 145" fill="none" stroke={BOAT_DARK} strokeWidth="3" strokeLinecap="round" />
{/* gunwale highlight */}
<path d="M 33 121 C 52 131 148 131 167 121" fill="none" stroke={BOAT_LIGHT} strokeWidth="4" strokeLinecap="round" />
</g>
</svg>
</div>
)
}
type TalkingHeadOverlayProps = {
ttsState: TTSState
getLevel: () => number
onDismiss?: () => void
}
function loadStoredPosition(): { x: number; y: number } {
try {
const raw = localStorage.getItem(POSITION_STORAGE_KEY)
if (raw) {
const parsed = JSON.parse(raw)
if (typeof parsed?.x === 'number' && typeof parsed?.y === 'number') {
return parsed
}
}
} catch {
// ignore corrupt stored position
}
return { x: 0, y: 0 }
}
/**
* Floating, draggable widget that hosts the talking head. Anchored to the
* bottom-right of the window (above the composer) and offset by a persisted
* drag position, so it hovers over whatever view is active.
*/
export function TalkingHeadOverlay({ ttsState, getLevel, onDismiss }: TalkingHeadOverlayProps) {
const [offset, setOffset] = useState(loadStoredPosition)
const [dragging, setDragging] = useState(false)
const dragStartRef = useRef<{ pointerX: number; pointerY: number; x: number; y: number } | null>(null)
const handlePointerDown = useCallback((e: React.PointerEvent<HTMLDivElement>) => {
if (e.button !== 0) return
dragStartRef.current = { pointerX: e.clientX, pointerY: e.clientY, x: offset.x, y: offset.y }
setDragging(true)
e.currentTarget.setPointerCapture(e.pointerId)
}, [offset])
const handlePointerMove = useCallback((e: React.PointerEvent<HTMLDivElement>) => {
const start = dragStartRef.current
if (!start) return
setOffset({
x: start.x + (e.clientX - start.pointerX),
y: start.y + (e.clientY - start.pointerY),
})
}, [])
const handlePointerUp = useCallback((e: React.PointerEvent<HTMLDivElement>) => {
if (!dragStartRef.current) return
dragStartRef.current = null
setDragging(false)
e.currentTarget.releasePointerCapture(e.pointerId)
}, [])
useEffect(() => {
if (dragging) return
try {
localStorage.setItem(POSITION_STORAGE_KEY, JSON.stringify(offset))
} catch {
// best-effort persistence
}
}, [offset, dragging])
return (
<div
className={cn(
'group fixed bottom-28 right-8 z-50 touch-none',
dragging ? 'cursor-grabbing' : 'cursor-grab'
)}
style={{
transform: `translate(${offset.x}px, ${offset.y}px)`,
animation: dragging ? undefined : 'talking-head-pop 0.35s cubic-bezier(0.34, 1.56, 0.64, 1)',
}}
onPointerDown={handlePointerDown}
onPointerMove={handlePointerMove}
onPointerUp={handlePointerUp}
onPointerCancel={handlePointerUp}
role="img"
aria-label="Rowboat talking head"
>
<style>{`
@keyframes talking-head-pop {
0% { opacity: 0; scale: 0.4; }
100% { opacity: 1; scale: 1; }
}
`}</style>
{onDismiss && (
<button
type="button"
onPointerDown={(e) => e.stopPropagation()}
onClick={onDismiss}
className="absolute -right-1 -top-1 z-10 flex h-5 w-5 items-center justify-center rounded-full border border-border bg-background text-muted-foreground opacity-0 shadow-sm transition-opacity hover:text-foreground group-hover:opacity-100"
aria-label="Hide talking head"
>
<X className="h-3 w-3" />
</button>
)}
<TalkingHead ttsState={ttsState} getLevel={getLevel} />
</div>
)
}
/** Small static mascot face used as the toolbar toggle icon. */
export function MascotFaceIcon({ className }: { className?: string }) {
return (
<svg viewBox="0 0 24 24" width="16" height="16" className={className} aria-hidden="true">
<circle cx="12" cy="12" r="9.5" fill="none" stroke="currentColor" strokeWidth="1.8" />
<ellipse cx="8.6" cy="10.5" rx="1.3" ry="1.8" fill="currentColor" />
<ellipse cx="15.4" cy="10.5" rx="1.3" ry="1.8" fill="currentColor" />
<path d="M 9 14.5 Q 12 17 15 14.5" fill="none" stroke="currentColor" strokeWidth="1.8" strokeLinecap="round" />
</svg>
)
}

60
apps/x/apps/renderer/src/hooks/useVoiceTTS.ts
View file

@ -14,10 +14,15 @@ function synthesize(text: string): Promise<SynthesizedAudio> {
);
}
function playAudio(dataUrl: string, audioRef: React.MutableRefObject<HTMLAudioElement | null>): Promise<void> {
function playAudio(
dataUrl: string,
audioRef: React.MutableRefObject<HTMLAudioElement | null>,
onAudioElement?: (audio: HTMLAudioElement) => void
): Promise<void> {
return new Promise<void>((resolve, reject) => {
const audio = new Audio(dataUrl);
audioRef.current = audio;
onAudioElement?.(audio);
audio.onended = () => {
console.log('[tts] audio ended');
resolve();
@ -42,6 +47,53 @@ export function useVoiceTTS() {
const processingRef = useRef(false);
// Pre-fetched audio ready to play immediately
const prefetchedRef = useRef<Promise<SynthesizedAudio> | null>(null);
// Web Audio analyser tap for lip-sync (talking head)
const audioCtxRef = useRef<AudioContext | null>(null);
const analyserRef = useRef<AnalyserNode | null>(null);
const levelBufferRef = useRef<Uint8Array | null>(null);
// Route playback through an AnalyserNode so consumers can read the live
// output level. If Web Audio wiring fails, the element still plays directly.
const connectAnalyser = useCallback((audio: HTMLAudioElement) => {
try {
let ctx = audioCtxRef.current;
if (!ctx) {
ctx = new AudioContext();
audioCtxRef.current = ctx;
const analyser = ctx.createAnalyser();
analyser.fftSize = 512;
analyser.smoothingTimeConstant = 0.5;
analyser.connect(ctx.destination);
analyserRef.current = analyser;
}
if (ctx.state === 'suspended') {
void ctx.resume();
}
const source = ctx.createMediaElementSource(audio);
source.connect(analyserRef.current!);
} catch (err) {
console.error('[tts] analyser hookup failed:', err);
}
}, []);
// Current output level, 0..1. Safe to call every animation frame.
const getLevel = useCallback((): number => {
const analyser = analyserRef.current;
if (!analyser) return 0;
let buffer = levelBufferRef.current;
if (!buffer || buffer.length !== analyser.fftSize) {
buffer = new Uint8Array(analyser.fftSize);
levelBufferRef.current = buffer;
}
analyser.getByteTimeDomainData(buffer);
let sum = 0;
for (let i = 0; i < buffer.length; i++) {
const d = (buffer[i] - 128) / 128;
sum += d * d;
}
const rms = Math.sqrt(sum / buffer.length);
return Math.min(1, rms * 4);
}, []);
const processQueue = useCallback(async () => {
if (processingRef.current) return;
@ -74,7 +126,7 @@ export function useVoiceTTS() {
prefetchedRef.current = synthesize(nextText);
}
await playAudio(audio.dataUrl, audioRef);
await playAudio(audio.dataUrl, audioRef, connectAnalyser);
} catch (err) {
console.error('[tts] error:', err);
prefetchedRef.current = null;
@ -85,7 +137,7 @@ export function useVoiceTTS() {
prefetchedRef.current = null;
processingRef.current = false;
setState('idle');
}, []);
}, [connectAnalyser]);
const speak = useCallback((text: string) => {
console.log('[tts] speak() called:', text.substring(0, 80));
@ -104,5 +156,5 @@ export function useVoiceTTS() {
setState('idle');
}, []);
return { state, speak, cancel };
return { state, speak, cancel, getLevel };
}