mirror of
https://github.com/rowboatlabs/rowboat.git
synced 2026-07-15 21:11:08 +02:00
Merge pull request #663 from rowboatlabs/video1
Video mode with screenshare and app navigation
This commit is contained in:
arkml
GitHub
commit
51c289ba36
28 changed files with 2736 additions and 329 deletions
|
|
@ -1,7 +1,8 @@
|
|||
import { ipcMain, BrowserWindow, shell, dialog, systemPreferences, desktopCapturer, app } from 'electron';
|
||||
import { ipcMain, BrowserWindow, shell, dialog, systemPreferences, desktopCapturer, app, screen } from 'electron';
|
||||
import { ipc } from '@x/shared';
|
||||
import path from 'node:path';
|
||||
import os from 'node:os';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import {
|
||||
connectProvider,
|
||||
disconnectProvider,
|
||||
|
|
@ -383,9 +384,36 @@ type InvokeHandlers = {
|
|||
[K in InvokeChannels]: InvokeHandler<K>;
|
||||
};
|
||||
|
||||
// In-flight streaming TTS requests, keyed by renderer-chosen requestId.
|
||||
const activeTtsStreams = new Map<string, AbortController>();
|
||||
|
||||
// Video-mode popout window (shown for the whole duration of a screen share,
|
||||
// floating over every app including Rowboat itself) and the last call state
|
||||
// pushed by the main window — replayed to the popout when it finishes loading.
|
||||
let videoPopoutWin: BrowserWindow | null = null;
|
||||
let lastVideoPopoutState: {
|
||||
ttsState: 'idle' | 'synthesizing' | 'speaking';
|
||||
status: 'listening' | 'thinking' | 'speaking' | null;
|
||||
cameraOn: boolean;
|
||||
screenSharing: boolean;
|
||||
interimText: string | null;
|
||||
} | null = null;
|
||||
|
||||
// Match only real app windows — getAllWindows() can also contain the popout
|
||||
// itself and hidden utility windows (e.g. PDF-export renderers), which must
|
||||
// not be shown, focused, or sent app events.
|
||||
function findMainAppWindow(): BrowserWindow | undefined {
|
||||
return BrowserWindow.getAllWindows().find((w) => {
|
||||
if (w === videoPopoutWin || w.isDestroyed()) return false;
|
||||
const url = w.webContents.getURL();
|
||||
const isAppWindow = url.startsWith('app://') || url.startsWith('http://localhost');
|
||||
return isAppWindow && !url.includes('#video-popout');
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Register all IPC handlers with type safety and runtime validation
|
||||
*
|
||||
*
|
||||
* This function ensures:
|
||||
* 1. All invoke channels have handlers (exhaustiveness checking)
|
||||
* 2. Handler signatures match channel definitions
|
||||
|
|
@ -1727,6 +1755,51 @@ export function setupIpcHandlers() {
|
|||
'voice:synthesize': async (_event, args) => {
|
||||
return voice.synthesizeSpeech(args.text);
|
||||
},
|
||||
'voice:synthesizeStreamStart': async (event, args) => {
|
||||
const { requestId, text } = args;
|
||||
const sender = event.sender;
|
||||
const controller = new AbortController();
|
||||
activeTtsStreams.set(requestId, controller);
|
||||
// Fire-and-forget: chunks are pushed to the renderer as they arrive so
|
||||
// playback can begin immediately; the invoke returns once started.
|
||||
void voice
|
||||
.synthesizeSpeechStream(
|
||||
text,
|
||||
(chunk) => {
|
||||
if (!sender.isDestroyed()) {
|
||||
sender.send('voice:tts-chunk', {
|
||||
requestId,
|
||||
chunkBase64: chunk.toString('base64'),
|
||||
done: false,
|
||||
});
|
||||
}
|
||||
},
|
||||
controller.signal,
|
||||
)
|
||||
.then(() => {
|
||||
if (!sender.isDestroyed()) {
|
||||
sender.send('voice:tts-chunk', { requestId, done: true });
|
||||
}
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
if (!sender.isDestroyed() && !controller.signal.aborted) {
|
||||
sender.send('voice:tts-chunk', {
|
||||
requestId,
|
||||
done: true,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
}
|
||||
})
|
||||
.finally(() => {
|
||||
activeTtsStreams.delete(requestId);
|
||||
});
|
||||
return { ok: true };
|
||||
},
|
||||
'voice:synthesizeStreamCancel': async (_event, args) => {
|
||||
activeTtsStreams.get(args.requestId)?.abort();
|
||||
activeTtsStreams.delete(args.requestId);
|
||||
return {};
|
||||
},
|
||||
'voice:ensureMicAccess': async () => {
|
||||
if (process.platform !== 'darwin') return { granted: true };
|
||||
const status = systemPreferences.getMediaAccessStatus('microphone');
|
||||
|
|
@ -1745,6 +1818,113 @@ export function setupIpcHandlers() {
|
|||
return { granted: false };
|
||||
}
|
||||
},
|
||||
'voice:ensureCameraAccess': async () => {
|
||||
if (process.platform !== 'darwin') return { granted: true };
|
||||
const status = systemPreferences.getMediaAccessStatus('camera');
|
||||
console.log('[video] Camera permission status:', status);
|
||||
if (status === 'granted') return { granted: true };
|
||||
// Same flow as the microphone: settle the native TCC prompt before the
|
||||
// renderer's getUserMedia so the first video click doesn't silently fail.
|
||||
try {
|
||||
const granted = await systemPreferences.askForMediaAccess('camera');
|
||||
console.log('[video] Camera permission after prompt:', granted);
|
||||
return { granted };
|
||||
} catch {
|
||||
return { granted: false };
|
||||
}
|
||||
},
|
||||
'video:setPopout': async (_event, args) => {
|
||||
if (!args.show) {
|
||||
if (videoPopoutWin && !videoPopoutWin.isDestroyed()) videoPopoutWin.destroy();
|
||||
videoPopoutWin = null;
|
||||
return {};
|
||||
}
|
||||
if (videoPopoutWin && !videoPopoutWin.isDestroyed()) return {};
|
||||
|
||||
const workArea = screen.getPrimaryDisplay().workArea;
|
||||
const width = 340;
|
||||
const height = 184;
|
||||
const ipcDir = path.dirname(fileURLToPath(import.meta.url));
|
||||
const preloadPath = app.isPackaged
|
||||
? path.join(ipcDir, '../preload/dist/preload.js')
|
||||
: path.join(ipcDir, '../../../preload/dist/preload.js');
|
||||
const win = new BrowserWindow({
|
||||
width,
|
||||
height,
|
||||
x: workArea.x + workArea.width - width - 24,
|
||||
y: workArea.y + 24,
|
||||
frame: false,
|
||||
resizable: false,
|
||||
alwaysOnTop: true,
|
||||
skipTaskbar: true,
|
||||
show: false,
|
||||
hasShadow: true,
|
||||
backgroundColor: '#171717',
|
||||
webPreferences: {
|
||||
nodeIntegration: false,
|
||||
contextIsolation: true,
|
||||
sandbox: true,
|
||||
preload: preloadPath,
|
||||
},
|
||||
});
|
||||
// Float above other apps on every workspace. Deliberately NOT
|
||||
// `visibleOnFullScreen: true`: on macOS that flag hides the app's Dock
|
||||
// icon for as long as such a window exists (the app becomes an
|
||||
// "agent" app), which reads as Rowboat having vanished. The trade-off
|
||||
// is the popout won't hover over other apps' fullscreen Spaces.
|
||||
win.setAlwaysOnTop(true, 'floating');
|
||||
win.setVisibleOnAllWorkspaces(true);
|
||||
win.webContents.once('did-finish-load', () => {
|
||||
if (lastVideoPopoutState) {
|
||||
win.webContents.send('video:popout-state', lastVideoPopoutState);
|
||||
}
|
||||
// showInactive: appearing must not steal focus from the app the user
|
||||
// switched to — that would immediately re-hide the popout.
|
||||
if (!win.isDestroyed()) win.showInactive();
|
||||
});
|
||||
win.on('closed', () => {
|
||||
if (videoPopoutWin === win) videoPopoutWin = null;
|
||||
});
|
||||
videoPopoutWin = win;
|
||||
if (app.isPackaged) {
|
||||
win.loadURL('app://-/index.html#video-popout');
|
||||
} else {
|
||||
win.loadURL('http://localhost:5173/#video-popout');
|
||||
}
|
||||
return {};
|
||||
},
|
||||
'video:popoutState': async (_event, args) => {
|
||||
lastVideoPopoutState = args;
|
||||
if (videoPopoutWin && !videoPopoutWin.isDestroyed()) {
|
||||
videoPopoutWin.webContents.send('video:popout-state', args);
|
||||
}
|
||||
return {};
|
||||
},
|
||||
'app:focusMainWindow': async () => {
|
||||
const main = findMainAppWindow();
|
||||
if (main) {
|
||||
if (main.isMinimized()) main.restore();
|
||||
main.show();
|
||||
main.focus();
|
||||
}
|
||||
return {};
|
||||
},
|
||||
'video:getPopoutState': async () => {
|
||||
return { state: lastVideoPopoutState };
|
||||
},
|
||||
'video:popoutAction': async (_event, args) => {
|
||||
// Relay a popout control-bar action to the app window, which owns the
|
||||
// call (mic, camera, screen capture) and executes it there. 'expand'
|
||||
// additionally brings the app window back to the foreground.
|
||||
const main = findMainAppWindow();
|
||||
if (args.action === 'expand' && main) {
|
||||
if (main.isMinimized()) main.restore();
|
||||
main.show();
|
||||
main.focus();
|
||||
}
|
||||
main?.webContents.send('video:popout-action', args);
|
||||
return {};
|
||||
},
|
||||
// Live-note handlers
|
||||
'live-note:run': async (_event, args) => {
|
||||
const result = await runLiveNoteAgent(args.filePath, 'manual', args.context);
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ import { ChatSidebar } from './components/chat-sidebar';
|
|||
import { useSessionChat } from '@/hooks/useSessionChat';
|
||||
import { ChatHeader } from './components/chat-header';
|
||||
import { ChatEmptyState } from './components/chat-empty-state';
|
||||
import { ChatInputWithMentions, type PermissionMode, type StagedAttachment } from './components/chat-input-with-mentions';
|
||||
import { ChatInputWithMentions, type CallPreset, type PermissionMode, type StagedAttachment } from './components/chat-input-with-mentions';
|
||||
import { ChatMessageAttachments } from '@/components/chat-message-attachments'
|
||||
import { GraphView, type GraphEdge, type GraphNode } from '@/components/graph-view';
|
||||
import { BasesView, type BaseConfig, DEFAULT_BASE_CONFIG } from '@/components/bases-view';
|
||||
|
|
@ -97,6 +97,7 @@ import {
|
|||
type ChatViewportAnchorState,
|
||||
type ChatTabViewState,
|
||||
type ConversationItem,
|
||||
type ToolCall,
|
||||
createEmptyChatTabViewState,
|
||||
getWebSearchCardData,
|
||||
getAppActionCardData,
|
||||
|
|
@ -118,12 +119,14 @@ import { AgentScheduleConfig } from '@x/shared/dist/agent-schedule.js'
|
|||
import { AgentScheduleState } from '@x/shared/dist/agent-schedule-state.js'
|
||||
import { toast } from "sonner"
|
||||
import { useVoiceMode } from '@/hooks/useVoiceMode'
|
||||
import { useVideoMode } from '@/hooks/useVideoMode'
|
||||
import { useVoiceTTS } from '@/hooks/useVoiceTTS'
|
||||
import { TalkingHeadOverlay } from '@/components/talking-head'
|
||||
import { VideoCallView } from '@/components/video-call-view'
|
||||
import { ProductTour, type TourNavTarget } from '@/components/product-tour'
|
||||
import { useMeetingTranscription, type CalendarEventMeta } from '@/hooks/useMeetingTranscription'
|
||||
import { useAnalyticsIdentity } from '@/hooks/useAnalyticsIdentity'
|
||||
import * as analytics from '@/lib/analytics'
|
||||
import { playAckCue } from '@/lib/call-sounds'
|
||||
import { useTheme } from '@/contexts/theme-context'
|
||||
|
||||
type DirEntry = z.infer<typeof workspace.DirEntry>
|
||||
|
|
@ -969,11 +972,19 @@ function App() {
|
|||
// Voice mode state
|
||||
const [voiceAvailable, setVoiceAvailable] = useState(false)
|
||||
const [ttsAvailable, setTtsAvailable] = useState(false)
|
||||
const [ttsEnabled, setTtsEnabled] = useState(false)
|
||||
// TTS plays only during calls now (the standing read-aloud toggle was
|
||||
// retired; a per-message "read aloud" action may replace it later).
|
||||
const ttsEnabledRef = useRef(false)
|
||||
const [ttsMode, setTtsMode] = useState<'summary' | 'full'>('summary')
|
||||
// Voice-to-voice latency marks for the current call turn (performance.now):
|
||||
// t0 = utterance accepted, submit = message sent, speak = first TTS
|
||||
// speak(). Emitted as call_turn_latency when audio actually starts.
|
||||
const callTurnMarksRef = useRef<{ t0: number; submit?: number; speak?: number } | null>(null)
|
||||
// Late-bound handle to handleStop (defined much further down) so early
|
||||
// call handlers can stop the run without reordering the component.
|
||||
const stopRunRef = useRef<(() => Promise<void>) | null>(null)
|
||||
// Read-aloud style: 'summary' for typed chat, forced to 'full' during a
|
||||
// call and restored after. Context decides — the user never picks it.
|
||||
const ttsModeRef = useRef<'summary' | 'full'>('summary')
|
||||
const [ttsAvatarEnabled, setTtsAvatarEnabled] = useState(false)
|
||||
const [tourActive, setTourActive] = useState(false)
|
||||
const [isRecording, setIsRecording] = useState(false)
|
||||
const voiceTextBufferRef = useRef('')
|
||||
|
|
@ -984,6 +995,13 @@ function App() {
|
|||
const ttsRef = useRef(tts)
|
||||
ttsRef.current = tts
|
||||
|
||||
// Latest assistant line handed to TTS — shown as the caption in the
|
||||
// full-screen call view while the assistant is speaking.
|
||||
const [assistantCaption, setAssistantCaption] = useState('')
|
||||
useEffect(() => {
|
||||
if (tts.state === 'idle') setAssistantCaption('')
|
||||
}, [tts.state])
|
||||
|
||||
// Speak newly completed <voice> blocks from the new runtime's live stream
|
||||
// (parity with the legacy text-delta voice extraction below). The store
|
||||
// accumulates completed blocks in chatState.voiceSegments; we speak only
|
||||
|
|
@ -1001,15 +1019,47 @@ function App() {
|
|||
const segment = voiceSegments[spokenVoiceRef.current.count]
|
||||
spokenVoiceRef.current.count += 1
|
||||
if (ttsEnabledRef.current) {
|
||||
const marks = callTurnMarksRef.current
|
||||
if (marks && marks.speak === undefined) marks.speak = performance.now()
|
||||
ttsRef.current.speak(segment)
|
||||
setAssistantCaption(segment)
|
||||
}
|
||||
}
|
||||
}, [voiceSegments, runId])
|
||||
|
||||
// Emit the turn's voice-to-voice latency breakdown once audio is audible.
|
||||
useEffect(() => {
|
||||
if (tts.state !== 'speaking') return
|
||||
const marks = callTurnMarksRef.current
|
||||
if (!marks || marks.submit === undefined || marks.speak === undefined) return
|
||||
callTurnMarksRef.current = null
|
||||
const now = performance.now()
|
||||
analytics.callTurnLatency({
|
||||
endpointToSubmitMs: marks.submit - marks.t0,
|
||||
submitToSpeakMs: marks.speak - marks.submit,
|
||||
speakToAudioMs: now - marks.speak,
|
||||
totalMs: now - marks.t0,
|
||||
})
|
||||
}, [tts.state])
|
||||
|
||||
const voice = useVoiceMode()
|
||||
const voiceRef = useRef(voice)
|
||||
voiceRef.current = voice
|
||||
|
||||
// Calls: one engine (hands-free voice loop + forced read-aloud TTS + frame
|
||||
// capture), started via presets that only differ in device defaults. The
|
||||
// presentation is DERIVED from devices, never picked: screen sharing →
|
||||
// floating popout; camera on → full-screen call; camera off → popout
|
||||
// (mascot pill). Handlers live below the voice/submit plumbing they drive.
|
||||
const video = useVideoMode()
|
||||
const [inCall, setInCall] = useState(false)
|
||||
const inCallRef = useRef(false)
|
||||
// User explicitly shrank the full-screen call to the floating pill.
|
||||
const [callMinimized, setCallMinimized] = useState(false)
|
||||
// Practice preset: adds the coaching persona to the system prompt.
|
||||
const [practiceMode, setPracticeMode] = useState(false)
|
||||
const practiceModeRef = useRef(false)
|
||||
|
||||
const handleToggleMeetingRef = useRef<(() => void) | undefined>(undefined)
|
||||
const meetingTranscription = useMeetingTranscription(() => {
|
||||
handleToggleMeetingRef.current?.()
|
||||
|
|
@ -1068,6 +1118,8 @@ function App() {
|
|||
}, [])
|
||||
|
||||
const handleStartRecording = useCallback(() => {
|
||||
// A live call owns the mic — ignore push-to-talk while one is running.
|
||||
if (inCallRef.current) return
|
||||
setIsRecording(true)
|
||||
isRecordingRef.current = true
|
||||
voice.start()
|
||||
|
|
@ -1092,43 +1144,207 @@ function App() {
|
|||
}
|
||||
}, [voice])
|
||||
|
||||
const handleToggleTts = useCallback(() => {
|
||||
setTtsEnabled(prev => {
|
||||
const next = !prev
|
||||
ttsEnabledRef.current = next
|
||||
if (!next) {
|
||||
ttsRef.current.cancel()
|
||||
setTtsAvatarEnabled(false)
|
||||
}
|
||||
return next
|
||||
})
|
||||
}, [])
|
||||
|
||||
// Talking-head mode implies voice output: enabling it turns TTS on,
|
||||
// disabling it turns both off.
|
||||
const handleToggleTtsAvatar = useCallback(() => {
|
||||
setTtsAvatarEnabled(prev => {
|
||||
const next = !prev
|
||||
setTtsEnabled(next)
|
||||
ttsEnabledRef.current = next
|
||||
if (!next) {
|
||||
ttsRef.current.cancel()
|
||||
}
|
||||
return next
|
||||
})
|
||||
}, [])
|
||||
|
||||
const handleTtsModeChange = useCallback((mode: 'summary' | 'full') => {
|
||||
setTtsMode(mode)
|
||||
ttsModeRef.current = mode
|
||||
}, [])
|
||||
|
||||
const handleCancelRecording = useCallback(() => {
|
||||
voice.cancel()
|
||||
setIsRecording(false)
|
||||
isRecordingRef.current = false
|
||||
}, [voice])
|
||||
|
||||
// Start a call. Presets only differ in device defaults — the engine
|
||||
// (continuous listening, auto-submitted utterances, forced read-aloud TTS,
|
||||
// frame capture) is identical for all of them. The default entry ('share',
|
||||
// the call button's main click) is "work together": screen shared, camera
|
||||
// off, floating pill — the user keeps working while the assistant watches
|
||||
// along. 'video'/'practice' open face-to-face full screen instead.
|
||||
const startCall = useCallback(async (preset: CallPreset) => {
|
||||
if (inCallRef.current) return
|
||||
const camera = preset === 'video' || preset === 'practice'
|
||||
const ok = await video.start({ camera })
|
||||
if (!ok) return // camera denied/unavailable — stay out of the call
|
||||
if (preset === 'share') {
|
||||
// If screen capture fails (usually the macOS Screen Recording
|
||||
// permission), continue as a voice call — sharing is one tap away on
|
||||
// the pill once permission is granted.
|
||||
const shared = await video.startScreenShare()
|
||||
if (!shared) {
|
||||
toast("Couldn't share your screen", {
|
||||
description: 'Grant Rowboat Screen Recording access, then tap the share button on the call.',
|
||||
action: {
|
||||
label: 'Open Settings',
|
||||
onClick: () => void window.ipc.invoke('meeting:openScreenRecordingSettings', null).catch(() => {}),
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A manual push-to-talk recording can't coexist with the call's mic.
|
||||
if (isRecordingRef.current) {
|
||||
voiceRef.current.cancel()
|
||||
setIsRecording(false)
|
||||
isRecordingRef.current = false
|
||||
}
|
||||
ttsEnabledRef.current = true
|
||||
ttsModeRef.current = 'full'
|
||||
void voiceRef.current.startContinuous((text) => {
|
||||
// Instant "heard you" feedback + start of the latency clock.
|
||||
playAckCue()
|
||||
callTurnMarksRef.current = { t0: performance.now() }
|
||||
pendingVoiceInputRef.current = true
|
||||
handlePromptSubmitRef.current?.({ text, files: [] })
|
||||
})
|
||||
|
||||
setPracticeMode(preset === 'practice')
|
||||
practiceModeRef.current = preset === 'practice'
|
||||
// Pill-first presets start minimized; face-to-face presets start expanded.
|
||||
setCallMinimized(preset === 'voice' || preset === 'share')
|
||||
inCallRef.current = true
|
||||
setInCall(true)
|
||||
analytics.callStarted(preset)
|
||||
}, [video])
|
||||
|
||||
const endCall = useCallback(() => {
|
||||
if (!inCallRef.current) return
|
||||
voiceRef.current.cancel()
|
||||
ttsEnabledRef.current = false
|
||||
ttsModeRef.current = 'summary'
|
||||
ttsRef.current.cancel()
|
||||
callTurnMarksRef.current = null
|
||||
video.stop()
|
||||
setPracticeMode(false)
|
||||
practiceModeRef.current = false
|
||||
setCallMinimized(false)
|
||||
inCallRef.current = false
|
||||
setInCall(false)
|
||||
}, [video])
|
||||
|
||||
// During a call, mute the mic while the assistant is thinking or speaking
|
||||
// so its own TTS (or a half-turn) never gets transcribed back at it.
|
||||
useEffect(() => {
|
||||
if (!inCall) return
|
||||
voiceRef.current.setPaused(activeIsProcessing || tts.state !== 'idle')
|
||||
}, [inCall, activeIsProcessing, tts.state])
|
||||
|
||||
// Screen sharing: frames of the shared screen ride along with each message
|
||||
// next to the webcam frames. The surface change (full screen → pill) falls
|
||||
// out of the derivation below.
|
||||
const handleToggleScreenShare = useCallback(async () => {
|
||||
if (video.screenState === 'live') {
|
||||
video.stopScreenShare()
|
||||
} else {
|
||||
await video.startScreenShare()
|
||||
}
|
||||
}, [video])
|
||||
|
||||
// Meet-style camera mute: the call (and any screen share) stays on, but no
|
||||
// webcam frames are captured while the camera is off. Deliberately does NOT
|
||||
// change the surface — turning your camera on from the pill puts your video
|
||||
// IN the pill; expanding to full screen is its own explicit action.
|
||||
const handleToggleCamera = useCallback(() => {
|
||||
void video.setCameraEnabled(!video.cameraOn)
|
||||
}, [video])
|
||||
|
||||
// Minimizing the full-screen call drops you back to working — and the pill
|
||||
// exists to work *together*, so sharing starts automatically (the symmetric
|
||||
// twin of expand, which stops it). If capture fails (permission), the call
|
||||
// still minimizes as a plain pill. `callMinimized` is also set so stopping
|
||||
// the share from the pill keeps you in the pill rather than snapping back
|
||||
// to full screen.
|
||||
const handleMinimizeCall = useCallback(async () => {
|
||||
setCallMinimized(true)
|
||||
await video.startScreenShare()
|
||||
}, [video])
|
||||
|
||||
// Interrupt the assistant: silence TTS immediately, skip anything already
|
||||
// queued from the in-flight turn, and stop the run if it's still
|
||||
// generating (if it already finished, stopping the speech is all there is
|
||||
// to do). Wired to the Stop control next to the mascot on both surfaces.
|
||||
const handleInterruptAssistant = useCallback(() => {
|
||||
ttsRef.current.cancel()
|
||||
setAssistantCaption('')
|
||||
if (voiceSegments) {
|
||||
spokenVoiceRef.current.count = voiceSegments.length
|
||||
}
|
||||
if (activeIsProcessing) {
|
||||
void stopRunRef.current?.()
|
||||
}
|
||||
}, [voiceSegments, activeIsProcessing])
|
||||
|
||||
// Current phase of the call (null when not in one).
|
||||
const videoCallStatus: 'listening' | 'thinking' | 'speaking' | null =
|
||||
inCall
|
||||
? tts.state === 'speaking'
|
||||
? 'speaking'
|
||||
: tts.state === 'synthesizing' || activeIsProcessing
|
||||
? 'thinking'
|
||||
: 'listening'
|
||||
: null
|
||||
|
||||
// The call's surface follows one rule: full screen and screen sharing are
|
||||
// mutually exclusive (a full-screen call covers the screen — sharing it
|
||||
// would show the call itself). Sharing → floating pill, always. Not
|
||||
// sharing → full screen unless the user shrank it (`callMinimized`).
|
||||
// Expanding the pill auto-stops any share; presenting from full screen
|
||||
// auto-collapses to the pill.
|
||||
const callSurface: 'fullscreen' | 'popout' | null = !inCall
|
||||
? null
|
||||
: video.screenState === 'live' || callMinimized
|
||||
? 'popout'
|
||||
: 'fullscreen'
|
||||
|
||||
useEffect(() => {
|
||||
void window.ipc.invoke('video:setPopout', { show: callSurface === 'popout' }).catch(() => {})
|
||||
}, [callSurface])
|
||||
|
||||
// Consent surface for screen sharing: an unmissable toast the moment any
|
||||
// share starts (auto-started calls included), with one-tap stop. The pill
|
||||
// also carries a persistent "Sharing screen" badge, and macOS shows its
|
||||
// purple recording indicator.
|
||||
const prevScreenStateRef = useRef(video.screenState)
|
||||
useEffect(() => {
|
||||
const prev = prevScreenStateRef.current
|
||||
prevScreenStateRef.current = video.screenState
|
||||
if (video.screenState === 'live' && prev !== 'live') {
|
||||
toast('Your screen is being shared', {
|
||||
description: 'The assistant sees snapshots of it along with what you say.',
|
||||
action: { label: 'Stop sharing', onClick: () => video.stopScreenShare() },
|
||||
duration: 6000,
|
||||
})
|
||||
}
|
||||
}, [video.screenState, video])
|
||||
|
||||
// Keep the popout's mascot/status/devices/caption mirror of the call fresh.
|
||||
// The main process caches the latest state and replays it when the popout
|
||||
// loads.
|
||||
useEffect(() => {
|
||||
if (!inCall) return
|
||||
void window.ipc
|
||||
.invoke('video:popoutState', {
|
||||
ttsState: tts.state,
|
||||
status: videoCallStatus,
|
||||
cameraOn: video.cameraOn,
|
||||
screenSharing: video.screenState === 'live',
|
||||
interimText: voice.interimText || null,
|
||||
})
|
||||
.catch(() => {})
|
||||
}, [inCall, tts.state, videoCallStatus, video.cameraOn, video.screenState, voice.interimText])
|
||||
|
||||
// Execute popout control-bar actions (the popout window has no access to
|
||||
// the call's mic/camera/capture — they live here). 'expand' goes full
|
||||
// screen, which by the exclusivity rule stops any running share; the main
|
||||
// process already refocused the app window.
|
||||
useEffect(() => {
|
||||
return window.ipc.on('video:popout-action', ({ action }) => {
|
||||
if (action === 'toggle-camera') handleToggleCamera()
|
||||
else if (action === 'toggle-share') void handleToggleScreenShare()
|
||||
else if (action === 'stop-speaking') handleInterruptAssistant()
|
||||
else if (action === 'end-call') endCall()
|
||||
else if (action === 'expand') {
|
||||
if (video.screenState === 'live') video.stopScreenShare()
|
||||
setCallMinimized(false)
|
||||
}
|
||||
})
|
||||
}, [handleToggleCamera, handleToggleScreenShare, handleInterruptAssistant, endCall, video])
|
||||
|
||||
// Enter to submit voice input, Escape to cancel
|
||||
useEffect(() => {
|
||||
const handleKeyDown = (e: KeyboardEvent) => {
|
||||
|
|
@ -2253,6 +2469,7 @@ function App() {
|
|||
console.log('[voice] extracted voice tag:', voiceContent)
|
||||
if (voiceContent && ttsEnabledRef.current) {
|
||||
ttsRef.current.speak(voiceContent)
|
||||
setAssistantCaption(voiceContent)
|
||||
}
|
||||
spokenIndexRef.current += voiceMatch.index + voiceMatch[0].length
|
||||
}
|
||||
|
|
@ -2608,15 +2825,33 @@ function App() {
|
|||
|
||||
setMessage('')
|
||||
|
||||
// Video chat mode: drain the webcam frames buffered since the last send
|
||||
// so they ride along with this message as inline image parts.
|
||||
const marks = callTurnMarksRef.current
|
||||
if (inCallRef.current && marks && marks.submit === undefined) {
|
||||
marks.submit = performance.now()
|
||||
}
|
||||
|
||||
const videoFrames = inCallRef.current ? video.collectFrames() : []
|
||||
|
||||
const userMessageId = `user-${Date.now()}`
|
||||
const displayAttachments: ChatMessage['attachments'] = hasAttachments
|
||||
? stagedAttachments.map((attachment) => ({
|
||||
path: attachment.path,
|
||||
filename: attachment.filename,
|
||||
mimeType: attachment.mimeType,
|
||||
size: attachment.size,
|
||||
thumbnailUrl: attachment.thumbnailUrl,
|
||||
}))
|
||||
const displayAttachments: ChatMessage['attachments'] = hasAttachments || videoFrames.length > 0
|
||||
? [
|
||||
...stagedAttachments.map((attachment) => ({
|
||||
path: attachment.path,
|
||||
filename: attachment.filename,
|
||||
mimeType: attachment.mimeType,
|
||||
size: attachment.size,
|
||||
thumbnailUrl: attachment.thumbnailUrl,
|
||||
})),
|
||||
...videoFrames.map((frame, index) => ({
|
||||
path: '',
|
||||
filename: `${frame.source}-frame-${index + 1}.jpg`,
|
||||
mimeType: frame.mediaType,
|
||||
thumbnailUrl: frame.dataUrl,
|
||||
isVideoFrame: true,
|
||||
})),
|
||||
]
|
||||
: undefined
|
||||
setConversation((prev) => [...prev, {
|
||||
id: userMessageId,
|
||||
|
|
@ -2668,6 +2903,8 @@ function App() {
|
|||
...(ttsEnabledRef.current ? { voiceOutput: ttsModeRef.current } : {}),
|
||||
...(searchEnabled ? { searchEnabled: true } : {}),
|
||||
...(codeMode ? { codeMode } : {}),
|
||||
...(inCallRef.current && (video.cameraOn || video.screenState === 'live') ? { videoMode: true } : {}),
|
||||
...(practiceModeRef.current ? { coachMode: true } : {}),
|
||||
},
|
||||
},
|
||||
},
|
||||
|
|
@ -2678,7 +2915,7 @@ function App() {
|
|||
middlePane: middlePane ?? { kind: 'empty' as const },
|
||||
})
|
||||
|
||||
if (hasAttachments || hasMentions) {
|
||||
if (hasAttachments || hasMentions || videoFrames.length > 0) {
|
||||
type ContentPart =
|
||||
| { type: 'text'; text: string }
|
||||
| {
|
||||
|
|
@ -2689,6 +2926,13 @@ function App() {
|
|||
size?: number
|
||||
lineNumber?: number
|
||||
}
|
||||
| {
|
||||
type: 'image'
|
||||
data: string
|
||||
mediaType: string
|
||||
source: 'camera' | 'screen'
|
||||
capturedAt: string
|
||||
}
|
||||
|
||||
const contentParts: ContentPart[] = []
|
||||
|
||||
|
|
@ -2720,6 +2964,16 @@ function App() {
|
|||
titleSource = stagedAttachments[0]?.filename ?? mentions?.[0]?.displayName ?? mentions?.[0]?.path ?? ''
|
||||
}
|
||||
|
||||
for (const frame of videoFrames) {
|
||||
contentParts.push({
|
||||
type: 'image',
|
||||
data: frame.data,
|
||||
mediaType: frame.mediaType,
|
||||
source: frame.source,
|
||||
capturedAt: frame.capturedAt,
|
||||
})
|
||||
}
|
||||
|
||||
const middlePaneContext = await buildMiddlePaneContext()
|
||||
await window.ipc.invoke('sessions:sendMessage', {
|
||||
sessionId: currentRunId,
|
||||
|
|
@ -2793,12 +3047,18 @@ function App() {
|
|||
if (!runId) return
|
||||
setStopClickedAt(Date.now())
|
||||
setIsStopping(true)
|
||||
// Stopping the run must also silence it — the TTS queue holds segments
|
||||
// that were already extracted from the stream and would keep playing
|
||||
// long after the turn is aborted.
|
||||
ttsRef.current.cancel()
|
||||
setAssistantCaption('')
|
||||
try {
|
||||
await sessionChat.stop()
|
||||
} catch (error) {
|
||||
console.error('Failed to stop turn:', error)
|
||||
}
|
||||
}, [runId, sessionChat])
|
||||
stopRunRef.current = handleStop
|
||||
|
||||
const handlePermissionResponse = useCallback(async (
|
||||
toolCallId: string,
|
||||
|
|
@ -4310,22 +4570,61 @@ function App() {
|
|||
// External search set by app-navigation tool (passed to BasesView)
|
||||
const [externalBaseSearch, setExternalBaseSearch] = useState<string | undefined>(undefined)
|
||||
|
||||
// Process pending app-navigation results
|
||||
useEffect(() => {
|
||||
const result = pendingAppNavRef.current
|
||||
if (!result) return
|
||||
pendingAppNavRef.current = null
|
||||
// Apply an app-navigation tool result to the UI. Shared by both event
|
||||
// paths (legacy runs:events and the session-chat turn runtime).
|
||||
const applyAppNavigation = useCallback((result: Record<string, unknown>) => {
|
||||
// During a call, navigation must be VISIBLE: the full-screen call view
|
||||
// would cover the very thing being shown — collapse it to the pill —
|
||||
// and if the user is in another app, bring Rowboat forward.
|
||||
const visibleActions = ['open-note', 'open-view', 'read-view', 'open-item', 'update-base-view', 'create-base']
|
||||
if (inCallRef.current && visibleActions.includes(result.action as string)) {
|
||||
setCallMinimized(true)
|
||||
void window.ipc.invoke('app:focusMainWindow', null).catch(() => {})
|
||||
}
|
||||
|
||||
// Views the assistant can open (or auto-open while reading them via
|
||||
// read-view — the user should SEE what's being read).
|
||||
const navigateToNamedView = (view: string) => {
|
||||
switch (view) {
|
||||
case 'graph': void navigateToView({ type: 'graph' }); break
|
||||
case 'bases': void navigateToView({ type: 'file', path: BASES_DEFAULT_TAB_PATH }); break
|
||||
case 'home': void navigateToView({ type: 'home' }); break
|
||||
case 'email': void navigateToView({ type: 'email' }); break
|
||||
case 'meetings': void navigateToView({ type: 'meetings' }); break
|
||||
case 'live-notes': void navigateToView({ type: 'live-notes' }); break
|
||||
case 'bg-tasks': void navigateToView({ type: 'bg-tasks' }); break
|
||||
case 'chat-history': void navigateToView({ type: 'chat-history' }); break
|
||||
case 'knowledge': void navigateToView({ type: 'knowledge-view' }); break
|
||||
case 'workspace': void navigateToView({ type: 'workspace' }); break
|
||||
case 'code': void navigateToView({ type: 'code' }); break
|
||||
}
|
||||
}
|
||||
|
||||
switch (result.action) {
|
||||
case 'open-note':
|
||||
navigateToFile(result.path as string)
|
||||
break
|
||||
case 'open-view':
|
||||
if (result.view === 'graph') void navigateToView({ type: 'graph' })
|
||||
if (result.view === 'bases') {
|
||||
void navigateToView({ type: 'file', path: BASES_DEFAULT_TAB_PATH })
|
||||
case 'read-view':
|
||||
navigateToNamedView(result.view as string)
|
||||
break
|
||||
case 'open-item': {
|
||||
switch (result.kind) {
|
||||
case 'email-thread':
|
||||
void navigateToView({ type: 'email', threadId: result.threadId as string })
|
||||
break
|
||||
case 'note':
|
||||
navigateToFile(result.path as string)
|
||||
break
|
||||
case 'bg-task':
|
||||
void navigateToView({ type: 'task', name: result.taskName as string })
|
||||
break
|
||||
case 'session':
|
||||
void navigateToView({ type: 'chat', runId: result.sessionId as string })
|
||||
break
|
||||
}
|
||||
break
|
||||
}
|
||||
case 'update-base-view': {
|
||||
// Navigate to bases if not already there
|
||||
const targetPath = selectedPath && isBaseFilePath(selectedPath) ? selectedPath : BASES_DEFAULT_TAB_PATH
|
||||
|
|
@ -4405,8 +4704,41 @@ function App() {
|
|||
}
|
||||
break
|
||||
}
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [navigateToFile, navigateToView, selectedPath])
|
||||
|
||||
// Legacy runs:events path: handleRunEvent stashes the result in a ref;
|
||||
// polled every render (the triggering event always causes one).
|
||||
useEffect(() => {
|
||||
const result = pendingAppNavRef.current
|
||||
if (!result) return
|
||||
pendingAppNavRef.current = null
|
||||
applyAppNavigation(result)
|
||||
})
|
||||
|
||||
// Turn-runtime path: the session-chat store surfaces tool results in the
|
||||
// conversation; apply newly completed app-navigation calls exactly once.
|
||||
// On session switch/load, everything already in the transcript happened in
|
||||
// the past — seed as processed without replaying navigations.
|
||||
const processedAppNavRef = useRef<{ key: string | null; ids: Set<string> }>({ key: null, ids: new Set() })
|
||||
useEffect(() => {
|
||||
const conversation = sessionChat.chatState?.conversation
|
||||
if (!conversation) return
|
||||
const completed = conversation.filter(
|
||||
(item): item is ToolCall => isToolCall(item) && item.name === 'app-navigation' && item.status === 'completed'
|
||||
)
|
||||
if (processedAppNavRef.current.key !== runId) {
|
||||
processedAppNavRef.current = { key: runId, ids: new Set(completed.map((t) => t.id)) }
|
||||
return
|
||||
}
|
||||
for (const tool of completed) {
|
||||
if (processedAppNavRef.current.ids.has(tool.id)) continue
|
||||
processedAppNavRef.current.ids.add(tool.id)
|
||||
const result = tool.result as Record<string, unknown> | undefined
|
||||
if (result && result.success) applyAppNavigation(result)
|
||||
}
|
||||
}, [sessionChat.chatState?.conversation, runId, applyAppNavigation])
|
||||
|
||||
const navigateToFullScreenChat = useCallback(() => {
|
||||
// Only treat this as navigation when coming from another view
|
||||
if (currentViewState.type !== 'chat') {
|
||||
|
|
@ -6330,13 +6662,10 @@ function App() {
|
|||
onSubmitRecording={isActive ? handleSubmitRecording : undefined}
|
||||
onCancelRecording={isActive ? handleCancelRecording : undefined}
|
||||
voiceAvailable={isActive && voiceAvailable}
|
||||
ttsAvailable={isActive && ttsAvailable}
|
||||
ttsEnabled={ttsEnabled}
|
||||
ttsMode={ttsMode}
|
||||
onToggleTts={isActive ? handleToggleTts : undefined}
|
||||
onTtsModeChange={isActive ? handleTtsModeChange : undefined}
|
||||
ttsAvatarEnabled={ttsAvatarEnabled}
|
||||
onToggleTtsAvatar={isActive ? handleToggleTtsAvatar : undefined}
|
||||
inCall={inCall}
|
||||
onStartCall={isActive ? startCall : undefined}
|
||||
onEndCall={isActive ? endCall : undefined}
|
||||
callAvailable={voiceAvailable && ttsAvailable}
|
||||
/>
|
||||
</div>
|
||||
)
|
||||
|
|
@ -6443,23 +6772,32 @@ function App() {
|
|||
onSubmitRecording={handleSubmitRecording}
|
||||
onCancelRecording={handleCancelRecording}
|
||||
voiceAvailable={voiceAvailable}
|
||||
ttsAvailable={ttsAvailable}
|
||||
ttsEnabled={ttsEnabled}
|
||||
ttsMode={ttsMode}
|
||||
onToggleTts={handleToggleTts}
|
||||
onTtsModeChange={handleTtsModeChange}
|
||||
ttsAvatarEnabled={ttsAvatarEnabled}
|
||||
onToggleTtsAvatar={handleToggleTtsAvatar}
|
||||
inCall={inCall}
|
||||
onStartCall={startCall}
|
||||
onEndCall={endCall}
|
||||
callAvailable={voiceAvailable && ttsAvailable}
|
||||
onComposioConnected={handleComposioConnected}
|
||||
/>
|
||||
)}
|
||||
{/* Talking head hovers over the active view while avatar voice mode is
|
||||
on (hidden during the tour, which shows its own mascot) */}
|
||||
{ttsAvatarEnabled && !tourActive && (
|
||||
<TalkingHeadOverlay
|
||||
{/* Full-screen call: user tile + animated mascot tile. Shown only
|
||||
when the derived surface says so (camera on, no screen share,
|
||||
not minimized) — otherwise the call lives in the floating
|
||||
popout window. */}
|
||||
{callSurface === 'fullscreen' && (
|
||||
<VideoCallView
|
||||
streamRef={video.streamRef}
|
||||
onToggleScreenShare={handleToggleScreenShare}
|
||||
cameraOn={video.cameraOn}
|
||||
onToggleCamera={handleToggleCamera}
|
||||
practiceMode={practiceMode}
|
||||
onMinimize={() => void handleMinimizeCall()}
|
||||
onInterrupt={handleInterruptAssistant}
|
||||
ttsState={tts.state}
|
||||
getLevel={tts.getLevel}
|
||||
onDismiss={handleToggleTtsAvatar}
|
||||
getTtsLevel={tts.getLevel}
|
||||
status={videoCallStatus ?? 'listening'}
|
||||
interimText={voice.interimText}
|
||||
assistantCaption={assistantCaption}
|
||||
onLeave={endCall}
|
||||
/>
|
||||
)}
|
||||
{/* Mascot-guided product tour */}
|
||||
|
|
|
|||
|
|
@ -15,16 +15,19 @@ import {
|
|||
FolderCog,
|
||||
FolderOpen,
|
||||
Globe,
|
||||
Headphones,
|
||||
ImagePlus,
|
||||
LoaderIcon,
|
||||
Lock,
|
||||
Mic,
|
||||
MoreHorizontal,
|
||||
Phone,
|
||||
PhoneOff,
|
||||
Plus,
|
||||
Presentation,
|
||||
ShieldCheck,
|
||||
Square,
|
||||
Terminal,
|
||||
Video,
|
||||
X,
|
||||
} from 'lucide-react'
|
||||
|
||||
|
|
@ -50,7 +53,6 @@ import {
|
|||
} from '@/lib/attachment-presentation'
|
||||
import { getExtension, getFileDisplayName, getMimeFromExtension, isImageMime } from '@/lib/file-utils'
|
||||
import { cn } from '@/lib/utils'
|
||||
import { MascotFaceIcon } from '@/components/talking-head'
|
||||
import {
|
||||
type FileMention,
|
||||
type PromptInputMessage,
|
||||
|
|
@ -211,6 +213,18 @@ function compactWorkDirPath(path: string) {
|
|||
return path.replace(/^\/Users\/[^/]+/, '~')
|
||||
}
|
||||
|
||||
// Call presets: front doors into the same call engine, differing only in
|
||||
// starting devices. 'share' is the call button's main click — the "work
|
||||
// together" default (screen shared, camera off, floating pill). The chevron
|
||||
// menu holds the deviations.
|
||||
export type CallPreset = 'voice' | 'video' | 'share' | 'practice'
|
||||
|
||||
const CALL_PRESET_MENU: Array<{ preset: CallPreset; label: string; description: string; Icon: typeof Phone }> = [
|
||||
{ preset: 'voice', label: 'Voice call', description: 'Just talk — nothing is shared, the mascot hovers while you work', Icon: AudioLines },
|
||||
{ preset: 'video', label: 'Video call', description: 'Camera on, face to face — it sees your expressions', Icon: Video },
|
||||
{ preset: 'practice', label: 'Practice session', description: 'Rehearse a pitch or interview with live coaching', Icon: Presentation },
|
||||
]
|
||||
|
||||
interface ChatInputInnerProps {
|
||||
onSubmit: (message: PromptInputMessage, mentions?: FileMention[], attachments?: StagedAttachment[], searchEnabled?: boolean, codeMode?: 'claude' | 'codex', permissionMode?: PermissionMode) => void
|
||||
onStop?: () => void
|
||||
|
|
@ -231,13 +245,13 @@ interface ChatInputInnerProps {
|
|||
onSubmitRecording?: () => void | Promise<void>
|
||||
onCancelRecording?: () => void
|
||||
voiceAvailable?: boolean
|
||||
ttsAvailable?: boolean
|
||||
ttsEnabled?: boolean
|
||||
ttsMode?: 'summary' | 'full'
|
||||
onToggleTts?: () => void
|
||||
onTtsModeChange?: (mode: 'summary' | 'full') => void
|
||||
ttsAvatarEnabled?: boolean
|
||||
onToggleTtsAvatar?: () => void
|
||||
/** A call is live (hands-free voice loop + spoken responses). */
|
||||
inCall?: boolean
|
||||
/** Start a call with the given preset's device defaults. */
|
||||
onStartCall?: (preset: CallPreset) => void
|
||||
onEndCall?: () => void
|
||||
/** Calls need both voice input (STT) and voice output (TTS) configured. */
|
||||
callAvailable?: boolean
|
||||
/** Fired when the user picks a different model in the dropdown (only when no run exists yet). */
|
||||
onSelectedModelChange?: (model: SelectedModel | null) => void
|
||||
/** Work directory for this chat (per-chat). Null when none is set. */
|
||||
|
|
@ -271,13 +285,10 @@ function ChatInputInner({
|
|||
onSubmitRecording,
|
||||
onCancelRecording,
|
||||
voiceAvailable,
|
||||
ttsAvailable,
|
||||
ttsEnabled,
|
||||
ttsMode,
|
||||
onToggleTts,
|
||||
onTtsModeChange,
|
||||
ttsAvatarEnabled,
|
||||
onToggleTtsAvatar,
|
||||
inCall,
|
||||
onStartCall,
|
||||
onEndCall,
|
||||
callAvailable,
|
||||
onSelectedModelChange,
|
||||
workDir = null,
|
||||
onWorkDirChange,
|
||||
|
|
@ -1276,80 +1287,71 @@ function ChatInputInner({
|
|||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
) : null}
|
||||
{onToggleTts && ttsAvailable && (
|
||||
{onStartCall && (
|
||||
<div className="flex shrink-0 items-center">
|
||||
<Tooltip delayDuration={CHAT_INPUT_TOOLTIP_DELAY_MS}>
|
||||
<TooltipTrigger asChild>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onToggleTts}
|
||||
onClick={() => {
|
||||
if (inCall) {
|
||||
onEndCall?.()
|
||||
} else if (callAvailable) {
|
||||
onStartCall('share')
|
||||
}
|
||||
}}
|
||||
className={cn(
|
||||
'relative flex h-7 w-7 shrink-0 items-center justify-center rounded-full transition-colors',
|
||||
ttsEnabled
|
||||
? 'text-foreground hover:bg-muted'
|
||||
: 'text-muted-foreground hover:bg-muted hover:text-foreground'
|
||||
'flex h-7 w-7 shrink-0 items-center justify-center rounded-full transition-colors',
|
||||
inCall
|
||||
? 'bg-red-600 text-white hover:bg-red-500'
|
||||
: callAvailable
|
||||
? 'text-muted-foreground hover:bg-muted hover:text-foreground'
|
||||
: 'cursor-default text-muted-foreground/40'
|
||||
)}
|
||||
aria-label={ttsEnabled ? 'Disable voice output' : 'Enable voice output'}
|
||||
aria-label={inCall ? 'End call' : 'Start a call'}
|
||||
>
|
||||
<Headphones className="h-4 w-4" />
|
||||
{!ttsEnabled && (
|
||||
<span className="absolute inset-0 flex items-center justify-center pointer-events-none">
|
||||
<span className="block h-[1.5px] w-5 -rotate-45 rounded-full bg-muted-foreground" />
|
||||
</span>
|
||||
)}
|
||||
{inCall ? <PhoneOff className="h-4 w-4" /> : <Phone className="h-4 w-4" />}
|
||||
</button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent side="top">
|
||||
{ttsEnabled ? 'Voice output on' : 'Voice output off'}
|
||||
{inCall
|
||||
? 'End call'
|
||||
: callAvailable
|
||||
? 'Start a call — it sees your screen while you talk it through'
|
||||
: 'Calls need voice input and output configured'}
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
{ttsEnabled && onTtsModeChange && (
|
||||
{!inCall && (
|
||||
<DropdownMenu>
|
||||
<DropdownMenuTrigger asChild>
|
||||
<button
|
||||
type="button"
|
||||
className="flex h-7 w-4 shrink-0 items-center justify-center text-muted-foreground transition-colors hover:text-foreground"
|
||||
aria-label="Call options"
|
||||
>
|
||||
<ChevronDown className="h-3 w-3" />
|
||||
</button>
|
||||
</DropdownMenuTrigger>
|
||||
<DropdownMenuContent align="end">
|
||||
<DropdownMenuRadioGroup value={ttsMode ?? 'summary'} onValueChange={(v) => onTtsModeChange(v as 'summary' | 'full')}>
|
||||
<DropdownMenuRadioItem value="summary">Speak summary</DropdownMenuRadioItem>
|
||||
<DropdownMenuRadioItem value="full">Speak full response</DropdownMenuRadioItem>
|
||||
</DropdownMenuRadioGroup>
|
||||
<DropdownMenuContent align="end" className="w-72">
|
||||
{CALL_PRESET_MENU.map(({ preset, label, description, Icon }) => (
|
||||
<DropdownMenuItem
|
||||
key={preset}
|
||||
disabled={!callAvailable}
|
||||
onSelect={() => onStartCall(preset)}
|
||||
className="items-start gap-3 py-2"
|
||||
>
|
||||
<Icon className="mt-0.5 h-4 w-4 shrink-0 text-muted-foreground" />
|
||||
<span className="min-w-0">
|
||||
<span className="block text-sm font-medium leading-tight">{label}</span>
|
||||
<span className="block pt-0.5 text-xs leading-tight text-muted-foreground">{description}</span>
|
||||
</span>
|
||||
</DropdownMenuItem>
|
||||
))}
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
{onToggleTtsAvatar && ttsAvailable && (
|
||||
<Tooltip delayDuration={CHAT_INPUT_TOOLTIP_DELAY_MS}>
|
||||
<TooltipTrigger asChild>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onToggleTtsAvatar}
|
||||
className={cn(
|
||||
'relative flex h-7 w-7 shrink-0 items-center justify-center rounded-full transition-colors',
|
||||
ttsAvatarEnabled
|
||||
? 'text-foreground hover:bg-muted'
|
||||
: 'text-muted-foreground hover:bg-muted hover:text-foreground'
|
||||
)}
|
||||
aria-label={ttsAvatarEnabled ? 'Disable talking head' : 'Enable talking head'}
|
||||
>
|
||||
<MascotFaceIcon />
|
||||
{!ttsAvatarEnabled && (
|
||||
<span className="absolute inset-0 flex items-center justify-center pointer-events-none">
|
||||
<span className="block h-[1.5px] w-5 -rotate-45 rounded-full bg-muted-foreground" />
|
||||
</span>
|
||||
)}
|
||||
</button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent side="top">
|
||||
{ttsAvatarEnabled ? 'Talking head on' : 'Talking head off'}
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
)}
|
||||
{voiceAvailable && onStartRecording && (
|
||||
<button
|
||||
type="button"
|
||||
|
|
@ -1519,13 +1521,10 @@ export interface ChatInputWithMentionsProps {
|
|||
onSubmitRecording?: () => void | Promise<void>
|
||||
onCancelRecording?: () => void
|
||||
voiceAvailable?: boolean
|
||||
ttsAvailable?: boolean
|
||||
ttsEnabled?: boolean
|
||||
ttsMode?: 'summary' | 'full'
|
||||
onToggleTts?: () => void
|
||||
onTtsModeChange?: (mode: 'summary' | 'full') => void
|
||||
ttsAvatarEnabled?: boolean
|
||||
onToggleTtsAvatar?: () => void
|
||||
inCall?: boolean
|
||||
onStartCall?: (preset: CallPreset) => void
|
||||
onEndCall?: () => void
|
||||
callAvailable?: boolean
|
||||
onSelectedModelChange?: (model: SelectedModel | null) => void
|
||||
workDir?: string | null
|
||||
onWorkDirChange?: (value: string | null) => void
|
||||
|
|
@ -1555,13 +1554,10 @@ export function ChatInputWithMentions({
|
|||
onSubmitRecording,
|
||||
onCancelRecording,
|
||||
voiceAvailable,
|
||||
ttsAvailable,
|
||||
ttsEnabled,
|
||||
ttsMode,
|
||||
onToggleTts,
|
||||
onTtsModeChange,
|
||||
ttsAvatarEnabled,
|
||||
onToggleTtsAvatar,
|
||||
inCall,
|
||||
onStartCall,
|
||||
onEndCall,
|
||||
callAvailable,
|
||||
onSelectedModelChange,
|
||||
workDir,
|
||||
onWorkDirChange,
|
||||
|
|
@ -1588,13 +1584,10 @@ export function ChatInputWithMentions({
|
|||
onSubmitRecording={onSubmitRecording}
|
||||
onCancelRecording={onCancelRecording}
|
||||
voiceAvailable={voiceAvailable}
|
||||
ttsAvailable={ttsAvailable}
|
||||
ttsEnabled={ttsEnabled}
|
||||
ttsMode={ttsMode}
|
||||
onToggleTts={onToggleTts}
|
||||
onTtsModeChange={onTtsModeChange}
|
||||
ttsAvatarEnabled={ttsAvatarEnabled}
|
||||
onToggleTtsAvatar={onToggleTtsAvatar}
|
||||
inCall={inCall}
|
||||
onStartCall={onStartCall}
|
||||
onEndCall={onEndCall}
|
||||
callAvailable={callAvailable}
|
||||
onSelectedModelChange={onSelectedModelChange}
|
||||
workDir={workDir}
|
||||
onWorkDirChange={onWorkDirChange}
|
||||
|
|
|
|||
|
|
@ -91,11 +91,28 @@ interface ChatMessageAttachmentsProps {
|
|||
export function ChatMessageAttachments({ attachments, className }: ChatMessageAttachmentsProps) {
|
||||
if (attachments.length === 0) return null
|
||||
|
||||
const imageAttachments = attachments.filter((attachment) => isImageMime(attachment.mimeType))
|
||||
const fileAttachments = attachments.filter((attachment) => !isImageMime(attachment.mimeType))
|
||||
const videoFrames = attachments.filter((attachment) => attachment.isVideoFrame)
|
||||
const imageAttachments = attachments.filter(
|
||||
(attachment) => !attachment.isVideoFrame && isImageMime(attachment.mimeType)
|
||||
)
|
||||
const fileAttachments = attachments.filter(
|
||||
(attachment) => !attachment.isVideoFrame && !isImageMime(attachment.mimeType)
|
||||
)
|
||||
|
||||
return (
|
||||
<div className={cn('flex flex-col items-end gap-2', className)}>
|
||||
{videoFrames.length > 0 && (
|
||||
<div className="flex max-w-[340px] flex-wrap justify-end gap-1">
|
||||
{videoFrames.map((frame, index) => (
|
||||
<img
|
||||
key={`frame-${index}`}
|
||||
src={frame.thumbnailUrl}
|
||||
alt={`Camera frame ${index + 1}`}
|
||||
className="h-12 w-auto rounded-md border border-border/60 bg-muted object-cover"
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
{imageAttachments.length > 0 && (
|
||||
<div className="flex flex-wrap justify-end gap-2">
|
||||
{imageAttachments.map((attachment, index) => (
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ import { MarkdownPreOverride } from '@/components/ai-elements/markdown-code-over
|
|||
import { defaultRemarkPlugins } from 'streamdown'
|
||||
import remarkBreaks from 'remark-breaks'
|
||||
import { type ChatTab } from '@/components/tab-bar'
|
||||
import { ChatInputWithMentions, type PermissionMode, type StagedAttachment, type SelectedModel } from '@/components/chat-input-with-mentions'
|
||||
import { ChatInputWithMentions, type CallPreset, type PermissionMode, type StagedAttachment, type SelectedModel } from '@/components/chat-input-with-mentions'
|
||||
import { ChatMessageAttachments } from '@/components/chat-message-attachments'
|
||||
import { useSidebar } from '@/components/ui/sidebar'
|
||||
import { wikiLabel } from '@/lib/wiki-links'
|
||||
|
|
@ -187,13 +187,10 @@ interface ChatSidebarProps {
|
|||
onSubmitRecording?: () => void | Promise<void>
|
||||
onCancelRecording?: () => void
|
||||
voiceAvailable?: boolean
|
||||
ttsAvailable?: boolean
|
||||
ttsEnabled?: boolean
|
||||
ttsMode?: 'summary' | 'full'
|
||||
onToggleTts?: () => void
|
||||
onTtsModeChange?: (mode: 'summary' | 'full') => void
|
||||
ttsAvatarEnabled?: boolean
|
||||
onToggleTtsAvatar?: () => void
|
||||
inCall?: boolean
|
||||
onStartCall?: (preset: CallPreset) => void
|
||||
onEndCall?: () => void
|
||||
callAvailable?: boolean
|
||||
onComposioConnected?: (toolkitSlug: string) => void
|
||||
}
|
||||
|
||||
|
|
@ -253,13 +250,10 @@ export function ChatSidebar({
|
|||
onSubmitRecording,
|
||||
onCancelRecording,
|
||||
voiceAvailable,
|
||||
ttsAvailable,
|
||||
ttsEnabled,
|
||||
ttsMode,
|
||||
onToggleTts,
|
||||
onTtsModeChange,
|
||||
ttsAvatarEnabled,
|
||||
onToggleTtsAvatar,
|
||||
inCall,
|
||||
onStartCall,
|
||||
onEndCall,
|
||||
callAvailable,
|
||||
onComposioConnected,
|
||||
}: ChatSidebarProps) {
|
||||
const { state: sidebarState } = useSidebar()
|
||||
|
|
@ -829,13 +823,10 @@ export function ChatSidebar({
|
|||
onSubmitRecording={isActive ? onSubmitRecording : undefined}
|
||||
onCancelRecording={isActive ? onCancelRecording : undefined}
|
||||
voiceAvailable={isActive && voiceAvailable}
|
||||
ttsAvailable={isActive && ttsAvailable}
|
||||
ttsEnabled={ttsEnabled}
|
||||
ttsMode={ttsMode}
|
||||
onToggleTts={isActive ? onToggleTts : undefined}
|
||||
onTtsModeChange={isActive ? onTtsModeChange : undefined}
|
||||
ttsAvatarEnabled={ttsAvatarEnabled}
|
||||
onToggleTtsAvatar={isActive ? onToggleTtsAvatar : undefined}
|
||||
inCall={inCall}
|
||||
onStartCall={isActive ? onStartCall : undefined}
|
||||
onEndCall={isActive ? onEndCall : undefined}
|
||||
callAvailable={callAvailable}
|
||||
/>
|
||||
</div>
|
||||
)
|
||||
|
|
|
|||
238
apps/x/apps/renderer/src/components/video-call-view.tsx
Normal file
238
apps/x/apps/renderer/src/components/video-call-view.tsx
Normal file
|
|
@ -0,0 +1,238 @@
|
|||
import { useEffect, useRef, useState } from 'react'
|
||||
import { Minimize2, MonitorUp, PhoneOff, Presentation, Square, User, Video, VideoOff } from 'lucide-react'
|
||||
|
||||
import { MascotFaceIcon, TalkingHead } from '@/components/talking-head'
|
||||
import type { TTSState } from '@/hooks/useVoiceTTS'
|
||||
import { cn } from '@/lib/utils'
|
||||
|
||||
export type VideoCallStatus = 'listening' | 'thinking' | 'speaking'
|
||||
|
||||
interface VideoCallViewProps {
|
||||
/** Live camera stream from useVideoMode — attached to the user's tile. */
|
||||
streamRef: React.MutableRefObject<MediaStream | null>
|
||||
cameraOn: boolean
|
||||
onToggleCamera: () => void
|
||||
/** Starting a share collapses this view into the floating popout (the
|
||||
* surface is derived from devices — see App.tsx). */
|
||||
onToggleScreenShare: () => void
|
||||
/** Practice preset: the assistant is coaching this session. */
|
||||
practiceMode?: boolean
|
||||
/** Shrink to the floating pill without touching any devices. */
|
||||
onMinimize: () => void
|
||||
/** Stop the assistant: silence speech and abort the run if still going. */
|
||||
onInterrupt: () => void
|
||||
ttsState: TTSState
|
||||
/** Live TTS output level — drives the mascot's mouth animation. */
|
||||
getTtsLevel: () => number
|
||||
status: VideoCallStatus
|
||||
/** Live transcript of the user's in-progress utterance. */
|
||||
interimText?: string
|
||||
/** The assistant line currently being spoken aloud. */
|
||||
assistantCaption?: string
|
||||
onLeave: () => void
|
||||
}
|
||||
|
||||
const STATUS_DISPLAY: Record<VideoCallStatus, { label: string; dotClass: string }> = {
|
||||
listening: { label: 'Listening', dotClass: 'bg-green-500 animate-pulse' },
|
||||
thinking: { label: 'Thinking…', dotClass: 'bg-amber-400' },
|
||||
speaking: { label: 'Speaking', dotClass: 'bg-sky-400 animate-pulse' },
|
||||
}
|
||||
|
||||
/**
|
||||
* Full-screen call surface: a Meet-style two-tile layout with the user's
|
||||
* webcam on one side and the mascot as the other participant. Shown only
|
||||
* while the camera is on with no screen share (the derived-surface rule in
|
||||
* App.tsx) — sharing or muting the camera moves the call into the floating
|
||||
* popout. The mascot animates with the assistant's speech; dismissing it
|
||||
* swaps in a Meet-style letter avatar ("R"). Live captions run along the
|
||||
* bottom.
|
||||
*/
|
||||
export function VideoCallView({
|
||||
streamRef,
|
||||
cameraOn,
|
||||
onToggleCamera,
|
||||
onToggleScreenShare,
|
||||
practiceMode,
|
||||
onMinimize,
|
||||
onInterrupt,
|
||||
ttsState,
|
||||
getTtsLevel,
|
||||
status,
|
||||
interimText,
|
||||
assistantCaption,
|
||||
onLeave,
|
||||
}: VideoCallViewProps) {
|
||||
const videoRef = useRef<HTMLVideoElement | null>(null)
|
||||
const [mascotVisible, setMascotVisible] = useState(true)
|
||||
|
||||
useEffect(() => {
|
||||
if (!cameraOn) return
|
||||
const videoEl = videoRef.current
|
||||
if (!videoEl) return
|
||||
videoEl.srcObject = streamRef.current
|
||||
videoEl.play().catch(() => {})
|
||||
return () => {
|
||||
videoEl.srcObject = null
|
||||
}
|
||||
}, [streamRef, cameraOn])
|
||||
|
||||
const userSpeaking = status === 'listening' && Boolean(interimText)
|
||||
const assistantSpeaking = ttsState === 'speaking'
|
||||
|
||||
const caption = assistantSpeaking && assistantCaption
|
||||
? { who: 'Rowboat', text: assistantCaption }
|
||||
: interimText
|
||||
? { who: 'You', text: interimText }
|
||||
: null
|
||||
|
||||
return (
|
||||
<div className="fixed inset-0 z-[100] flex flex-col bg-neutral-950">
|
||||
{practiceMode && (
|
||||
<span className="absolute left-4 top-4 z-10 flex items-center gap-1.5 rounded-full bg-violet-600/90 px-3 py-1 text-xs font-medium text-white">
|
||||
<Presentation className="h-3.5 w-3.5" />
|
||||
Practice session
|
||||
</span>
|
||||
)}
|
||||
<button
|
||||
type="button"
|
||||
onClick={onMinimize}
|
||||
className="absolute right-4 top-4 z-10 flex h-8 w-8 items-center justify-center rounded-full bg-neutral-800 text-white/80 transition-colors hover:bg-neutral-700 hover:text-white"
|
||||
aria-label="Minimize call (shares your screen)"
|
||||
title="Minimize — shares your screen so it can help you work"
|
||||
>
|
||||
<Minimize2 className="h-4 w-4" />
|
||||
</button>
|
||||
|
||||
{/* Participant tiles */}
|
||||
<div className="grid min-h-0 flex-1 grid-cols-1 gap-3 p-4 pb-2 md:grid-cols-2">
|
||||
{/* User */}
|
||||
<div
|
||||
className={cn(
|
||||
'relative flex items-center justify-center overflow-hidden rounded-2xl bg-neutral-900 transition-shadow',
|
||||
userSpeaking && 'ring-2 ring-green-500/80'
|
||||
)}
|
||||
>
|
||||
{cameraOn ? (
|
||||
<video
|
||||
ref={videoRef}
|
||||
muted
|
||||
playsInline
|
||||
className="h-full w-full object-cover"
|
||||
style={{ transform: 'scaleX(-1)' }}
|
||||
/>
|
||||
) : (
|
||||
<span className="flex h-40 w-40 items-center justify-center rounded-full bg-neutral-700 text-neutral-400" aria-label="Camera off">
|
||||
<User className="h-20 w-20" />
|
||||
</span>
|
||||
)}
|
||||
<span className="absolute bottom-3 left-3 rounded-md bg-black/50 px-2 py-0.5 text-sm text-white">
|
||||
You
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{/* Assistant */}
|
||||
<div
|
||||
className={cn(
|
||||
'group relative flex items-center justify-center overflow-hidden rounded-2xl bg-neutral-900 transition-shadow',
|
||||
assistantSpeaking && 'ring-2 ring-sky-400/80'
|
||||
)}
|
||||
>
|
||||
{mascotVisible ? (
|
||||
<TalkingHead ttsState={ttsState} getLevel={getTtsLevel} size={220} />
|
||||
) : (
|
||||
<span
|
||||
className="flex h-40 w-40 items-center justify-center rounded-full bg-sky-600 text-7xl font-medium text-white"
|
||||
aria-label="Rowboat"
|
||||
>
|
||||
R
|
||||
</span>
|
||||
)}
|
||||
<span className="absolute bottom-3 left-3 rounded-md bg-black/50 px-2 py-0.5 text-sm text-white">
|
||||
Rowboat
|
||||
</span>
|
||||
{status !== 'listening' && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={onInterrupt}
|
||||
className="absolute bottom-3 right-3 flex items-center gap-1.5 rounded-md bg-red-600/90 px-2.5 py-1 text-sm font-medium text-white transition-colors hover:bg-red-500"
|
||||
aria-label="Stop the assistant"
|
||||
title={status === 'speaking' ? 'Stop speaking' : 'Stop responding'}
|
||||
>
|
||||
<Square className="h-3 w-3 fill-current" />
|
||||
Stop
|
||||
</button>
|
||||
)}
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setMascotVisible((v) => !v)}
|
||||
className="absolute right-3 top-3 rounded-md bg-black/50 px-2 py-1 text-xs text-white/80 opacity-0 transition-opacity hover:text-white group-hover:opacity-100"
|
||||
>
|
||||
{mascotVisible ? 'Hide mascot' : 'Show mascot'}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Captions */}
|
||||
<div className="flex h-14 items-center justify-center px-6">
|
||||
{caption && (
|
||||
<div className="max-w-3xl truncate rounded-lg bg-black/60 px-4 py-2 text-sm text-white/90">
|
||||
<span className="mr-2 font-semibold text-white">{caption.who}:</span>
|
||||
{caption.text}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Control bar */}
|
||||
<div className="flex items-center justify-center gap-4 pb-5">
|
||||
<span className="flex items-center gap-2 rounded-full bg-neutral-800 px-3 py-1.5 text-xs font-medium text-white/90">
|
||||
<span className={cn('block h-2 w-2 rounded-full', STATUS_DISPLAY[status].dotClass)} />
|
||||
{STATUS_DISPLAY[status].label}
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onToggleCamera}
|
||||
className={cn(
|
||||
'flex h-10 w-10 items-center justify-center rounded-full transition-colors',
|
||||
cameraOn
|
||||
? 'bg-neutral-800 text-white/90 hover:bg-neutral-700'
|
||||
: 'bg-red-600 text-white hover:bg-red-500'
|
||||
)}
|
||||
aria-label={cameraOn ? 'Turn off camera' : 'Turn on camera'}
|
||||
title={cameraOn ? 'Turn off camera' : 'Turn on camera'}
|
||||
>
|
||||
{cameraOn ? <Video className="h-5 w-5" /> : <VideoOff className="h-5 w-5" />}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onToggleScreenShare}
|
||||
className="flex h-10 w-10 items-center justify-center rounded-full bg-neutral-800 text-white/90 transition-colors hover:bg-neutral-700"
|
||||
aria-label="Present your screen"
|
||||
title="Present your screen"
|
||||
>
|
||||
<MonitorUp className="h-5 w-5" />
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setMascotVisible((v) => !v)}
|
||||
className="relative flex h-10 w-10 items-center justify-center rounded-full bg-neutral-800 text-white/90 transition-colors hover:bg-neutral-700"
|
||||
aria-label={mascotVisible ? 'Hide mascot' : 'Show mascot'}
|
||||
>
|
||||
<MascotFaceIcon />
|
||||
{!mascotVisible && (
|
||||
<span className="absolute inset-0 flex items-center justify-center pointer-events-none">
|
||||
<span className="block h-[1.5px] w-6 -rotate-45 rounded-full bg-white/80" />
|
||||
</span>
|
||||
)}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onLeave}
|
||||
className="flex h-10 w-14 items-center justify-center rounded-full bg-red-600 text-white transition-colors hover:bg-red-500"
|
||||
aria-label="End call"
|
||||
>
|
||||
<PhoneOff className="h-5 w-5" />
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
207
apps/x/apps/renderer/src/components/video-popout.tsx
Normal file
207
apps/x/apps/renderer/src/components/video-popout.tsx
Normal file
|
|
@ -0,0 +1,207 @@
|
|||
import { useCallback, useEffect, useRef, useState } from 'react'
|
||||
import { Maximize2, MonitorUp, PhoneOff, Square, User, Video, VideoOff } from 'lucide-react'
|
||||
|
||||
import { TalkingHead } from '@/components/talking-head'
|
||||
|
||||
type PopoutState = {
|
||||
ttsState: 'idle' | 'synthesizing' | 'speaking'
|
||||
status: 'listening' | 'thinking' | 'speaking' | null
|
||||
cameraOn: boolean
|
||||
screenSharing: boolean
|
||||
interimText: string | null
|
||||
}
|
||||
|
||||
const STATUS_DISPLAY: Record<NonNullable<PopoutState['status']>, { label: string; dotClass: string }> = {
|
||||
listening: { label: 'Listening', dotClass: 'bg-green-500 animate-pulse' },
|
||||
thinking: { label: 'Thinking…', dotClass: 'bg-amber-400' },
|
||||
speaking: { label: 'Speaking', dotClass: 'bg-sky-400 animate-pulse' },
|
||||
}
|
||||
|
||||
const dragRegion = { WebkitAppRegion: 'drag' } as React.CSSProperties
|
||||
const noDragRegion = { WebkitAppRegion: 'no-drag' } as React.CSSProperties
|
||||
|
||||
/**
|
||||
* Content of the always-on-top popout window shown for the whole duration of
|
||||
* a screen share (Meet-style floating mini-call) — it floats over every app,
|
||||
* including Rowboat itself, and is the call's control surface while sharing:
|
||||
* camera toggle, share toggle, end-call. Rendered in its own BrowserWindow
|
||||
* (see `video:setPopout` in the main process); call state streams in over
|
||||
* the `video:popout-state` push channel and control actions round-trip back
|
||||
* through `video:popoutAction`. Captures its own webcam feed — MediaStreams
|
||||
* can't cross windows.
|
||||
*/
|
||||
export function VideoPopout() {
|
||||
// Camera defaults OFF: guessing "on" would flash the user's video for a
|
||||
// beat before the real state arrives — which reads as a bug. The true
|
||||
// state is fetched immediately below.
|
||||
const [state, setState] = useState<PopoutState>({ ttsState: 'idle', status: null, cameraOn: false, screenSharing: false, interimText: null })
|
||||
const videoRef = useRef<HTMLVideoElement | null>(null)
|
||||
|
||||
useEffect(() => {
|
||||
const cleanup = window.ipc.on('video:popout-state', (next) => setState(next))
|
||||
// The main process replays the cached state on did-finish-load, but that
|
||||
// can race this listener's registration — fetch it explicitly too.
|
||||
window.ipc
|
||||
.invoke('video:getPopoutState', null)
|
||||
.then(({ state: cached }) => {
|
||||
if (cached) setState(cached)
|
||||
})
|
||||
.catch(() => {})
|
||||
return cleanup
|
||||
}, [])
|
||||
|
||||
// Own camera feed, following the main window's camera-on/off state.
|
||||
useEffect(() => {
|
||||
if (!state.cameraOn) return
|
||||
let stream: MediaStream | null = null
|
||||
let cancelled = false
|
||||
navigator.mediaDevices
|
||||
.getUserMedia({ video: { width: { ideal: 640 }, facingMode: 'user' }, audio: false })
|
||||
.then((s) => {
|
||||
if (cancelled) {
|
||||
s.getTracks().forEach((t) => t.stop())
|
||||
return
|
||||
}
|
||||
stream = s
|
||||
if (videoRef.current) {
|
||||
videoRef.current.srcObject = s
|
||||
videoRef.current.play().catch(() => {})
|
||||
}
|
||||
})
|
||||
.catch((err) => console.error('[popout] camera failed:', err))
|
||||
return () => {
|
||||
cancelled = true
|
||||
stream?.getTracks().forEach((t) => t.stop())
|
||||
if (videoRef.current) videoRef.current.srcObject = null
|
||||
}
|
||||
}, [state.cameraOn])
|
||||
|
||||
// The popout has no TTS audio pipeline — synthesize a plausible mouth level
|
||||
// so the mascot still animates while the assistant speaks in the main window.
|
||||
const getLevel = useCallback(() => 0.45 + 0.35 * Math.sin(performance.now() / 90), [])
|
||||
|
||||
const sendAction = useCallback((action: 'toggle-camera' | 'toggle-share' | 'stop-speaking' | 'end-call' | 'expand') => {
|
||||
void window.ipc.invoke('video:popoutAction', { action }).catch(() => {})
|
||||
}, [])
|
||||
|
||||
const statusDisplay = state.status ? STATUS_DISPLAY[state.status] : null
|
||||
|
||||
return (
|
||||
<div
|
||||
className="relative flex h-screen w-screen select-none flex-col gap-1.5 bg-neutral-900 p-1.5"
|
||||
style={dragRegion}
|
||||
>
|
||||
<div className="flex min-h-0 flex-1 gap-1.5">
|
||||
<div className="relative flex-1 overflow-hidden rounded-lg bg-neutral-800">
|
||||
{state.cameraOn ? (
|
||||
<video
|
||||
ref={videoRef}
|
||||
muted
|
||||
playsInline
|
||||
className="h-full w-full object-cover"
|
||||
style={{ transform: 'scaleX(-1)' }}
|
||||
/>
|
||||
) : (
|
||||
<div className="flex h-full w-full items-center justify-center">
|
||||
<span className="flex h-12 w-12 items-center justify-center rounded-full bg-neutral-700 text-neutral-400">
|
||||
<User className="h-6 w-6" />
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
<span className="absolute bottom-1 left-1.5 rounded bg-black/50 px-1 py-px text-[10px] text-white">
|
||||
You
|
||||
</span>
|
||||
{/* Persistent consent badge — the user must always be able to see
|
||||
at a glance that their screen is going out. */}
|
||||
{state.screenSharing && (
|
||||
<span className="absolute left-1.5 top-1.5 flex items-center gap-1 rounded-full bg-sky-600/90 px-1.5 py-0.5 text-[10px] font-medium text-white">
|
||||
<span className="block h-1.5 w-1.5 animate-pulse rounded-full bg-white" />
|
||||
Sharing screen
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<div className="relative flex flex-1 items-center justify-center overflow-hidden rounded-lg bg-neutral-800">
|
||||
<TalkingHead ttsState={state.ttsState} getLevel={getLevel} size={84} />
|
||||
<span className="absolute bottom-1 left-1.5 rounded bg-black/50 px-1 py-px text-[10px] text-white">
|
||||
Rowboat
|
||||
</span>
|
||||
{statusDisplay && (
|
||||
<span className="absolute right-1.5 top-1.5 flex items-center gap-1 rounded-full bg-black/50 px-1.5 py-0.5 text-[10px] font-medium text-white">
|
||||
<span className={`block h-1.5 w-1.5 rounded-full ${statusDisplay.dotClass}`} />
|
||||
{statusDisplay.label}
|
||||
</span>
|
||||
)}
|
||||
{(state.status === 'speaking' || state.status === 'thinking') && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => sendAction('stop-speaking')}
|
||||
className="absolute bottom-1 right-1.5 flex items-center gap-1 rounded bg-red-600/90 px-1.5 py-0.5 text-[10px] font-medium text-white hover:bg-red-500"
|
||||
style={noDragRegion}
|
||||
aria-label="Stop the assistant"
|
||||
title={state.status === 'speaking' ? 'Stop speaking' : 'Stop responding'}
|
||||
>
|
||||
<Square className="h-2.5 w-2.5 fill-current" />
|
||||
Stop
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
{/* Live caption of the in-progress utterance, floating over the tiles */}
|
||||
{state.interimText && (
|
||||
<div className="pointer-events-none absolute inset-x-1.5 bottom-9 flex justify-center">
|
||||
<span className="max-w-full truncate rounded bg-black/70 px-1.5 py-0.5 text-[10px] text-white/90">
|
||||
{state.interimText}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Control bar — actions execute in the main app window */}
|
||||
<div className="flex h-7 shrink-0 items-center justify-center gap-2" style={noDragRegion}>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => sendAction('toggle-camera')}
|
||||
className={`flex h-6 w-6 items-center justify-center rounded-full transition-colors ${
|
||||
state.cameraOn
|
||||
? 'bg-neutral-700 text-white/90 hover:bg-neutral-600'
|
||||
: 'bg-red-600 text-white hover:bg-red-500'
|
||||
}`}
|
||||
aria-label={state.cameraOn ? 'Turn off camera' : 'Turn on camera'}
|
||||
title={state.cameraOn ? 'Turn off camera' : 'Turn on camera'}
|
||||
>
|
||||
{state.cameraOn ? <Video className="h-3.5 w-3.5" /> : <VideoOff className="h-3.5 w-3.5" />}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => sendAction('toggle-share')}
|
||||
className={`flex h-6 w-6 items-center justify-center rounded-full transition-colors ${
|
||||
state.screenSharing
|
||||
? 'bg-sky-600 text-white hover:bg-sky-500'
|
||||
: 'bg-neutral-700 text-white/90 hover:bg-neutral-600'
|
||||
}`}
|
||||
aria-label={state.screenSharing ? 'Stop sharing screen' : 'Share your screen'}
|
||||
title={state.screenSharing ? 'Stop sharing screen' : 'Share your screen'}
|
||||
>
|
||||
<MonitorUp className="h-3.5 w-3.5" />
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => sendAction('end-call')}
|
||||
className="flex h-6 w-8 items-center justify-center rounded-full bg-red-600 text-white transition-colors hover:bg-red-500"
|
||||
aria-label="End call"
|
||||
title="End call"
|
||||
>
|
||||
<PhoneOff className="h-3.5 w-3.5" />
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => sendAction('expand')}
|
||||
className="flex h-6 w-6 items-center justify-center rounded-full bg-neutral-700 text-white/90 transition-colors hover:bg-neutral-600"
|
||||
aria-label="Expand to full screen (stops screen sharing)"
|
||||
title="Expand to full screen (stops sharing)"
|
||||
>
|
||||
<Maximize2 className="h-3.5 w-3.5" />
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
312
apps/x/apps/renderer/src/hooks/useVideoMode.ts
Normal file
312
apps/x/apps/renderer/src/hooks/useVideoMode.ts
Normal file
|
|
@ -0,0 +1,312 @@
|
|||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
|
||||
export type VideoModeState = 'idle' | 'starting' | 'live';
|
||||
export type ScreenShareState = 'idle' | 'starting' | 'live';
|
||||
|
||||
export interface CapturedVideoFrame {
|
||||
/** base64-encoded JPEG bytes (no data: prefix) — shape of the UserImagePart wire format */
|
||||
data: string;
|
||||
mediaType: string;
|
||||
capturedAt: string; // ISO timestamp
|
||||
/** data: URL of the same frame, for direct display in the transcript */
|
||||
dataUrl: string;
|
||||
source: 'camera' | 'screen';
|
||||
}
|
||||
|
||||
// Frames are grabbed once per second — dense enough to catch expression and
|
||||
// posture changes while the user talks. Per message we attach at most
|
||||
// MAX_*_FRAMES_PER_MESSAGE frames, evenly sampled across the window since the
|
||||
// last send, so long monologues don't balloon the request.
|
||||
const CAPTURE_INTERVAL_MS = 1000;
|
||||
const MAX_CAMERA_FRAMES_PER_MESSAGE = 12;
|
||||
// Screen frames are ~4x the resolution (and tokens) of camera frames, and the
|
||||
// latest view matters far more than the trajectory — keep the cap small.
|
||||
const MAX_SCREEN_FRAMES_PER_MESSAGE = 4;
|
||||
// Rolling buffer bound (~2 minutes). The buffer only needs to cover the gap
|
||||
// between two sends; anything older is stale context anyway.
|
||||
const MAX_BUFFERED_FRAMES = 120;
|
||||
// Downscale targets. 512px wide JPEG keeps a webcam frame around 20-40KB —
|
||||
// cheap enough to inline a dozen per message as multimodal image parts.
|
||||
// Screen captures keep 1280px so on-screen text stays legible to the model.
|
||||
const CAMERA_FRAME_WIDTH = 512;
|
||||
const SCREEN_FRAME_WIDTH = 1280;
|
||||
const CAMERA_JPEG_QUALITY = 0.65;
|
||||
const SCREEN_JPEG_QUALITY = 0.7;
|
||||
|
||||
interface BufferedFrame {
|
||||
dataUrl: string;
|
||||
capturedAt: string;
|
||||
ts: number;
|
||||
}
|
||||
|
||||
// One capture pipeline: stream → offscreen <video> → canvas JPEG → ring buffer.
|
||||
interface CapturePipe {
|
||||
stream: MediaStream | null;
|
||||
videoEl: HTMLVideoElement | null;
|
||||
canvas: HTMLCanvasElement | null;
|
||||
interval: ReturnType<typeof setInterval> | null;
|
||||
frames: BufferedFrame[];
|
||||
lastCollectTs: number;
|
||||
}
|
||||
|
||||
const emptyPipe = (): CapturePipe => ({
|
||||
stream: null,
|
||||
videoEl: null,
|
||||
canvas: null,
|
||||
interval: null,
|
||||
frames: [],
|
||||
lastCollectTs: 0,
|
||||
});
|
||||
|
||||
function capturePipeFrame(pipe: CapturePipe, width: number, quality: number) {
|
||||
const videoEl = pipe.videoEl;
|
||||
if (!videoEl || videoEl.readyState < 2 || videoEl.videoWidth === 0) return;
|
||||
if (!pipe.canvas) {
|
||||
pipe.canvas = document.createElement('canvas');
|
||||
}
|
||||
const canvas = pipe.canvas;
|
||||
const scale = Math.min(1, width / videoEl.videoWidth);
|
||||
canvas.width = Math.round(videoEl.videoWidth * scale);
|
||||
canvas.height = Math.round(videoEl.videoHeight * scale);
|
||||
const ctx = canvas.getContext('2d');
|
||||
if (!ctx) return;
|
||||
ctx.drawImage(videoEl, 0, 0, canvas.width, canvas.height);
|
||||
const dataUrl = canvas.toDataURL('image/jpeg', quality);
|
||||
// A near-empty data URL means the frame was blank (source still warming up)
|
||||
if (dataUrl.length < 100) return;
|
||||
pipe.frames.push({ dataUrl, capturedAt: new Date().toISOString(), ts: Date.now() });
|
||||
if (pipe.frames.length > MAX_BUFFERED_FRAMES) {
|
||||
pipe.frames.splice(0, pipe.frames.length - MAX_BUFFERED_FRAMES);
|
||||
}
|
||||
}
|
||||
|
||||
function attachPipeSource(pipe: CapturePipe, stream: MediaStream, grab: () => void) {
|
||||
pipe.stream = stream;
|
||||
// Offscreen <video> that feeds the capture canvas; any visible preview
|
||||
// attaches to the same MediaStream separately.
|
||||
const videoEl = document.createElement('video');
|
||||
videoEl.muted = true;
|
||||
videoEl.playsInline = true;
|
||||
videoEl.srcObject = stream;
|
||||
pipe.videoEl = videoEl;
|
||||
videoEl.play().catch(() => {});
|
||||
// First frame as soon as the source delivers data, then steady-state cadence.
|
||||
videoEl.addEventListener('loadeddata', () => grab(), { once: true });
|
||||
pipe.interval = setInterval(grab, CAPTURE_INTERVAL_MS);
|
||||
}
|
||||
|
||||
function teardownPipe(pipe: CapturePipe) {
|
||||
if (pipe.interval) {
|
||||
clearInterval(pipe.interval);
|
||||
pipe.interval = null;
|
||||
}
|
||||
if (pipe.videoEl) {
|
||||
pipe.videoEl.srcObject = null;
|
||||
pipe.videoEl = null;
|
||||
}
|
||||
if (pipe.stream) {
|
||||
pipe.stream.getTracks().forEach((t) => t.stop());
|
||||
pipe.stream = null;
|
||||
}
|
||||
pipe.frames = [];
|
||||
pipe.lastCollectTs = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Drain frames captured since the previous collection, evenly sampled down to
|
||||
* `max` (always keeping the newest). Falls back to the single most recent
|
||||
* frame when nothing new accumulated (rapid-fire messages), so every message
|
||||
* carries at least one frame once the source has warmed up.
|
||||
*/
|
||||
function drainPipe(pipe: CapturePipe, max: number, source: CapturedVideoFrame['source']): CapturedVideoFrame[] {
|
||||
const all = pipe.frames;
|
||||
if (all.length === 0) return [];
|
||||
|
||||
let window_ = all.filter((f) => f.ts > pipe.lastCollectTs);
|
||||
if (window_.length === 0) {
|
||||
window_ = [all[all.length - 1]];
|
||||
}
|
||||
pipe.lastCollectTs = window_[window_.length - 1].ts;
|
||||
|
||||
let sampled: BufferedFrame[];
|
||||
if (window_.length <= max) {
|
||||
sampled = window_;
|
||||
} else {
|
||||
sampled = [];
|
||||
const step = (window_.length - 1) / (max - 1);
|
||||
for (let i = 0; i < max; i++) {
|
||||
sampled.push(window_[Math.round(i * step)]);
|
||||
}
|
||||
}
|
||||
|
||||
return sampled.map((f) => ({
|
||||
data: f.dataUrl.slice(f.dataUrl.indexOf(',') + 1),
|
||||
mediaType: 'image/jpeg',
|
||||
capturedAt: f.capturedAt,
|
||||
dataUrl: f.dataUrl,
|
||||
source,
|
||||
}));
|
||||
}
|
||||
|
||||
export function useVideoMode() {
|
||||
const [state, setState] = useState<VideoModeState>('idle');
|
||||
const [screenState, setScreenState] = useState<ScreenShareState>('idle');
|
||||
// Camera can be turned off mid-session (Meet-style) while the mode — and
|
||||
// any screen share — keeps running. Resets to on for the next session.
|
||||
const [cameraOn, setCameraOn] = useState(true);
|
||||
const cameraPipeRef = useRef<CapturePipe>(emptyPipe());
|
||||
const screenPipeRef = useRef<CapturePipe>(emptyPipe());
|
||||
// Stable stream refs for preview components (<video srcObject>).
|
||||
const streamRef = useRef<MediaStream | null>(null);
|
||||
const screenStreamRef = useRef<MediaStream | null>(null);
|
||||
const stateRef = useRef<VideoModeState>('idle');
|
||||
stateRef.current = state;
|
||||
const screenStateRef = useRef<ScreenShareState>('idle');
|
||||
screenStateRef.current = screenState;
|
||||
|
||||
const captureCameraFrame = useCallback(() => {
|
||||
capturePipeFrame(cameraPipeRef.current, CAMERA_FRAME_WIDTH, CAMERA_JPEG_QUALITY);
|
||||
}, []);
|
||||
|
||||
const captureScreenFrame = useCallback(() => {
|
||||
capturePipeFrame(screenPipeRef.current, SCREEN_FRAME_WIDTH, SCREEN_JPEG_QUALITY);
|
||||
}, []);
|
||||
|
||||
const stopScreenShare = useCallback(() => {
|
||||
teardownPipe(screenPipeRef.current);
|
||||
screenStreamRef.current = null;
|
||||
setScreenState('idle');
|
||||
}, []);
|
||||
|
||||
const stop = useCallback(() => {
|
||||
teardownPipe(cameraPipeRef.current);
|
||||
streamRef.current = null;
|
||||
setState('idle');
|
||||
setCameraOn(true);
|
||||
stopScreenShare();
|
||||
}, [stopScreenShare]);
|
||||
|
||||
// Acquire the webcam and start its capture pipeline. Shared by start()
|
||||
// and by re-enabling the camera mid-session.
|
||||
const acquireCamera = useCallback(async (): Promise<boolean> => {
|
||||
// Settle the macOS TCC camera permission before getUserMedia, same as
|
||||
// voice mode does for the mic — otherwise the first click silently
|
||||
// fails while the native prompt is still up.
|
||||
const access = await window.ipc
|
||||
.invoke('voice:ensureCameraAccess', null)
|
||||
.catch(() => ({ granted: true }));
|
||||
if (!access.granted) {
|
||||
console.error('[video] Camera access denied');
|
||||
return false;
|
||||
}
|
||||
|
||||
let stream: MediaStream | null = null;
|
||||
try {
|
||||
stream = await navigator.mediaDevices.getUserMedia({
|
||||
video: { width: { ideal: 1280 }, height: { ideal: 720 }, facingMode: 'user' },
|
||||
audio: false,
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('[video] Camera access failed:', err);
|
||||
return false;
|
||||
}
|
||||
|
||||
streamRef.current = stream;
|
||||
attachPipeSource(cameraPipeRef.current, stream, captureCameraFrame);
|
||||
return true;
|
||||
}, [captureCameraFrame]);
|
||||
|
||||
/**
|
||||
* Turn the camera off/on without leaving video mode (Meet-style). While
|
||||
* off, no webcam frames are captured or attached; screen-share frames
|
||||
* (if presenting) keep flowing.
|
||||
*/
|
||||
const setCameraEnabled = useCallback(async (enabled: boolean): Promise<boolean> => {
|
||||
if (stateRef.current !== 'live') return false;
|
||||
if (enabled) {
|
||||
const ok = await acquireCamera();
|
||||
if (ok) setCameraOn(true);
|
||||
return ok;
|
||||
}
|
||||
teardownPipe(cameraPipeRef.current);
|
||||
streamRef.current = null;
|
||||
setCameraOn(false);
|
||||
return true;
|
||||
}, [acquireCamera]);
|
||||
|
||||
/**
|
||||
* Start video mode. `camera: false` starts a camera-less session (voice
|
||||
* call / screen-share-only) — the mode is live so frames can flow from
|
||||
* other sources, and the camera can be enabled later via setCameraEnabled.
|
||||
*/
|
||||
const start = useCallback(async ({ camera = true }: { camera?: boolean } = {}): Promise<boolean> => {
|
||||
if (stateRef.current !== 'idle') return true;
|
||||
setState('starting');
|
||||
if (camera) {
|
||||
const ok = await acquireCamera();
|
||||
if (!ok) {
|
||||
setState('idle');
|
||||
return false;
|
||||
}
|
||||
}
|
||||
setCameraOn(camera);
|
||||
setState('live');
|
||||
return true;
|
||||
}, [acquireCamera]);
|
||||
|
||||
/**
|
||||
* Share the screen. The main process auto-approves getDisplayMedia with
|
||||
* the primary screen (see setDisplayMediaRequestHandler in main.ts), so
|
||||
* no source picker appears. Returns false if capture couldn't start
|
||||
* (usually the macOS Screen Recording permission).
|
||||
*/
|
||||
const startScreenShare = useCallback(async (): Promise<boolean> => {
|
||||
if (screenStateRef.current !== 'idle') return true;
|
||||
setScreenState('starting');
|
||||
|
||||
// Surfaces the macOS Screen Recording permission state and, on first
|
||||
// use, registers the app in System Settings (same flow meetings use).
|
||||
await window.ipc.invoke('meeting:checkScreenPermission', null).catch(() => null);
|
||||
|
||||
let stream: MediaStream | null = null;
|
||||
try {
|
||||
stream = await navigator.mediaDevices.getDisplayMedia({
|
||||
video: { frameRate: { ideal: 5 } },
|
||||
audio: false,
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('[video] Screen share failed:', err);
|
||||
setScreenState('idle');
|
||||
return false;
|
||||
}
|
||||
|
||||
screenStreamRef.current = stream;
|
||||
// The capture can end outside our UI (display unplugged, OS revokes) —
|
||||
// tear down cleanly so the UI doesn't show a dead share.
|
||||
stream.getVideoTracks()[0]?.addEventListener('ended', () => stopScreenShare(), { once: true });
|
||||
attachPipeSource(screenPipeRef.current, stream, captureScreenFrame);
|
||||
setScreenState('live');
|
||||
return true;
|
||||
}, [captureScreenFrame, stopScreenShare]);
|
||||
|
||||
/**
|
||||
* Drain webcam + screen-share frames buffered since the last send, tagged
|
||||
* by source. Webcam frames come first, then screen frames.
|
||||
*/
|
||||
const collectFrames = useCallback((): CapturedVideoFrame[] => {
|
||||
if (stateRef.current !== 'live') return [];
|
||||
// Grab a frame right now so the message always includes the moment of send.
|
||||
captureCameraFrame();
|
||||
const frames = drainPipe(cameraPipeRef.current, MAX_CAMERA_FRAMES_PER_MESSAGE, 'camera');
|
||||
if (screenStateRef.current === 'live') {
|
||||
captureScreenFrame();
|
||||
frames.push(...drainPipe(screenPipeRef.current, MAX_SCREEN_FRAMES_PER_MESSAGE, 'screen'));
|
||||
}
|
||||
return frames;
|
||||
}, [captureCameraFrame, captureScreenFrame]);
|
||||
|
||||
// Release the camera/screen if the component unmounts with video mode on.
|
||||
useEffect(() => stop, [stop]);
|
||||
|
||||
return { state, screenState, cameraOn, streamRef, screenStreamRef, start, stop, startScreenShare, stopScreenShare, setCameraEnabled, collectFrames };
|
||||
}
|
||||
|
|
@ -19,7 +19,35 @@ const DEEPGRAM_PARAMS = new URLSearchParams({
|
|||
endpointing: '100',
|
||||
no_delay: 'true',
|
||||
});
|
||||
const DEEPGRAM_LISTEN_URL = `wss://api.deepgram.com/v1/listen?${DEEPGRAM_PARAMS.toString()}`;
|
||||
// Hands-free (continuous) mode: Deepgram's endpoint fires FAST (600ms of
|
||||
// silence) and we apply smart hold logic on our side — if the transcript
|
||||
// already reads as a complete thought (terminal punctuation) the utterance
|
||||
// fires immediately, otherwise we hold INCOMPLETE_HOLD_MS longer in case the
|
||||
// user was mid-thought. Net effect: complete sentences turn around ~1.2s
|
||||
// faster than the old fixed 1800ms endpoint, while thinking pauses still get
|
||||
// the same total grace (~1.8s).
|
||||
const CONTINUOUS_ENDPOINTING_MS = 600;
|
||||
const INCOMPLETE_HOLD_MS = 1200;
|
||||
// While the mic is paused (assistant speaking), keep the idle Deepgram socket
|
||||
// alive — it closes after ~10s without audio otherwise.
|
||||
const KEEPALIVE_INTERVAL_MS = 5000;
|
||||
|
||||
// Deepgram punctuates finals (punctuate=true) — a transcript ending in
|
||||
// terminal punctuation (optionally inside a closing quote/paren) is treated
|
||||
// as a complete thought.
|
||||
const COMPLETE_THOUGHT_RE = /[.!?…]["')\]]*\s*$/;
|
||||
|
||||
function deepgramParams(continuous: boolean): URLSearchParams {
|
||||
if (!continuous) return DEEPGRAM_PARAMS;
|
||||
const params = new URLSearchParams(DEEPGRAM_PARAMS);
|
||||
params.set('endpointing', String(CONTINUOUS_ENDPOINTING_MS));
|
||||
// Second end-of-speech signal: speech_final can be missed (it often rides
|
||||
// on a result with an empty transcript, or never fires when background
|
||||
// noise keeps the endpointer engaged). UtteranceEnd is word-timing based
|
||||
// and arrives as its own message type, so we listen for both.
|
||||
params.set('utterance_end_ms', '1000');
|
||||
return params;
|
||||
}
|
||||
|
||||
// Cap on retained per-frame amplitude samples (~64ms/frame ⇒ ~5 min of history).
|
||||
// The waveform only ever displays the most recent window, so older samples are dropped.
|
||||
|
|
@ -53,6 +81,13 @@ export function useVoiceMode() {
|
|||
const audioLevelsRef = useRef<number[]>([]);
|
||||
// Running peak amplitude for the waveform auto-gain (see PEAK_DECAY/MIN_PEAK).
|
||||
const audioPeakRef = useRef(0);
|
||||
// Hands-free mode: invoked with each completed utterance (speech_final).
|
||||
const continuousCbRef = useRef<((text: string) => void) | null>(null);
|
||||
// While true (assistant is speaking), mic audio is dropped instead of streamed.
|
||||
const pausedRef = useRef(false);
|
||||
const keepAliveTimerRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
||||
// Pending mid-thought hold (smart endpointing) — see maybeEndUtterance.
|
||||
const holdTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
|
||||
// Refresh cached auth details (called on warmup, not on mic click)
|
||||
const refreshAuth = useCallback(async () => {
|
||||
|
|
@ -71,9 +106,44 @@ export function useVoiceMode() {
|
|||
}
|
||||
}, [refreshRowboatAccount]);
|
||||
|
||||
// Hands-free mode: flush the accumulated utterance to the callback.
|
||||
// Both end-of-speech signals may fire for the same utterance — the second
|
||||
// finds an empty buffer and is a no-op.
|
||||
const fireContinuousUtterance = useCallback(() => {
|
||||
if (holdTimerRef.current) {
|
||||
clearTimeout(holdTimerRef.current);
|
||||
holdTimerRef.current = null;
|
||||
}
|
||||
if (!continuousCbRef.current || pausedRef.current) return;
|
||||
const utterance = transcriptBufferRef.current.trim();
|
||||
transcriptBufferRef.current = '';
|
||||
interimRef.current = '';
|
||||
setInterimText('');
|
||||
if (utterance) continuousCbRef.current(utterance);
|
||||
}, []);
|
||||
|
||||
// Smart endpoint: Deepgram's endpoint fires fast (600ms). If the
|
||||
// transcript reads as a complete thought, hand it off immediately; if it
|
||||
// trails off mid-sentence ("so what I want is…"), hold a little longer —
|
||||
// resumed speech cancels the hold and the utterance keeps growing.
|
||||
const maybeEndUtterance = useCallback(() => {
|
||||
if (!continuousCbRef.current || pausedRef.current) return;
|
||||
const buffered = transcriptBufferRef.current.trim();
|
||||
if (!buffered) return;
|
||||
if (COMPLETE_THOUGHT_RE.test(buffered)) {
|
||||
fireContinuousUtterance();
|
||||
return;
|
||||
}
|
||||
if (holdTimerRef.current) clearTimeout(holdTimerRef.current);
|
||||
holdTimerRef.current = setTimeout(() => {
|
||||
holdTimerRef.current = null;
|
||||
fireContinuousUtterance();
|
||||
}, INCOMPLETE_HOLD_MS);
|
||||
}, [fireContinuousUtterance]);
|
||||
|
||||
// Create and connect a Deepgram WebSocket using cached auth.
|
||||
// Starts the connection and returns immediately (does not wait for open).
|
||||
const connectWs = useCallback(async () => {
|
||||
const connectWs = useCallback(async (continuous = false) => {
|
||||
if (wsRef.current && (wsRef.current.readyState === WebSocket.OPEN || wsRef.current.readyState === WebSocket.CONNECTING)) return;
|
||||
|
||||
// Refresh auth if we don't have it cached yet
|
||||
|
|
@ -82,12 +152,13 @@ export function useVoiceMode() {
|
|||
}
|
||||
if (!cachedAuth) return;
|
||||
|
||||
const params = deepgramParams(continuous);
|
||||
let ws: WebSocket;
|
||||
if (cachedAuth.type === 'rowboat') {
|
||||
const listenUrl = buildDeepgramListenUrl(cachedAuth.url, DEEPGRAM_PARAMS);
|
||||
const listenUrl = buildDeepgramListenUrl(cachedAuth.url, params);
|
||||
ws = new WebSocket(listenUrl, ['bearer', cachedAuth.token]);
|
||||
} else {
|
||||
ws = new WebSocket(DEEPGRAM_LISTEN_URL, ['token', cachedAuth.apiKey]);
|
||||
ws = new WebSocket(`wss://api.deepgram.com/v1/listen?${params.toString()}`, ['token', cachedAuth.apiKey]);
|
||||
}
|
||||
wsRef.current = ws;
|
||||
|
||||
|
|
@ -103,16 +174,43 @@ export function useVoiceMode() {
|
|||
|
||||
ws.onmessage = (event) => {
|
||||
const data = JSON.parse(event.data);
|
||||
if (!data.channel?.alternatives?.[0]) return;
|
||||
|
||||
// Hands-free mode: word-timing based end-of-speech marker.
|
||||
if (data.type === 'UtteranceEnd') {
|
||||
maybeEndUtterance();
|
||||
return;
|
||||
}
|
||||
|
||||
if (!data.channel?.alternatives?.[0]) return;
|
||||
const transcript = data.channel.alternatives[0].transcript;
|
||||
if (!transcript) return;
|
||||
|
||||
// The user resumed speaking — cancel any pending mid-thought hold
|
||||
// so the utterance keeps growing instead of firing under them.
|
||||
if (transcript && holdTimerRef.current) {
|
||||
clearTimeout(holdTimerRef.current);
|
||||
holdTimerRef.current = null;
|
||||
}
|
||||
|
||||
if (data.is_final) {
|
||||
transcriptBufferRef.current += (transcriptBufferRef.current ? ' ' : '') + transcript;
|
||||
interimRef.current = '';
|
||||
setInterimText(transcriptBufferRef.current);
|
||||
} else {
|
||||
// NOTE: the endpoint marker (speech_final) usually arrives on a
|
||||
// result whose transcript is EMPTY — the silence after the user
|
||||
// stops talking. Empty finals must still reach the speech_final
|
||||
// check below or hands-free utterances never complete.
|
||||
if (transcript) {
|
||||
transcriptBufferRef.current += (transcriptBufferRef.current ? ' ' : '') + transcript;
|
||||
interimRef.current = '';
|
||||
}
|
||||
// Hands-free mode: an endpoint may complete the utterance —
|
||||
// immediately for complete thoughts, after a short hold for
|
||||
// mid-sentence trails.
|
||||
if (continuousCbRef.current && data.speech_final) {
|
||||
maybeEndUtterance();
|
||||
return;
|
||||
}
|
||||
if (transcript) {
|
||||
setInterimText(transcriptBufferRef.current);
|
||||
}
|
||||
} else if (transcript) {
|
||||
interimRef.current = transcript;
|
||||
setInterimText(transcriptBufferRef.current + (transcriptBufferRef.current ? ' ' : '') + transcript);
|
||||
}
|
||||
|
|
@ -127,8 +225,17 @@ export function useVoiceMode() {
|
|||
ws.onclose = () => {
|
||||
console.log('[voice] WebSocket closed');
|
||||
wsRef.current = null;
|
||||
// A hands-free call is long-lived — if the socket drops while the
|
||||
// call is still on, reconnect instead of silently going deaf.
|
||||
if (continuousCbRef.current) {
|
||||
setTimeout(() => {
|
||||
if (continuousCbRef.current && !wsRef.current) {
|
||||
void connectWs(true);
|
||||
}
|
||||
}, 1000);
|
||||
}
|
||||
};
|
||||
}, [refreshAuth]);
|
||||
}, [refreshAuth, maybeEndUtterance]);
|
||||
|
||||
const waitForWsOpen = useCallback(async (timeoutMs = 1500): Promise<boolean> => {
|
||||
const ws = wsRef.current;
|
||||
|
|
@ -191,6 +298,16 @@ export function useVoiceMode() {
|
|||
wsRef.current.close();
|
||||
wsRef.current = null;
|
||||
}
|
||||
continuousCbRef.current = null;
|
||||
pausedRef.current = false;
|
||||
if (holdTimerRef.current) {
|
||||
clearTimeout(holdTimerRef.current);
|
||||
holdTimerRef.current = null;
|
||||
}
|
||||
if (keepAliveTimerRef.current) {
|
||||
clearInterval(keepAliveTimerRef.current);
|
||||
keepAliveTimerRef.current = null;
|
||||
}
|
||||
audioBufferRef.current = [];
|
||||
audioLevelsRef.current = [];
|
||||
audioPeakRef.current = 0;
|
||||
|
|
@ -200,7 +317,7 @@ export function useVoiceMode() {
|
|||
setState('idle');
|
||||
}, [stopInputCapture]);
|
||||
|
||||
const start = useCallback(async () => {
|
||||
const start = useCallback(async (continuous = false) => {
|
||||
if (state !== 'idle') return;
|
||||
|
||||
transcriptBufferRef.current = '';
|
||||
|
|
@ -235,7 +352,7 @@ export function useVoiceMode() {
|
|||
console.error('Microphone access denied:', err);
|
||||
return null;
|
||||
}),
|
||||
connectWs(),
|
||||
connectWs(continuous),
|
||||
]);
|
||||
|
||||
if (!stream) {
|
||||
|
|
@ -259,6 +376,9 @@ export function useVoiceMode() {
|
|||
processorRef.current = processor;
|
||||
|
||||
processor.onaudioprocess = (e) => {
|
||||
// Paused (assistant is speaking in a call): drop mic audio so the
|
||||
// assistant's own TTS never gets transcribed back at it.
|
||||
if (pausedRef.current) return;
|
||||
const float32 = e.inputBuffer.getChannelData(0);
|
||||
const int16 = new Int16Array(float32.length);
|
||||
let sumSquares = 0;
|
||||
|
|
@ -283,8 +403,13 @@ export function useVoiceMode() {
|
|||
if (wsRef.current?.readyState === WebSocket.OPEN) {
|
||||
wsRef.current.send(buffer);
|
||||
} else {
|
||||
// WebSocket still connecting — buffer the audio
|
||||
// WebSocket still connecting (or reconnecting mid-call) —
|
||||
// buffer the audio, bounded so an unreachable server during a
|
||||
// long call can't grow it without limit (~30s at 64ms/chunk).
|
||||
audioBufferRef.current.push(buffer);
|
||||
if (audioBufferRef.current.length > 500) {
|
||||
audioBufferRef.current.shift();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
|
@ -318,10 +443,46 @@ export function useVoiceMode() {
|
|||
stopAudioCapture();
|
||||
}, [stopAudioCapture]);
|
||||
|
||||
/**
|
||||
* Hands-free (call) mode: listen continuously and invoke `onUtterance`
|
||||
* with each completed utterance. Runs until cancel()/stop.
|
||||
*/
|
||||
const startContinuous = useCallback(async (onUtterance: (text: string) => void) => {
|
||||
continuousCbRef.current = onUtterance;
|
||||
await start(true);
|
||||
}, [start]);
|
||||
|
||||
/**
|
||||
* Mute/unmute the continuous stream (used while the assistant is
|
||||
* thinking/speaking). Keeps the Deepgram socket alive with KeepAlives and
|
||||
* discards any half-heard utterance from before the pause.
|
||||
*/
|
||||
const setPaused = useCallback((paused: boolean) => {
|
||||
if (pausedRef.current === paused) return;
|
||||
pausedRef.current = paused;
|
||||
if (paused) {
|
||||
if (holdTimerRef.current) {
|
||||
clearTimeout(holdTimerRef.current);
|
||||
holdTimerRef.current = null;
|
||||
}
|
||||
transcriptBufferRef.current = '';
|
||||
interimRef.current = '';
|
||||
setInterimText('');
|
||||
keepAliveTimerRef.current = setInterval(() => {
|
||||
if (wsRef.current?.readyState === WebSocket.OPEN) {
|
||||
wsRef.current.send(JSON.stringify({ type: 'KeepAlive' }));
|
||||
}
|
||||
}, KEEPALIVE_INTERVAL_MS);
|
||||
} else if (keepAliveTimerRef.current) {
|
||||
clearInterval(keepAliveTimerRef.current);
|
||||
keepAliveTimerRef.current = null;
|
||||
}
|
||||
}, []);
|
||||
|
||||
/** Pre-cache auth details so mic click skips IPC round-trips */
|
||||
const warmup = useCallback(() => {
|
||||
refreshAuth().catch(() => {});
|
||||
}, [refreshAuth]);
|
||||
|
||||
return { state, interimText, audioLevelsRef, start, submit, cancel, warmup };
|
||||
return { state, interimText, audioLevelsRef, start, submit, cancel, warmup, startContinuous, setPaused };
|
||||
}
|
||||
|
|
|
|||
|
|
@ -47,6 +47,8 @@ function playAudio(
|
|||
/** A queue entry: text to synthesize, or a ready-to-play audio URL (e.g. a bundled clip). */
|
||||
type QueueItem = { text: string } | { url: string };
|
||||
|
||||
type TtsChunkMsg = { requestId: string; chunkBase64?: string; done: boolean; error?: string };
|
||||
|
||||
export function useVoiceTTS() {
|
||||
const [state, setState] = useState<TTSState>('idle');
|
||||
const audioRef = useRef<HTMLAudioElement | null>(null);
|
||||
|
|
@ -54,6 +56,10 @@ export function useVoiceTTS() {
|
|||
const processingRef = useRef(false);
|
||||
// Pre-fetched audio ready to play immediately
|
||||
const prefetchedRef = useRef<Promise<SynthesizedAudio> | null>(null);
|
||||
// Streaming synthesis: per-request chunk handlers + the in-flight request
|
||||
// id (so cancel() can abort the main-process fetch).
|
||||
const streamHandlersRef = useRef<Map<string, (msg: TtsChunkMsg) => void>>(new Map());
|
||||
const activeStreamIdRef = useRef<string | null>(null);
|
||||
// Bumped by cancel(). A queue loop that awaited across a cancel sees a
|
||||
// stale generation and exits instead of playing audio that was cancelled
|
||||
// while still synthesizing (which would overlap the next utterance).
|
||||
|
|
@ -107,6 +113,127 @@ export function useVoiceTTS() {
|
|||
analyserRef.current = null;
|
||||
}, []);
|
||||
|
||||
// Route streaming TTS chunks to whichever request is waiting for them.
|
||||
useEffect(() => {
|
||||
return window.ipc.on('voice:tts-chunk', (msg) => {
|
||||
streamHandlersRef.current.get(msg.requestId)?.(msg);
|
||||
});
|
||||
}, []);
|
||||
|
||||
/**
|
||||
* Streaming synthesis + playback via MediaSource: audio starts on the
|
||||
* first chunk instead of after the full body. Rejects (for caller
|
||||
* fallback to non-streaming synth) if the stream fails before any audio
|
||||
* arrived; resolves when playback finishes.
|
||||
*/
|
||||
const streamSynthesizeAndPlay = useCallback((text: string, onStarted: () => void): Promise<void> => {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
if (typeof MediaSource === 'undefined' || !MediaSource.isTypeSupported('audio/mpeg')) {
|
||||
reject(new Error('MSE audio/mpeg unsupported'));
|
||||
return;
|
||||
}
|
||||
const requestId = `tts-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;
|
||||
const mediaSource = new MediaSource();
|
||||
const audio = new Audio();
|
||||
audio.src = URL.createObjectURL(mediaSource);
|
||||
audioRef.current = audio;
|
||||
connectAnalyser(audio);
|
||||
activeStreamIdRef.current = requestId;
|
||||
|
||||
let sourceBuffer: SourceBuffer | null = null;
|
||||
const pending: Uint8Array[] = [];
|
||||
let streamDone = false;
|
||||
let gotAudio = false;
|
||||
let settled = false;
|
||||
|
||||
const cleanup = () => {
|
||||
streamHandlersRef.current.delete(requestId);
|
||||
if (activeStreamIdRef.current === requestId) activeStreamIdRef.current = null;
|
||||
URL.revokeObjectURL(audio.src);
|
||||
};
|
||||
const finish = (err?: Error) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
cleanup();
|
||||
if (err) reject(err);
|
||||
else resolve();
|
||||
};
|
||||
|
||||
// Drain pending chunks into the SourceBuffer one at a time
|
||||
// (appendBuffer is async; only one append may be in flight).
|
||||
const pump = () => {
|
||||
if (!sourceBuffer || sourceBuffer.updating || settled) return;
|
||||
const chunk = pending.shift();
|
||||
if (chunk) {
|
||||
try {
|
||||
sourceBuffer.appendBuffer(chunk as BufferSource);
|
||||
} catch (e) {
|
||||
finish(e as Error);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (streamDone && mediaSource.readyState === 'open') {
|
||||
try {
|
||||
mediaSource.endOfStream();
|
||||
} catch { /* already ended */ }
|
||||
}
|
||||
};
|
||||
|
||||
mediaSource.addEventListener('sourceopen', () => {
|
||||
try {
|
||||
sourceBuffer = mediaSource.addSourceBuffer('audio/mpeg');
|
||||
} catch (e) {
|
||||
finish(e as Error);
|
||||
return;
|
||||
}
|
||||
sourceBuffer.addEventListener('updateend', pump);
|
||||
pump();
|
||||
}, { once: true });
|
||||
|
||||
streamHandlersRef.current.set(requestId, (msg) => {
|
||||
if (msg.error && !gotAudio) {
|
||||
streamDone = true;
|
||||
finish(new Error(msg.error));
|
||||
return;
|
||||
}
|
||||
if (msg.chunkBase64) {
|
||||
gotAudio = true;
|
||||
const bin = atob(msg.chunkBase64);
|
||||
const bytes = new Uint8Array(bin.length);
|
||||
for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
|
||||
pending.push(bytes);
|
||||
pump();
|
||||
}
|
||||
if (msg.done) {
|
||||
streamDone = true;
|
||||
pump();
|
||||
}
|
||||
});
|
||||
|
||||
audio.addEventListener('playing', () => onStarted(), { once: true });
|
||||
audio.onended = () => finish();
|
||||
// pause() (from cancel) must settle this promise too; natural end
|
||||
// also fires 'pause' just before 'ended'; double-settle is a no-op.
|
||||
audio.onpause = () => finish();
|
||||
audio.onerror = () => finish(new Error('stream playback failed'));
|
||||
|
||||
window.ipc
|
||||
.invoke('voice:synthesizeStreamStart', { requestId, text })
|
||||
.then((res) => {
|
||||
if (!res.ok) finish(new Error(res.error || 'stream start failed'));
|
||||
})
|
||||
.catch((e) => finish(e as Error));
|
||||
|
||||
// Starts as soon as the first appended data is decodable.
|
||||
audio.play().catch(() => { /* surfaced via onerror / chunk error */ });
|
||||
|
||||
// Nothing arrived at all — bail so the caller can fall back.
|
||||
setTimeout(() => {
|
||||
if (!gotAudio && !settled) finish(new Error('stream timeout'));
|
||||
}, 10_000);
|
||||
});
|
||||
}, [connectAnalyser]);
|
||||
|
||||
const getLevel = useCallback((): number => {
|
||||
const analyser = analyserRef.current;
|
||||
if (!analyser) return 0;
|
||||
|
|
@ -130,10 +257,41 @@ export function useVoiceTTS() {
|
|||
processingRef.current = true;
|
||||
const gen = generationRef.current;
|
||||
|
||||
// Kick off full-body pre-fetch for the next queued text while the
|
||||
// current one plays — keeps sentence-to-sentence playback gapless.
|
||||
const prefetchNext = () => {
|
||||
const next = queueRef.current[0];
|
||||
if (next && 'text' in next && next.text.trim() && !prefetchedRef.current) {
|
||||
console.log('[tts] pre-fetching next:', next.text.substring(0, 80));
|
||||
prefetchedRef.current = synthesize(next.text);
|
||||
}
|
||||
};
|
||||
|
||||
while (queueRef.current.length > 0) {
|
||||
const item = queueRef.current.shift()!;
|
||||
if ('text' in item && !item.text.trim()) continue;
|
||||
|
||||
// Cold start (nothing playing, nothing pre-fetched): stream the
|
||||
// synthesis so audio begins on the first chunk instead of after
|
||||
// the full body — this is where first-response latency lives.
|
||||
if ('text' in item && !prefetchedRef.current) {
|
||||
setState('synthesizing');
|
||||
console.log('[tts] stream-synthesizing:', item.text.substring(0, 80));
|
||||
try {
|
||||
await streamSynthesizeAndPlay(item.text, () => {
|
||||
if (generationRef.current !== gen) return;
|
||||
setState('speaking');
|
||||
prefetchNext();
|
||||
});
|
||||
if (generationRef.current !== gen) return;
|
||||
continue;
|
||||
} catch (err) {
|
||||
if (generationRef.current !== gen) return;
|
||||
console.error('[tts] stream failed, falling back to full synth:', err);
|
||||
// fall through to the non-streaming path below
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
// Pre-recorded URL plays as-is; text uses the pre-fetched
|
||||
// result if available, otherwise synthesizes now.
|
||||
|
|
@ -156,12 +314,7 @@ export function useVoiceTTS() {
|
|||
if (generationRef.current !== gen) return;
|
||||
setState('speaking');
|
||||
|
||||
// Kick off pre-fetch for next chunk while this one plays
|
||||
const next = queueRef.current[0];
|
||||
if (next && 'text' in next && next.text.trim()) {
|
||||
console.log('[tts] pre-fetching next:', next.text.substring(0, 80));
|
||||
prefetchedRef.current = synthesize(next.text);
|
||||
}
|
||||
prefetchNext();
|
||||
|
||||
await playAudio(audio.dataUrl, audioRef, connectAnalyser);
|
||||
if (generationRef.current !== gen) return;
|
||||
|
|
@ -176,7 +329,7 @@ export function useVoiceTTS() {
|
|||
prefetchedRef.current = null;
|
||||
processingRef.current = false;
|
||||
setState('idle');
|
||||
}, [connectAnalyser]);
|
||||
}, [connectAnalyser, streamSynthesizeAndPlay]);
|
||||
|
||||
const speak = useCallback((text: string) => {
|
||||
console.log('[tts] speak() called:', text.substring(0, 80));
|
||||
|
|
@ -196,6 +349,13 @@ export function useVoiceTTS() {
|
|||
generationRef.current++;
|
||||
queueRef.current = [];
|
||||
prefetchedRef.current = null;
|
||||
// Abort any in-flight streaming synthesis in the main process.
|
||||
if (activeStreamIdRef.current) {
|
||||
void window.ipc
|
||||
.invoke('voice:synthesizeStreamCancel', { requestId: activeStreamIdRef.current })
|
||||
.catch(() => {});
|
||||
activeStreamIdRef.current = null;
|
||||
}
|
||||
if (audioRef.current) {
|
||||
audioRef.current.pause();
|
||||
audioRef.current = null;
|
||||
|
|
|
|||
|
|
@ -65,6 +65,26 @@ export function voiceInputStarted() {
|
|||
posthog.capture('voice_input_started')
|
||||
}
|
||||
|
||||
export function callStarted(preset: 'voice' | 'video' | 'share' | 'practice') {
|
||||
posthog.capture('call_started', { preset })
|
||||
}
|
||||
|
||||
// Voice-to-voice latency breakdown for one call turn (all milliseconds):
|
||||
// utterance accepted → message submitted → first TTS speak() → audio playing.
|
||||
export function callTurnLatency(props: {
|
||||
endpointToSubmitMs: number
|
||||
submitToSpeakMs: number
|
||||
speakToAudioMs: number
|
||||
totalMs: number
|
||||
}) {
|
||||
posthog.capture('call_turn_latency', {
|
||||
endpoint_to_submit_ms: Math.round(props.endpointToSubmitMs),
|
||||
submit_to_speak_ms: Math.round(props.submitToSpeakMs),
|
||||
speak_to_audio_ms: Math.round(props.speakToAudioMs),
|
||||
total_ms: Math.round(props.totalMs),
|
||||
})
|
||||
}
|
||||
|
||||
export function searchExecuted(types: string[]) {
|
||||
posthog.capture('search_executed', { types })
|
||||
}
|
||||
|
|
|
|||
30
apps/x/apps/renderer/src/lib/call-sounds.ts
Normal file
30
apps/x/apps/renderer/src/lib/call-sounds.ts
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
// Tiny synthesized UI sounds for calls — no audio assets, one lazy context.
|
||||
|
||||
let ctx: AudioContext | null = null
|
||||
|
||||
/**
|
||||
* Soft rising blip played the instant an utterance is accepted — sub-second
|
||||
* acknowledgment makes the (still ongoing) model turn feel responsive
|
||||
* instead of dead air.
|
||||
*/
|
||||
export function playAckCue() {
|
||||
try {
|
||||
if (!ctx) ctx = new AudioContext()
|
||||
if (ctx.state === 'suspended') void ctx.resume()
|
||||
const t = ctx.currentTime
|
||||
const osc = ctx.createOscillator()
|
||||
const gain = ctx.createGain()
|
||||
osc.type = 'sine'
|
||||
osc.frequency.setValueAtTime(880, t)
|
||||
osc.frequency.exponentialRampToValueAtTime(1320, t + 0.08)
|
||||
gain.gain.setValueAtTime(0.0001, t)
|
||||
gain.gain.exponentialRampToValueAtTime(0.08, t + 0.015)
|
||||
gain.gain.exponentialRampToValueAtTime(0.0001, t + 0.12)
|
||||
osc.connect(gain)
|
||||
gain.connect(ctx.destination)
|
||||
osc.start(t)
|
||||
osc.stop(t + 0.13)
|
||||
} catch {
|
||||
// cosmetic — never let a sound failure affect the call
|
||||
}
|
||||
}
|
||||
|
|
@ -10,6 +10,9 @@ export interface MessageAttachment {
|
|||
mimeType: string
|
||||
size?: number
|
||||
thumbnailUrl?: string
|
||||
/** Live webcam frame from video chat mode — rendered as a compact filmstrip.
|
||||
* Carries no path; thumbnailUrl holds the frame as a data: URL. */
|
||||
isVideoFrame?: boolean
|
||||
}
|
||||
|
||||
export interface ChatMessage {
|
||||
|
|
@ -198,6 +201,22 @@ const summarizeFilterUpdates = (updates: Record<string, unknown>): string => {
|
|||
return parts.length > 0 ? parts.join(', ') : 'Updated view'
|
||||
}
|
||||
|
||||
const APP_VIEW_LABELS: Record<string, string> = {
|
||||
home: 'home',
|
||||
email: 'email',
|
||||
meetings: 'meetings',
|
||||
'live-notes': 'live notes',
|
||||
'bg-tasks': 'background agents',
|
||||
'chat-history': 'chat history',
|
||||
knowledge: 'knowledge',
|
||||
workspace: 'workspace',
|
||||
code: 'code',
|
||||
bases: 'bases',
|
||||
graph: 'graph',
|
||||
}
|
||||
|
||||
const appViewLabel = (view: unknown): string => APP_VIEW_LABELS[view as string] ?? String(view ?? 'view')
|
||||
|
||||
export const getAppActionCardData = (tool: ToolCall): AppActionCardData | null => {
|
||||
if (tool.name !== 'app-navigation') return null
|
||||
const result = tool.result as Record<string, unknown> | undefined
|
||||
|
|
@ -209,7 +228,9 @@ export const getAppActionCardData = (tool: ToolCall): AppActionCardData | null =
|
|||
const action = input.action as string
|
||||
switch (action) {
|
||||
case 'open-note': return { action, label: `Opening ${(input.path as string || '').split('/').pop()?.replace(/\.md$/, '') || 'note'}...` }
|
||||
case 'open-view': return { action, label: `Opening ${input.view} view...` }
|
||||
case 'open-view': return { action, label: `Opening ${appViewLabel(input.view)}...` }
|
||||
case 'read-view': return { action, label: `Reading ${appViewLabel(input.view)}...` }
|
||||
case 'open-item': return { action, label: 'Opening...' }
|
||||
case 'update-base-view': return { action, label: 'Updating view...' }
|
||||
case 'create-base': return { action, label: `Creating "${input.name}"...` }
|
||||
case 'get-base-state': return null // renders as normal tool block
|
||||
|
|
@ -224,7 +245,31 @@ export const getAppActionCardData = (tool: ToolCall): AppActionCardData | null =
|
|||
return { action: 'open-note', label: `Opened ${name}` }
|
||||
}
|
||||
case 'open-view':
|
||||
return { action: 'open-view', label: `Opened ${result.view} view` }
|
||||
return { action: 'open-view', label: `Opened ${appViewLabel(result.view)}` }
|
||||
case 'read-view': {
|
||||
const counted =
|
||||
(result.threads as unknown[] | undefined)?.length ??
|
||||
(result.agents as unknown[] | undefined)?.length ??
|
||||
(result.sessions as unknown[] | undefined)?.length
|
||||
return {
|
||||
action: 'read-view',
|
||||
label: counted !== undefined
|
||||
? `Read ${appViewLabel(result.view)} (${counted} item${counted === 1 ? '' : 's'})`
|
||||
: `Read ${appViewLabel(result.view)}`,
|
||||
}
|
||||
}
|
||||
case 'open-item': {
|
||||
switch (result.kind) {
|
||||
case 'email-thread': return { action: 'open-item', label: 'Opened email thread' }
|
||||
case 'note': {
|
||||
const name = (result.path as string || '').split('/').pop()?.replace(/\.md$/, '') || 'note'
|
||||
return { action: 'open-item', label: `Opened ${name}` }
|
||||
}
|
||||
case 'bg-task': return { action: 'open-item', label: `Opened agent "${result.taskName}"` }
|
||||
case 'session': return { action: 'open-item', label: 'Opened chat' }
|
||||
default: return { action: 'open-item', label: 'Opened item' }
|
||||
}
|
||||
}
|
||||
case 'update-base-view':
|
||||
return {
|
||||
action: 'update-base-view',
|
||||
|
|
|
|||
|
|
@ -41,6 +41,9 @@ export function runLogToConversation(log: RunLog): ConversationItem[] {
|
|||
filename?: string
|
||||
mimeType?: string
|
||||
size?: number
|
||||
data?: string
|
||||
mediaType?: string
|
||||
source?: string
|
||||
toolCallId?: string
|
||||
toolName?: string
|
||||
arguments?: unknown
|
||||
|
|
@ -52,13 +55,24 @@ export function runLogToConversation(log: RunLog): ConversationItem[] {
|
|||
.join('')
|
||||
|
||||
const attachmentParts = parts.filter((p) => p.type === 'attachment' && p.path)
|
||||
if (attachmentParts.length > 0) {
|
||||
msgAttachments = attachmentParts.map((p) => ({
|
||||
path: p.path!,
|
||||
filename: p.filename || p.path!.split('/').pop() || p.path!,
|
||||
mimeType: p.mimeType || 'application/octet-stream',
|
||||
size: p.size,
|
||||
}))
|
||||
// Video-mode webcam frames — inline base64 image parts, shown as a filmstrip
|
||||
const imageParts = parts.filter((p) => p.type === 'image' && p.data)
|
||||
if (attachmentParts.length > 0 || imageParts.length > 0) {
|
||||
msgAttachments = [
|
||||
...attachmentParts.map((p) => ({
|
||||
path: p.path!,
|
||||
filename: p.filename || p.path!.split('/').pop() || p.path!,
|
||||
mimeType: p.mimeType || 'application/octet-stream',
|
||||
size: p.size,
|
||||
})),
|
||||
...imageParts.map((p, index) => ({
|
||||
path: '',
|
||||
filename: `${p.source === 'screen' ? 'screen' : 'camera'}-frame-${index + 1}.jpg`,
|
||||
mimeType: p.mediaType || 'image/jpeg',
|
||||
thumbnailUrl: `data:${p.mediaType || 'image/jpeg'};base64,${p.data}`,
|
||||
isVideoFrame: true,
|
||||
})),
|
||||
]
|
||||
}
|
||||
|
||||
if (msg.role === 'assistant') {
|
||||
|
|
|
|||
|
|
@ -96,6 +96,25 @@ describe('voice output', () => {
|
|||
expect(overlay.voiceSegments).toEqual(['hello there', 'bye'])
|
||||
})
|
||||
|
||||
it('emits an early clause from a long open block, then the remainder on close', () => {
|
||||
let overlay = emptyOverlay()
|
||||
const longClause = 'Okay so the first thing I would look at here is the error message,'
|
||||
overlay = applyOverlay(overlay, delta(`<voice>${longClause} because`))
|
||||
// Open block crossed the early-speech threshold at a clause boundary.
|
||||
expect(overlay.voiceSegments).toEqual([longClause])
|
||||
overlay = applyOverlay(overlay, delta(' it tells you the root cause.</voice>'))
|
||||
// Remainder only — the early clause is not repeated.
|
||||
expect(overlay.voiceSegments).toEqual([longClause, 'because it tells you the root cause.'])
|
||||
})
|
||||
|
||||
it('does not emit early clauses from short open blocks', () => {
|
||||
let overlay = emptyOverlay()
|
||||
overlay = applyOverlay(overlay, delta('<voice>Sure, one sec'))
|
||||
expect(overlay.voiceSegments).toEqual([])
|
||||
overlay = applyOverlay(overlay, delta('.</voice>'))
|
||||
expect(overlay.voiceSegments).toEqual(['Sure, one sec.'])
|
||||
})
|
||||
|
||||
it('keeps segments but resets the scan on model_call_completed', () => {
|
||||
let overlay = emptyOverlay()
|
||||
overlay = applyOverlay(overlay, delta('<voice>one</voice>'))
|
||||
|
|
|
|||
|
|
@ -34,14 +34,20 @@ export type LiveOverlay = {
|
|||
text: string
|
||||
reasoning: string
|
||||
toolOutput: Record<string, string>
|
||||
// Contents of completed <voice>…</voice> blocks seen while streaming, in
|
||||
// order, monotonically growing for the lifetime of the overlay (i.e. one
|
||||
// active turn). Consumers speak segments beyond what they've already
|
||||
// spoken; the overlay reset on turn switch starts a fresh list.
|
||||
// Speakable segments seen while streaming, in order, monotonically growing
|
||||
// for the lifetime of the overlay (i.e. one active turn). Usually the
|
||||
// contents of completed <voice>…</voice> blocks, but a long still-open
|
||||
// block may emit an early clause (see EARLY_SPEECH_MIN_CHARS) so speech
|
||||
// can start before the sentence finishes generating. Consumers speak
|
||||
// segments beyond what they've already spoken; the overlay reset on turn
|
||||
// switch starts a fresh list.
|
||||
voiceSegments: string[]
|
||||
// Scan cursor into `text` — everything before it has been checked for
|
||||
// complete voice blocks.
|
||||
voiceScanIndex: number
|
||||
// Chars of the currently-open voice block's content already emitted as an
|
||||
// early clause — the block's remainder (on close) excludes them.
|
||||
voicePartialConsumed: number
|
||||
}
|
||||
|
||||
export const emptyOverlay = (): LiveOverlay => ({
|
||||
|
|
@ -50,6 +56,7 @@ export const emptyOverlay = (): LiveOverlay => ({
|
|||
toolOutput: {},
|
||||
voiceSegments: [],
|
||||
voiceScanIndex: 0,
|
||||
voicePartialConsumed: 0,
|
||||
})
|
||||
|
||||
// The model emits <voice>…</voice> around speakable text when voice output
|
||||
|
|
@ -59,6 +66,17 @@ export function stripVoiceTags(text: string): string {
|
|||
}
|
||||
|
||||
const VOICE_BLOCK = /<voice>([\s\S]*?)<\/voice>/g
|
||||
const VOICE_OPEN_TAG = '<voice>'
|
||||
|
||||
// Early speech: once an open block has this many unconsumed chars, its last
|
||||
// complete clause is emitted immediately instead of waiting for </voice> —
|
||||
// TTS starts on the first clause while the rest of the sentence generates.
|
||||
const EARLY_SPEECH_MIN_CHARS = 60
|
||||
// ...but never emit a fragment shorter than this (prosody suffers).
|
||||
const EARLY_SPEECH_MIN_EMIT = 30
|
||||
// Clause boundaries (punctuation, optionally inside closing quote/paren,
|
||||
// followed by whitespace or end-of-buffer).
|
||||
const CLAUSE_BOUNDARY = /[,;:.!?…—]["')\]]*(?=\s|$)/g
|
||||
|
||||
// Accumulates deltas; canonical durable events supersede the buffers (the
|
||||
// committed transcript now contains what was streaming).
|
||||
|
|
@ -68,15 +86,43 @@ export function applyOverlay(overlay: LiveOverlay, event: TurnStreamEvent): Live
|
|||
const text = overlay.text + event.delta
|
||||
// Extract complete voice blocks past the scan cursor. Incomplete
|
||||
// blocks (opening tag seen, closing not yet) stay unconsumed until a
|
||||
// later delta completes them.
|
||||
// later delta completes them. The first complete block may have had an
|
||||
// early clause emitted while it was open — skip those chars.
|
||||
const segments: string[] = []
|
||||
let scanIndex = overlay.voiceScanIndex
|
||||
let partialConsumed = overlay.voicePartialConsumed
|
||||
VOICE_BLOCK.lastIndex = scanIndex
|
||||
for (let m = VOICE_BLOCK.exec(text); m; m = VOICE_BLOCK.exec(text)) {
|
||||
const content = m[1].trim()
|
||||
const content = m[1].slice(partialConsumed).trim()
|
||||
partialConsumed = 0
|
||||
if (content) segments.push(content)
|
||||
scanIndex = m.index + m[0].length
|
||||
}
|
||||
|
||||
// Early speech: if a voice block is still open and has accumulated a
|
||||
// long unconsumed run, emit its last complete clause now — speech can
|
||||
// start while the rest of the sentence is still generating.
|
||||
const openIdx = text.indexOf(VOICE_OPEN_TAG, scanIndex)
|
||||
if (openIdx !== -1) {
|
||||
const unconsumed = text.slice(openIdx + VOICE_OPEN_TAG.length + partialConsumed)
|
||||
if (unconsumed.length >= EARLY_SPEECH_MIN_CHARS) {
|
||||
let lastBoundaryEnd = -1
|
||||
CLAUSE_BOUNDARY.lastIndex = 0
|
||||
for (let b = CLAUSE_BOUNDARY.exec(unconsumed); b; b = CLAUSE_BOUNDARY.exec(unconsumed)) {
|
||||
lastBoundaryEnd = b.index + b[0].length
|
||||
}
|
||||
if (lastBoundaryEnd >= EARLY_SPEECH_MIN_EMIT) {
|
||||
const clause = unconsumed.slice(0, lastBoundaryEnd).trim()
|
||||
if (clause) segments.push(clause)
|
||||
partialConsumed += lastBoundaryEnd
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// No open block — any partial bookkeeping belongs to a block that
|
||||
// has since closed.
|
||||
partialConsumed = 0
|
||||
}
|
||||
|
||||
return {
|
||||
...overlay,
|
||||
text,
|
||||
|
|
@ -84,12 +130,13 @@ export function applyOverlay(overlay: LiveOverlay, event: TurnStreamEvent): Live
|
|||
? { voiceSegments: [...overlay.voiceSegments, ...segments] }
|
||||
: {}),
|
||||
voiceScanIndex: scanIndex,
|
||||
voicePartialConsumed: partialConsumed,
|
||||
}
|
||||
}
|
||||
case 'reasoning_delta':
|
||||
return { ...overlay, reasoning: overlay.reasoning + event.delta }
|
||||
case 'model_call_completed':
|
||||
return { ...overlay, text: '', reasoning: '', voiceScanIndex: 0 }
|
||||
return { ...overlay, text: '', reasoning: '', voiceScanIndex: 0, voicePartialConsumed: 0 }
|
||||
case 'tool_progress': {
|
||||
const progress = event.progress
|
||||
if (
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import { PostHogProvider } from 'posthog-js/react'
|
|||
import type { CaptureResult } from 'posthog-js'
|
||||
import { ThemeProvider } from '@/contexts/theme-context'
|
||||
import { configureAnalyticsContext } from './lib/analytics'
|
||||
import { VideoPopout } from '@/components/video-popout'
|
||||
|
||||
// Fetch the stable installation ID from main so renderer + main share one
|
||||
// PostHog distinct_id. Falls back to PostHog's auto-generated anonymous ID
|
||||
|
|
@ -57,4 +58,14 @@ async function bootstrap() {
|
|||
// The loaded callback applies api_url/app_version once PostHog has initialized.
|
||||
}
|
||||
|
||||
bootstrap()
|
||||
// The video-mode popout window loads the same bundle with a hash route and
|
||||
// renders only the mini-call UI — no analytics or app bootstrap needed.
|
||||
if (window.location.hash === '#video-popout') {
|
||||
createRoot(document.getElementById('root')!).render(
|
||||
<StrictMode>
|
||||
<VideoPopout />
|
||||
</StrictMode>,
|
||||
)
|
||||
} else {
|
||||
bootstrap()
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue