mirror of
https://github.com/rowboatlabs/rowboat.git
synced 2026-07-12 21:02:17 +02:00
added screen share
This commit is contained in:
Arjun
parent
42353c63f4
commit
48a3ff6a36
7 changed files with 392 additions and 177 deletions
|
|
@ -1221,6 +1221,16 @@ function App() {
|
|||
voiceRef.current.setPaused(activeIsProcessing || tts.state !== 'idle')
|
||||
}, [videoChatMode, activeIsProcessing, tts.state])
|
||||
|
||||
// Screen sharing (any video mode): frames of the shared screen ride along
|
||||
// with each message next to the webcam frames.
|
||||
const handleToggleScreenShare = useCallback(async () => {
|
||||
if (video.screenState === 'live') {
|
||||
video.stopScreenShare()
|
||||
} else {
|
||||
await video.startScreenShare()
|
||||
}
|
||||
}, [video])
|
||||
|
||||
// Enter to submit voice input, Escape to cancel
|
||||
useEffect(() => {
|
||||
const handleKeyDown = (e: KeyboardEvent) => {
|
||||
|
|
@ -2717,7 +2727,7 @@ function App() {
|
|||
})),
|
||||
...videoFrames.map((frame, index) => ({
|
||||
path: '',
|
||||
filename: `camera-frame-${index + 1}.jpg`,
|
||||
filename: `${frame.source}-frame-${index + 1}.jpg`,
|
||||
mimeType: frame.mediaType,
|
||||
thumbnailUrl: frame.dataUrl,
|
||||
isVideoFrame: true,
|
||||
|
|
@ -2800,7 +2810,7 @@ function App() {
|
|||
type: 'image'
|
||||
data: string
|
||||
mediaType: string
|
||||
source: 'camera'
|
||||
source: 'camera' | 'screen'
|
||||
capturedAt: string
|
||||
}
|
||||
|
||||
|
|
@ -2839,7 +2849,7 @@ function App() {
|
|||
type: 'image',
|
||||
data: frame.data,
|
||||
mediaType: frame.mediaType,
|
||||
source: 'camera',
|
||||
source: frame.source,
|
||||
capturedAt: frame.capturedAt,
|
||||
})
|
||||
}
|
||||
|
|
@ -6599,12 +6609,17 @@ function App() {
|
|||
: undefined
|
||||
}
|
||||
interimText={videoChatMode === 'call' ? voice.interimText : undefined}
|
||||
isScreenSharing={video.screenState === 'live'}
|
||||
onToggleScreenShare={handleToggleScreenShare}
|
||||
/>
|
||||
)}
|
||||
{/* Full-screen Meet-style call: user tile + animated mascot tile */}
|
||||
{videoChatMode === 'meeting' && (
|
||||
<VideoCallView
|
||||
streamRef={video.streamRef}
|
||||
screenStreamRef={video.screenStreamRef}
|
||||
isScreenSharing={video.screenState === 'live'}
|
||||
onToggleScreenShare={handleToggleScreenShare}
|
||||
ttsState={tts.state}
|
||||
getTtsLevel={tts.getLevel}
|
||||
status={
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { useEffect, useRef, useState } from 'react'
|
||||
import { PhoneOff } from 'lucide-react'
|
||||
import { MonitorUp, PhoneOff } from 'lucide-react'
|
||||
|
||||
import { MascotFaceIcon, TalkingHead } from '@/components/talking-head'
|
||||
import type { TTSState } from '@/hooks/useVoiceTTS'
|
||||
|
|
@ -10,6 +10,10 @@ export type VideoCallStatus = 'listening' | 'thinking' | 'speaking'
|
|||
interface VideoCallViewProps {
|
||||
/** Live camera stream from useVideoMode — attached to the user's tile. */
|
||||
streamRef: React.MutableRefObject<MediaStream | null>
|
||||
/** Live screen-share stream — shown as the presentation tile when sharing. */
|
||||
screenStreamRef: React.MutableRefObject<MediaStream | null>
|
||||
isScreenSharing: boolean
|
||||
onToggleScreenShare: () => void
|
||||
ttsState: TTSState
|
||||
/** Live TTS output level — drives the mascot's mouth animation. */
|
||||
getTtsLevel: () => number
|
||||
|
|
@ -27,23 +31,17 @@ const STATUS_DISPLAY: Record<VideoCallStatus, { label: string; dotClass: string
|
|||
speaking: { label: 'Speaking', dotClass: 'bg-sky-400 animate-pulse' },
|
||||
}
|
||||
|
||||
/**
|
||||
* Full-screen hands-free call: a Meet-style two-tile layout with the user's
|
||||
* webcam on one side and the mascot as the other participant. The mascot
|
||||
* animates with the assistant's speech; dismissing it swaps in a Meet-style
|
||||
* letter avatar ("R"). Live captions run along the bottom.
|
||||
*/
|
||||
export function VideoCallView({
|
||||
/** Attach a MediaStream ref to a <video> element for the lifetime of the mount. */
|
||||
function StreamVideo({
|
||||
streamRef,
|
||||
ttsState,
|
||||
getTtsLevel,
|
||||
status,
|
||||
interimText,
|
||||
assistantCaption,
|
||||
onLeave,
|
||||
}: VideoCallViewProps) {
|
||||
mirrored,
|
||||
className,
|
||||
}: {
|
||||
streamRef: React.MutableRefObject<MediaStream | null>
|
||||
mirrored?: boolean
|
||||
className?: string
|
||||
}) {
|
||||
const videoRef = useRef<HTMLVideoElement | null>(null)
|
||||
const [mascotVisible, setMascotVisible] = useState(true)
|
||||
|
||||
useEffect(() => {
|
||||
const videoEl = videoRef.current
|
||||
|
|
@ -55,6 +53,38 @@ export function VideoCallView({
|
|||
}
|
||||
}, [streamRef])
|
||||
|
||||
return (
|
||||
<video
|
||||
ref={videoRef}
|
||||
muted
|
||||
playsInline
|
||||
className={className}
|
||||
style={mirrored ? { transform: 'scaleX(-1)' } : undefined}
|
||||
/>
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Full-screen hands-free call: a Meet-style layout with the user's webcam on
|
||||
* one side and the mascot as the other participant. While presenting, the
|
||||
* shared screen becomes the big tile and the participants shrink into a side
|
||||
* rail. The mascot animates with the assistant's speech; dismissing it swaps
|
||||
* in a Meet-style letter avatar ("R"). Live captions run along the bottom.
|
||||
*/
|
||||
export function VideoCallView({
|
||||
streamRef,
|
||||
screenStreamRef,
|
||||
isScreenSharing,
|
||||
onToggleScreenShare,
|
||||
ttsState,
|
||||
getTtsLevel,
|
||||
status,
|
||||
interimText,
|
||||
assistantCaption,
|
||||
onLeave,
|
||||
}: VideoCallViewProps) {
|
||||
const [mascotVisible, setMascotVisible] = useState(true)
|
||||
|
||||
const userSpeaking = status === 'listening' && Boolean(interimText)
|
||||
const assistantSpeaking = ttsState === 'speaking'
|
||||
|
||||
|
|
@ -64,58 +94,77 @@ export function VideoCallView({
|
|||
? { who: 'You', text: interimText }
|
||||
: null
|
||||
|
||||
const userTile = (
|
||||
<div
|
||||
className={cn(
|
||||
'relative flex items-center justify-center overflow-hidden rounded-2xl bg-neutral-900 transition-shadow',
|
||||
userSpeaking && 'ring-2 ring-green-500/80',
|
||||
isScreenSharing && 'aspect-video w-full'
|
||||
)}
|
||||
>
|
||||
<StreamVideo streamRef={streamRef} mirrored className="h-full w-full object-cover" />
|
||||
<span className="absolute bottom-3 left-3 rounded-md bg-black/50 px-2 py-0.5 text-sm text-white">
|
||||
You
|
||||
</span>
|
||||
</div>
|
||||
)
|
||||
|
||||
const assistantTile = (
|
||||
<div
|
||||
className={cn(
|
||||
'group relative flex items-center justify-center overflow-hidden rounded-2xl bg-neutral-900 transition-shadow',
|
||||
assistantSpeaking && 'ring-2 ring-sky-400/80',
|
||||
isScreenSharing && 'aspect-video w-full'
|
||||
)}
|
||||
>
|
||||
{mascotVisible ? (
|
||||
<TalkingHead ttsState={ttsState} getLevel={getTtsLevel} size={isScreenSharing ? 96 : 220} />
|
||||
) : (
|
||||
<span
|
||||
className={cn(
|
||||
'flex items-center justify-center rounded-full bg-sky-600 font-medium text-white',
|
||||
isScreenSharing ? 'h-16 w-16 text-3xl' : 'h-40 w-40 text-7xl'
|
||||
)}
|
||||
aria-label="Rowboat"
|
||||
>
|
||||
R
|
||||
</span>
|
||||
)}
|
||||
<span className="absolute bottom-3 left-3 rounded-md bg-black/50 px-2 py-0.5 text-sm text-white">
|
||||
Rowboat
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setMascotVisible((v) => !v)}
|
||||
className="absolute right-3 top-3 rounded-md bg-black/50 px-2 py-1 text-xs text-white/80 opacity-0 transition-opacity hover:text-white group-hover:opacity-100"
|
||||
>
|
||||
{mascotVisible ? 'Hide mascot' : 'Show mascot'}
|
||||
</button>
|
||||
</div>
|
||||
)
|
||||
|
||||
return (
|
||||
<div className="fixed inset-0 z-[100] flex flex-col bg-neutral-950">
|
||||
{/* Participant tiles */}
|
||||
<div className="grid min-h-0 flex-1 grid-cols-1 gap-3 p-4 pb-2 md:grid-cols-2">
|
||||
{/* User */}
|
||||
<div
|
||||
className={cn(
|
||||
'relative flex items-center justify-center overflow-hidden rounded-2xl bg-neutral-900 transition-shadow',
|
||||
userSpeaking && 'ring-2 ring-green-500/80'
|
||||
)}
|
||||
>
|
||||
<video
|
||||
ref={videoRef}
|
||||
muted
|
||||
playsInline
|
||||
className="h-full w-full object-cover"
|
||||
style={{ transform: 'scaleX(-1)' }}
|
||||
/>
|
||||
<span className="absolute bottom-3 left-3 rounded-md bg-black/50 px-2 py-0.5 text-sm text-white">
|
||||
You
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{/* Assistant */}
|
||||
<div
|
||||
className={cn(
|
||||
'group relative flex items-center justify-center overflow-hidden rounded-2xl bg-neutral-900 transition-shadow',
|
||||
assistantSpeaking && 'ring-2 ring-sky-400/80'
|
||||
)}
|
||||
>
|
||||
{mascotVisible ? (
|
||||
<TalkingHead ttsState={ttsState} getLevel={getTtsLevel} size={220} />
|
||||
) : (
|
||||
<span
|
||||
className="flex h-40 w-40 items-center justify-center rounded-full bg-sky-600 text-7xl font-medium text-white"
|
||||
aria-label="Rowboat"
|
||||
>
|
||||
R
|
||||
{/* Participant tiles — Meet-style presentation layout while sharing */}
|
||||
{isScreenSharing ? (
|
||||
<div className="flex min-h-0 flex-1 gap-3 p-4 pb-2">
|
||||
<div className="relative flex flex-1 items-center justify-center overflow-hidden rounded-2xl bg-neutral-900">
|
||||
<StreamVideo streamRef={screenStreamRef} className="h-full w-full object-contain" />
|
||||
<span className="absolute bottom-3 left-3 rounded-md bg-black/50 px-2 py-0.5 text-sm text-white">
|
||||
Your screen
|
||||
</span>
|
||||
)}
|
||||
<span className="absolute bottom-3 left-3 rounded-md bg-black/50 px-2 py-0.5 text-sm text-white">
|
||||
Rowboat
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setMascotVisible((v) => !v)}
|
||||
className="absolute right-3 top-3 rounded-md bg-black/50 px-2 py-1 text-xs text-white/80 opacity-0 transition-opacity hover:text-white group-hover:opacity-100"
|
||||
>
|
||||
{mascotVisible ? 'Hide mascot' : 'Show mascot'}
|
||||
</button>
|
||||
</div>
|
||||
<div className="flex w-52 shrink-0 flex-col gap-3">
|
||||
{userTile}
|
||||
{assistantTile}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
) : (
|
||||
<div className="grid min-h-0 flex-1 grid-cols-1 gap-3 p-4 pb-2 md:grid-cols-2">
|
||||
{userTile}
|
||||
{assistantTile}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Captions */}
|
||||
<div className="flex h-14 items-center justify-center px-6">
|
||||
|
|
@ -135,10 +184,22 @@ export function VideoCallView({
|
|||
</span>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setMascotVisible((v) => !v)}
|
||||
onClick={onToggleScreenShare}
|
||||
className={cn(
|
||||
'relative flex h-10 w-10 items-center justify-center rounded-full bg-neutral-800 text-white/90 transition-colors hover:bg-neutral-700',
|
||||
'flex h-10 w-10 items-center justify-center rounded-full transition-colors',
|
||||
isScreenSharing
|
||||
? 'bg-sky-600 text-white hover:bg-sky-500'
|
||||
: 'bg-neutral-800 text-white/90 hover:bg-neutral-700'
|
||||
)}
|
||||
aria-label={isScreenSharing ? 'Stop presenting' : 'Present your screen'}
|
||||
title={isScreenSharing ? 'Stop presenting' : 'Present your screen'}
|
||||
>
|
||||
<MonitorUp className="h-5 w-5" />
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setMascotVisible((v) => !v)}
|
||||
className="relative flex h-10 w-10 items-center justify-center rounded-full bg-neutral-800 text-white/90 transition-colors hover:bg-neutral-700"
|
||||
aria-label={mascotVisible ? 'Hide mascot' : 'Show mascot'}
|
||||
>
|
||||
<MascotFaceIcon />
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { useEffect, useRef } from 'react'
|
||||
import { X } from 'lucide-react'
|
||||
import { MonitorUp, X } from 'lucide-react'
|
||||
|
||||
interface VideoPreviewOverlayProps {
|
||||
/** Live camera stream from useVideoMode — attached to the preview element. */
|
||||
|
|
@ -9,6 +9,8 @@ interface VideoPreviewOverlayProps {
|
|||
callStatus?: 'listening' | 'thinking' | 'speaking'
|
||||
/** Hands-free call mode: live transcript of the in-progress utterance. */
|
||||
interimText?: string
|
||||
isScreenSharing?: boolean
|
||||
onToggleScreenShare?: () => void
|
||||
}
|
||||
|
||||
const CALL_STATUS_DISPLAY: Record<NonNullable<VideoPreviewOverlayProps['callStatus']>, { label: string; dotClass: string }> = {
|
||||
|
|
@ -22,7 +24,7 @@ const CALL_STATUS_DISPLAY: Record<NonNullable<VideoPreviewOverlayProps['callStat
|
|||
* on. Mirrored like a selfie camera so the user's movements feel natural.
|
||||
* Sits above the composer dock, mirroring the talking-head overlay's corner.
|
||||
*/
|
||||
export function VideoPreviewOverlay({ streamRef, onTurnOff, callStatus, interimText }: VideoPreviewOverlayProps) {
|
||||
export function VideoPreviewOverlay({ streamRef, onTurnOff, callStatus, interimText, isScreenSharing, onToggleScreenShare }: VideoPreviewOverlayProps) {
|
||||
const videoRef = useRef<HTMLVideoElement | null>(null)
|
||||
|
||||
useEffect(() => {
|
||||
|
|
@ -54,6 +56,21 @@ export function VideoPreviewOverlay({ streamRef, onTurnOff, callStatus, interimT
|
|||
>
|
||||
<X className="h-3 w-3" />
|
||||
</button>
|
||||
{onToggleScreenShare && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={onToggleScreenShare}
|
||||
className={`absolute -left-1.5 -top-1.5 z-10 flex h-5 w-5 items-center justify-center rounded-full border shadow-sm transition-opacity ${
|
||||
isScreenSharing
|
||||
? 'border-sky-500 bg-sky-600 text-white opacity-100'
|
||||
: 'border-border bg-background text-muted-foreground opacity-0 hover:text-foreground group-hover:opacity-100'
|
||||
}`}
|
||||
aria-label={isScreenSharing ? 'Stop sharing screen' : 'Share your screen'}
|
||||
title={isScreenSharing ? 'Stop sharing screen' : 'Share your screen'}
|
||||
>
|
||||
<MonitorUp className="h-3 w-3" />
|
||||
</button>
|
||||
)}
|
||||
<video
|
||||
ref={videoRef}
|
||||
muted
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
|
||||
export type VideoModeState = 'idle' | 'starting' | 'live';
|
||||
export type ScreenShareState = 'idle' | 'starting' | 'live';
|
||||
|
||||
export interface CapturedVideoFrame {
|
||||
/** base64-encoded JPEG bytes (no data: prefix) — shape of the UserImagePart wire format */
|
||||
|
|
@ -9,21 +10,28 @@ export interface CapturedVideoFrame {
|
|||
capturedAt: string; // ISO timestamp
|
||||
/** data: URL of the same frame, for direct display in the transcript */
|
||||
dataUrl: string;
|
||||
source: 'camera' | 'screen';
|
||||
}
|
||||
|
||||
// Frames are grabbed once per second — dense enough to catch expression and
|
||||
// posture changes while the user talks. Per message we attach at most
|
||||
// MAX_FRAMES_PER_MESSAGE frames, evenly sampled across the window since the
|
||||
// MAX_*_FRAMES_PER_MESSAGE frames, evenly sampled across the window since the
|
||||
// last send, so long monologues don't balloon the request.
|
||||
const CAPTURE_INTERVAL_MS = 1000;
|
||||
const MAX_FRAMES_PER_MESSAGE = 12;
|
||||
const MAX_CAMERA_FRAMES_PER_MESSAGE = 12;
|
||||
// Screen frames are ~4x the resolution (and tokens) of camera frames, and the
|
||||
// latest view matters far more than the trajectory — keep the cap small.
|
||||
const MAX_SCREEN_FRAMES_PER_MESSAGE = 4;
|
||||
// Rolling buffer bound (~2 minutes). The buffer only needs to cover the gap
|
||||
// between two sends; anything older is stale context anyway.
|
||||
const MAX_BUFFERED_FRAMES = 120;
|
||||
// Downscale target. 512px wide JPEG keeps a frame around 20-40KB — cheap
|
||||
// enough to inline a dozen per message as multimodal image parts.
|
||||
const FRAME_WIDTH = 512;
|
||||
const JPEG_QUALITY = 0.65;
|
||||
// Downscale targets. 512px wide JPEG keeps a webcam frame around 20-40KB —
|
||||
// cheap enough to inline a dozen per message as multimodal image parts.
|
||||
// Screen captures keep 1280px so on-screen text stays legible to the model.
|
||||
const CAMERA_FRAME_WIDTH = 512;
|
||||
const SCREEN_FRAME_WIDTH = 1280;
|
||||
const CAMERA_JPEG_QUALITY = 0.65;
|
||||
const SCREEN_JPEG_QUALITY = 0.7;
|
||||
|
||||
interface BufferedFrame {
|
||||
dataUrl: string;
|
||||
|
|
@ -31,58 +39,148 @@ interface BufferedFrame {
|
|||
ts: number;
|
||||
}
|
||||
|
||||
// One capture pipeline: stream → offscreen <video> → canvas JPEG → ring buffer.
|
||||
interface CapturePipe {
|
||||
stream: MediaStream | null;
|
||||
videoEl: HTMLVideoElement | null;
|
||||
canvas: HTMLCanvasElement | null;
|
||||
interval: ReturnType<typeof setInterval> | null;
|
||||
frames: BufferedFrame[];
|
||||
lastCollectTs: number;
|
||||
}
|
||||
|
||||
const emptyPipe = (): CapturePipe => ({
|
||||
stream: null,
|
||||
videoEl: null,
|
||||
canvas: null,
|
||||
interval: null,
|
||||
frames: [],
|
||||
lastCollectTs: 0,
|
||||
});
|
||||
|
||||
function capturePipeFrame(pipe: CapturePipe, width: number, quality: number) {
|
||||
const videoEl = pipe.videoEl;
|
||||
if (!videoEl || videoEl.readyState < 2 || videoEl.videoWidth === 0) return;
|
||||
if (!pipe.canvas) {
|
||||
pipe.canvas = document.createElement('canvas');
|
||||
}
|
||||
const canvas = pipe.canvas;
|
||||
const scale = Math.min(1, width / videoEl.videoWidth);
|
||||
canvas.width = Math.round(videoEl.videoWidth * scale);
|
||||
canvas.height = Math.round(videoEl.videoHeight * scale);
|
||||
const ctx = canvas.getContext('2d');
|
||||
if (!ctx) return;
|
||||
ctx.drawImage(videoEl, 0, 0, canvas.width, canvas.height);
|
||||
const dataUrl = canvas.toDataURL('image/jpeg', quality);
|
||||
// A near-empty data URL means the frame was blank (source still warming up)
|
||||
if (dataUrl.length < 100) return;
|
||||
pipe.frames.push({ dataUrl, capturedAt: new Date().toISOString(), ts: Date.now() });
|
||||
if (pipe.frames.length > MAX_BUFFERED_FRAMES) {
|
||||
pipe.frames.splice(0, pipe.frames.length - MAX_BUFFERED_FRAMES);
|
||||
}
|
||||
}
|
||||
|
||||
function attachPipeSource(pipe: CapturePipe, stream: MediaStream, grab: () => void) {
|
||||
pipe.stream = stream;
|
||||
// Offscreen <video> that feeds the capture canvas; any visible preview
|
||||
// attaches to the same MediaStream separately.
|
||||
const videoEl = document.createElement('video');
|
||||
videoEl.muted = true;
|
||||
videoEl.playsInline = true;
|
||||
videoEl.srcObject = stream;
|
||||
pipe.videoEl = videoEl;
|
||||
videoEl.play().catch(() => {});
|
||||
// First frame as soon as the source delivers data, then steady-state cadence.
|
||||
videoEl.addEventListener('loadeddata', () => grab(), { once: true });
|
||||
pipe.interval = setInterval(grab, CAPTURE_INTERVAL_MS);
|
||||
}
|
||||
|
||||
function teardownPipe(pipe: CapturePipe) {
|
||||
if (pipe.interval) {
|
||||
clearInterval(pipe.interval);
|
||||
pipe.interval = null;
|
||||
}
|
||||
if (pipe.videoEl) {
|
||||
pipe.videoEl.srcObject = null;
|
||||
pipe.videoEl = null;
|
||||
}
|
||||
if (pipe.stream) {
|
||||
pipe.stream.getTracks().forEach((t) => t.stop());
|
||||
pipe.stream = null;
|
||||
}
|
||||
pipe.frames = [];
|
||||
pipe.lastCollectTs = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Drain frames captured since the previous collection, evenly sampled down to
|
||||
* `max` (always keeping the newest). Falls back to the single most recent
|
||||
* frame when nothing new accumulated (rapid-fire messages), so every message
|
||||
* carries at least one frame once the source has warmed up.
|
||||
*/
|
||||
function drainPipe(pipe: CapturePipe, max: number, source: CapturedVideoFrame['source']): CapturedVideoFrame[] {
|
||||
const all = pipe.frames;
|
||||
if (all.length === 0) return [];
|
||||
|
||||
let window_ = all.filter((f) => f.ts > pipe.lastCollectTs);
|
||||
if (window_.length === 0) {
|
||||
window_ = [all[all.length - 1]];
|
||||
}
|
||||
pipe.lastCollectTs = window_[window_.length - 1].ts;
|
||||
|
||||
let sampled: BufferedFrame[];
|
||||
if (window_.length <= max) {
|
||||
sampled = window_;
|
||||
} else {
|
||||
sampled = [];
|
||||
const step = (window_.length - 1) / (max - 1);
|
||||
for (let i = 0; i < max; i++) {
|
||||
sampled.push(window_[Math.round(i * step)]);
|
||||
}
|
||||
}
|
||||
|
||||
return sampled.map((f) => ({
|
||||
data: f.dataUrl.slice(f.dataUrl.indexOf(',') + 1),
|
||||
mediaType: 'image/jpeg',
|
||||
capturedAt: f.capturedAt,
|
||||
dataUrl: f.dataUrl,
|
||||
source,
|
||||
}));
|
||||
}
|
||||
|
||||
export function useVideoMode() {
|
||||
const [state, setState] = useState<VideoModeState>('idle');
|
||||
const [screenState, setScreenState] = useState<ScreenShareState>('idle');
|
||||
const cameraPipeRef = useRef<CapturePipe>(emptyPipe());
|
||||
const screenPipeRef = useRef<CapturePipe>(emptyPipe());
|
||||
// Stable stream refs for preview components (<video srcObject>).
|
||||
const streamRef = useRef<MediaStream | null>(null);
|
||||
const videoElRef = useRef<HTMLVideoElement | null>(null);
|
||||
const canvasRef = useRef<HTMLCanvasElement | null>(null);
|
||||
const intervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
||||
const framesRef = useRef<BufferedFrame[]>([]);
|
||||
const lastCollectTsRef = useRef(0);
|
||||
const screenStreamRef = useRef<MediaStream | null>(null);
|
||||
const stateRef = useRef<VideoModeState>('idle');
|
||||
stateRef.current = state;
|
||||
const screenStateRef = useRef<ScreenShareState>('idle');
|
||||
screenStateRef.current = screenState;
|
||||
|
||||
const captureFrame = useCallback(() => {
|
||||
const videoEl = videoElRef.current;
|
||||
if (!videoEl || videoEl.readyState < 2 || videoEl.videoWidth === 0) return;
|
||||
let canvas = canvasRef.current;
|
||||
if (!canvas) {
|
||||
canvas = document.createElement('canvas');
|
||||
canvasRef.current = canvas;
|
||||
}
|
||||
const scale = FRAME_WIDTH / videoEl.videoWidth;
|
||||
canvas.width = FRAME_WIDTH;
|
||||
canvas.height = Math.round(videoEl.videoHeight * scale);
|
||||
const ctx = canvas.getContext('2d');
|
||||
if (!ctx) return;
|
||||
ctx.drawImage(videoEl, 0, 0, canvas.width, canvas.height);
|
||||
const dataUrl = canvas.toDataURL('image/jpeg', JPEG_QUALITY);
|
||||
// A near-empty data URL means the frame was blank (camera still warming up)
|
||||
if (dataUrl.length < 100) return;
|
||||
const frames = framesRef.current;
|
||||
frames.push({ dataUrl, capturedAt: new Date().toISOString(), ts: Date.now() });
|
||||
if (frames.length > MAX_BUFFERED_FRAMES) {
|
||||
frames.splice(0, frames.length - MAX_BUFFERED_FRAMES);
|
||||
}
|
||||
const captureCameraFrame = useCallback(() => {
|
||||
capturePipeFrame(cameraPipeRef.current, CAMERA_FRAME_WIDTH, CAMERA_JPEG_QUALITY);
|
||||
}, []);
|
||||
|
||||
const captureScreenFrame = useCallback(() => {
|
||||
capturePipeFrame(screenPipeRef.current, SCREEN_FRAME_WIDTH, SCREEN_JPEG_QUALITY);
|
||||
}, []);
|
||||
|
||||
const stopScreenShare = useCallback(() => {
|
||||
teardownPipe(screenPipeRef.current);
|
||||
screenStreamRef.current = null;
|
||||
setScreenState('idle');
|
||||
}, []);
|
||||
|
||||
const stop = useCallback(() => {
|
||||
if (intervalRef.current) {
|
||||
clearInterval(intervalRef.current);
|
||||
intervalRef.current = null;
|
||||
}
|
||||
if (videoElRef.current) {
|
||||
videoElRef.current.srcObject = null;
|
||||
videoElRef.current = null;
|
||||
}
|
||||
if (streamRef.current) {
|
||||
streamRef.current.getTracks().forEach((t) => t.stop());
|
||||
streamRef.current = null;
|
||||
}
|
||||
framesRef.current = [];
|
||||
lastCollectTsRef.current = 0;
|
||||
teardownPipe(cameraPipeRef.current);
|
||||
streamRef.current = null;
|
||||
setState('idle');
|
||||
}, []);
|
||||
stopScreenShare();
|
||||
}, [stopScreenShare]);
|
||||
|
||||
const start = useCallback(async (): Promise<boolean> => {
|
||||
if (stateRef.current !== 'idle') return true;
|
||||
|
|
@ -113,62 +211,64 @@ export function useVideoMode() {
|
|||
}
|
||||
|
||||
streamRef.current = stream;
|
||||
// Offscreen <video> that feeds the capture canvas; the visible preview
|
||||
// attaches to the same MediaStream separately.
|
||||
const videoEl = document.createElement('video');
|
||||
videoEl.muted = true;
|
||||
videoEl.playsInline = true;
|
||||
videoEl.srcObject = stream;
|
||||
videoElRef.current = videoEl;
|
||||
videoEl.play().catch(() => {});
|
||||
// First frame as soon as the camera delivers data, then steady-state cadence.
|
||||
videoEl.addEventListener('loadeddata', () => captureFrame(), { once: true });
|
||||
intervalRef.current = setInterval(captureFrame, CAPTURE_INTERVAL_MS);
|
||||
attachPipeSource(cameraPipeRef.current, stream, captureCameraFrame);
|
||||
setState('live');
|
||||
return true;
|
||||
}, [captureFrame]);
|
||||
}, [captureCameraFrame]);
|
||||
|
||||
/**
|
||||
* Drain frames captured since the previous collection, evenly sampled down
|
||||
* to MAX_FRAMES_PER_MESSAGE (always keeping the newest). Falls back to the
|
||||
* single most recent frame when nothing new accumulated (rapid-fire
|
||||
* messages), so every video-mode message carries at least one frame once
|
||||
* the camera has warmed up.
|
||||
* Share the screen. The main process auto-approves getDisplayMedia with
|
||||
* the primary screen (see setDisplayMediaRequestHandler in main.ts), so
|
||||
* no source picker appears. Returns false if capture couldn't start
|
||||
* (usually the macOS Screen Recording permission).
|
||||
*/
|
||||
const startScreenShare = useCallback(async (): Promise<boolean> => {
|
||||
if (screenStateRef.current !== 'idle') return true;
|
||||
setScreenState('starting');
|
||||
|
||||
// Surfaces the macOS Screen Recording permission state and, on first
|
||||
// use, registers the app in System Settings (same flow meetings use).
|
||||
await window.ipc.invoke('meeting:checkScreenPermission', null).catch(() => null);
|
||||
|
||||
let stream: MediaStream | null = null;
|
||||
try {
|
||||
stream = await navigator.mediaDevices.getDisplayMedia({
|
||||
video: { frameRate: { ideal: 5 } },
|
||||
audio: false,
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('[video] Screen share failed:', err);
|
||||
setScreenState('idle');
|
||||
return false;
|
||||
}
|
||||
|
||||
screenStreamRef.current = stream;
|
||||
// The capture can end outside our UI (display unplugged, OS revokes) —
|
||||
// tear down cleanly so the UI doesn't show a dead share.
|
||||
stream.getVideoTracks()[0]?.addEventListener('ended', () => stopScreenShare(), { once: true });
|
||||
attachPipeSource(screenPipeRef.current, stream, captureScreenFrame);
|
||||
setScreenState('live');
|
||||
return true;
|
||||
}, [captureScreenFrame, stopScreenShare]);
|
||||
|
||||
/**
|
||||
* Drain webcam + screen-share frames buffered since the last send, tagged
|
||||
* by source. Webcam frames come first, then screen frames.
|
||||
*/
|
||||
const collectFrames = useCallback((): CapturedVideoFrame[] => {
|
||||
if (stateRef.current !== 'live') return [];
|
||||
// Grab a frame right now so the message always includes the moment of send.
|
||||
captureFrame();
|
||||
const all = framesRef.current;
|
||||
if (all.length === 0) return [];
|
||||
|
||||
let window_ = all.filter((f) => f.ts > lastCollectTsRef.current);
|
||||
if (window_.length === 0) {
|
||||
window_ = [all[all.length - 1]];
|
||||
captureCameraFrame();
|
||||
const frames = drainPipe(cameraPipeRef.current, MAX_CAMERA_FRAMES_PER_MESSAGE, 'camera');
|
||||
if (screenStateRef.current === 'live') {
|
||||
captureScreenFrame();
|
||||
frames.push(...drainPipe(screenPipeRef.current, MAX_SCREEN_FRAMES_PER_MESSAGE, 'screen'));
|
||||
}
|
||||
lastCollectTsRef.current = window_[window_.length - 1].ts;
|
||||
return frames;
|
||||
}, [captureCameraFrame, captureScreenFrame]);
|
||||
|
||||
let sampled: BufferedFrame[];
|
||||
if (window_.length <= MAX_FRAMES_PER_MESSAGE) {
|
||||
sampled = window_;
|
||||
} else {
|
||||
sampled = [];
|
||||
const step = (window_.length - 1) / (MAX_FRAMES_PER_MESSAGE - 1);
|
||||
for (let i = 0; i < MAX_FRAMES_PER_MESSAGE; i++) {
|
||||
sampled.push(window_[Math.round(i * step)]);
|
||||
}
|
||||
}
|
||||
|
||||
return sampled.map((f) => ({
|
||||
data: f.dataUrl.slice(f.dataUrl.indexOf(',') + 1),
|
||||
mediaType: 'image/jpeg',
|
||||
capturedAt: f.capturedAt,
|
||||
dataUrl: f.dataUrl,
|
||||
}));
|
||||
}, [captureFrame]);
|
||||
|
||||
// Release the camera if the component unmounts with video mode on.
|
||||
// Release the camera/screen if the component unmounts with video mode on.
|
||||
useEffect(() => stop, [stop]);
|
||||
|
||||
return { state, streamRef, start, stop, collectFrames };
|
||||
return { state, screenState, streamRef, screenStreamRef, start, stop, startScreenShare, stopScreenShare, collectFrames };
|
||||
}
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ export function runLogToConversation(log: RunLog): ConversationItem[] {
|
|||
size?: number
|
||||
data?: string
|
||||
mediaType?: string
|
||||
source?: string
|
||||
toolCallId?: string
|
||||
toolName?: string
|
||||
arguments?: unknown
|
||||
|
|
@ -66,7 +67,7 @@ export function runLogToConversation(log: RunLog): ConversationItem[] {
|
|||
})),
|
||||
...imageParts.map((p, index) => ({
|
||||
path: '',
|
||||
filename: `camera-frame-${index + 1}.jpg`,
|
||||
filename: `${p.source === 'screen' ? 'screen' : 'camera'}-frame-${index + 1}.jpg`,
|
||||
mimeType: p.mediaType || 'image/jpeg',
|
||||
thumbnailUrl: `data:${p.mediaType || 'image/jpeg'};base64,${p.data}`,
|
||||
isVideoFrame: true,
|
||||
|
|
|
|||
|
|
@ -392,6 +392,12 @@ How to use the frames:
|
|||
- When the user practices something performative (a pitch, presentation, interview, talk) or asks for delivery feedback, give specific, actionable coaching grounded in what you actually see — cite concrete observations ("in the last few frames you looked down and away from the camera") rather than generic advice. Cover posture, eye contact, facial expressiveness, gesturing, and visible energy.
|
||||
- If they show something to the camera (an object, document, whiteboard), read or describe it and respond accordingly.
|
||||
|
||||
Screen sharing:
|
||||
- The user may also share their screen. Screen-share frames arrive in a separately labeled group after the webcam frames; they show the user's screen, not the user.
|
||||
- When screen frames are present, treat them as the primary subject: the user is usually asking about, or working on, what's visible there. Read the screen carefully — code, documents, error messages, UI state — and help with it concretely.
|
||||
- The LAST screen frame is the most current view of their screen; earlier ones show how it changed while they spoke.
|
||||
- Frames are downscaled captures: small text may be hard to read. If something crucial is illegible, say what you need ("zoom into the error message" / "make that panel bigger") rather than guessing.
|
||||
|
||||
Etiquette:
|
||||
- Do not narrate or list what you see in every response. Bring up visual observations only when relevant to the user's request or clearly worth mentioning.
|
||||
- Never comment on the user's physical appearance, attractiveness, or personal attributes — visual feedback is strictly about delivery, expression, and body language.
|
||||
|
|
@ -961,11 +967,15 @@ export function convertFromMessages(messages: z.infer<typeof Message>[]): ModelM
|
|||
});
|
||||
} else {
|
||||
// New content parts array — collapse text/attachments to text
|
||||
// for the LLM; inline image parts (video-mode webcam frames)
|
||||
// are passed through as real multimodal image parts.
|
||||
// for the LLM; inline image parts (video-mode webcam and
|
||||
// screen-share frames) are passed through as real multimodal
|
||||
// image parts, grouped under labeled text headers so the
|
||||
// model knows which images show the user vs their screen.
|
||||
const textSegments: string[] = userMessageContextPrefix ? [userMessageContextPrefix] : [];
|
||||
const attachmentLines: string[] = [];
|
||||
const imageParts: Array<{ type: "image"; image: string; mediaType: string }> = [];
|
||||
type EncodedImagePart = { type: "image"; image: string; mediaType: string };
|
||||
const cameraParts: EncodedImagePart[] = [];
|
||||
const screenParts: EncodedImagePart[] = [];
|
||||
const frameTimes: string[] = [];
|
||||
|
||||
for (const part of msg.content) {
|
||||
|
|
@ -974,7 +984,8 @@ export function convertFromMessages(messages: z.infer<typeof Message>[]): ModelM
|
|||
const lineStr = part.lineNumber ? ` (line ${part.lineNumber})` : '';
|
||||
attachmentLines.push(`- ${part.filename} (${part.mimeType}${sizeStr}) at ${part.path}${lineStr}`);
|
||||
} else if (part.type === "image") {
|
||||
imageParts.push({ type: "image", image: part.data, mediaType: part.mediaType });
|
||||
const target = part.source === "screen" ? screenParts : cameraParts;
|
||||
target.push({ type: "image", image: part.data, mediaType: part.mediaType });
|
||||
if (part.capturedAt) frameTimes.push(part.capturedAt);
|
||||
} else {
|
||||
textSegments.push(part.text);
|
||||
|
|
@ -989,19 +1000,29 @@ export function convertFromMessages(messages: z.infer<typeof Message>[]): ModelM
|
|||
}
|
||||
}
|
||||
|
||||
if (imageParts.length > 0) {
|
||||
const imageCount = cameraParts.length + screenParts.length;
|
||||
if (imageCount > 0) {
|
||||
const span = frameTimes.length >= 2
|
||||
? ` spanning ${frameTimes[0]} to ${frameTimes[frameTimes.length - 1]}`
|
||||
: frameTimes.length === 1
|
||||
? ` captured at ${frameTimes[0]}`
|
||||
: '';
|
||||
textSegments.push(`[Video mode: ${imageParts.length} live webcam frame${imageParts.length === 1 ? '' : 's'} of the user attached below, oldest to newest,${span ? span + ',' : ''} recorded while they composed this message.]`);
|
||||
const kinds: string[] = [];
|
||||
if (cameraParts.length > 0) kinds.push(`${cameraParts.length} live webcam frame${cameraParts.length === 1 ? '' : 's'} of the user`);
|
||||
if (screenParts.length > 0) kinds.push(`${screenParts.length} frame${screenParts.length === 1 ? '' : 's'} of the user's shared screen`);
|
||||
textSegments.push(`[Video mode: ${kinds.join(' and ')} attached below, each group oldest to newest,${span ? span + ',' : ''} recorded while they composed this message.]`);
|
||||
const content: Array<{ type: "text"; text: string } | EncodedImagePart> = [
|
||||
{ type: "text", text: textSegments.join("\n") },
|
||||
];
|
||||
if (cameraParts.length > 0) {
|
||||
content.push({ type: "text", text: "Webcam frames (oldest to newest):" }, ...cameraParts);
|
||||
}
|
||||
if (screenParts.length > 0) {
|
||||
content.push({ type: "text", text: "Screen-share frames (oldest to newest):" }, ...screenParts);
|
||||
}
|
||||
result.push({
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: textSegments.join("\n") },
|
||||
...imageParts,
|
||||
],
|
||||
content,
|
||||
providerOptions,
|
||||
});
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ export const UserImagePart = z.object({
|
|||
type: z.literal("image"),
|
||||
data: z.string(), // base64-encoded image bytes (no data: prefix)
|
||||
mediaType: z.string(), // MIME type ("image/jpeg")
|
||||
source: z.enum(["camera"]).optional(),
|
||||
source: z.enum(["camera", "screen"]).optional(),
|
||||
capturedAt: z.string().optional(), // ISO timestamp of capture
|
||||
});
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue