+ {videoFrames.length > 0 && (
+
+ {videoFrames.map((frame, index) => (
+
+ ))}
+
+ )}
{imageAttachments.length > 0 && (
{imageAttachments.map((attachment, index) => (
diff --git a/apps/x/apps/renderer/src/components/chat-sidebar.tsx b/apps/x/apps/renderer/src/components/chat-sidebar.tsx
index fdaf9903..8181eed3 100644
--- a/apps/x/apps/renderer/src/components/chat-sidebar.tsx
+++ b/apps/x/apps/renderer/src/components/chat-sidebar.tsx
@@ -194,6 +194,8 @@ interface ChatSidebarProps {
onTtsModeChange?: (mode: 'summary' | 'full') => void
ttsAvatarEnabled?: boolean
onToggleTtsAvatar?: () => void
+ videoEnabled?: boolean
+ onToggleVideo?: () => void
onComposioConnected?: (toolkitSlug: string) => void
}
@@ -260,6 +262,8 @@ export function ChatSidebar({
onTtsModeChange,
ttsAvatarEnabled,
onToggleTtsAvatar,
+ videoEnabled,
+ onToggleVideo,
onComposioConnected,
}: ChatSidebarProps) {
const { state: sidebarState } = useSidebar()
@@ -836,6 +840,8 @@ export function ChatSidebar({
onTtsModeChange={isActive ? onTtsModeChange : undefined}
ttsAvatarEnabled={ttsAvatarEnabled}
onToggleTtsAvatar={isActive ? onToggleTtsAvatar : undefined}
+ videoEnabled={videoEnabled}
+ onToggleVideo={isActive ? onToggleVideo : undefined}
/>
)
diff --git a/apps/x/apps/renderer/src/components/video-preview-overlay.tsx b/apps/x/apps/renderer/src/components/video-preview-overlay.tsx
new file mode 100644
index 00000000..491ab911
--- /dev/null
+++ b/apps/x/apps/renderer/src/components/video-preview-overlay.tsx
@@ -0,0 +1,60 @@
+import { useEffect, useRef } from 'react'
+import { X } from 'lucide-react'
+
+interface VideoPreviewOverlayProps {
+ /** Live camera stream from useVideoMode — attached to the preview element. */
+ streamRef: React.MutableRefObject
+ onTurnOff: () => void
+}
+
+/**
+ * Floating picture-in-picture webcam preview shown while video chat mode is
+ * on. Mirrored like a selfie camera so the user's movements feel natural.
+ * Sits above the composer dock, mirroring the talking-head overlay's corner.
+ */
+export function VideoPreviewOverlay({ streamRef, onTurnOff }: VideoPreviewOverlayProps) {
+ const videoRef = useRef(null)
+
+ useEffect(() => {
+ const videoEl = videoRef.current
+ if (!videoEl) return
+ videoEl.srcObject = streamRef.current
+ videoEl.play().catch(() => {})
+ return () => {
+ videoEl.srcObject = null
+ }
+ }, [streamRef])
+
+ return (
+
+
+
+
+
+
+ Video on
+
+
+ )
+}
diff --git a/apps/x/apps/renderer/src/hooks/useVideoMode.ts b/apps/x/apps/renderer/src/hooks/useVideoMode.ts
new file mode 100644
index 00000000..5d42784a
--- /dev/null
+++ b/apps/x/apps/renderer/src/hooks/useVideoMode.ts
@@ -0,0 +1,174 @@
+import { useCallback, useEffect, useRef, useState } from 'react';
+
+export type VideoModeState = 'idle' | 'starting' | 'live';
+
+export interface CapturedVideoFrame {
+ /** base64-encoded JPEG bytes (no data: prefix) — shape of the UserImagePart wire format */
+ data: string;
+ mediaType: string;
+ capturedAt: string; // ISO timestamp
+ /** data: URL of the same frame, for direct display in the transcript */
+ dataUrl: string;
+}
+
+// Frames are grabbed once per second — dense enough to catch expression and
+// posture changes while the user talks. Per message we attach at most
+// MAX_FRAMES_PER_MESSAGE frames, evenly sampled across the window since the
+// last send, so long monologues don't balloon the request.
+const CAPTURE_INTERVAL_MS = 1000;
+const MAX_FRAMES_PER_MESSAGE = 12;
+// Rolling buffer bound (~2 minutes). The buffer only needs to cover the gap
+// between two sends; anything older is stale context anyway.
+const MAX_BUFFERED_FRAMES = 120;
+// Downscale target. 512px wide JPEG keeps a frame around 20-40KB — cheap
+// enough to inline a dozen per message as multimodal image parts.
+const FRAME_WIDTH = 512;
+const JPEG_QUALITY = 0.65;
+
+interface BufferedFrame {
+ dataUrl: string;
+ capturedAt: string;
+ ts: number;
+}
+
+export function useVideoMode() {
+ const [state, setState] = useState('idle');
+ const streamRef = useRef(null);
+ const videoElRef = useRef(null);
+ const canvasRef = useRef(null);
+ const intervalRef = useRef | null>(null);
+ const framesRef = useRef([]);
+ const lastCollectTsRef = useRef(0);
+ const stateRef = useRef('idle');
+ stateRef.current = state;
+
+ const captureFrame = useCallback(() => {
+ const videoEl = videoElRef.current;
+ if (!videoEl || videoEl.readyState < 2 || videoEl.videoWidth === 0) return;
+ let canvas = canvasRef.current;
+ if (!canvas) {
+ canvas = document.createElement('canvas');
+ canvasRef.current = canvas;
+ }
+ const scale = FRAME_WIDTH / videoEl.videoWidth;
+ canvas.width = FRAME_WIDTH;
+ canvas.height = Math.round(videoEl.videoHeight * scale);
+ const ctx = canvas.getContext('2d');
+ if (!ctx) return;
+ ctx.drawImage(videoEl, 0, 0, canvas.width, canvas.height);
+ const dataUrl = canvas.toDataURL('image/jpeg', JPEG_QUALITY);
+ // A near-empty data URL means the frame was blank (camera still warming up)
+ if (dataUrl.length < 100) return;
+ const frames = framesRef.current;
+ frames.push({ dataUrl, capturedAt: new Date().toISOString(), ts: Date.now() });
+ if (frames.length > MAX_BUFFERED_FRAMES) {
+ frames.splice(0, frames.length - MAX_BUFFERED_FRAMES);
+ }
+ }, []);
+
+ const stop = useCallback(() => {
+ if (intervalRef.current) {
+ clearInterval(intervalRef.current);
+ intervalRef.current = null;
+ }
+ if (videoElRef.current) {
+ videoElRef.current.srcObject = null;
+ videoElRef.current = null;
+ }
+ if (streamRef.current) {
+ streamRef.current.getTracks().forEach((t) => t.stop());
+ streamRef.current = null;
+ }
+ framesRef.current = [];
+ lastCollectTsRef.current = 0;
+ setState('idle');
+ }, []);
+
+ const start = useCallback(async (): Promise => {
+ if (stateRef.current !== 'idle') return true;
+ setState('starting');
+
+ // Settle the macOS TCC camera permission before getUserMedia, same as
+ // voice mode does for the mic — otherwise the first click silently
+ // fails while the native prompt is still up.
+ const access = await window.ipc
+ .invoke('voice:ensureCameraAccess', null)
+ .catch(() => ({ granted: true }));
+ if (!access.granted) {
+ console.error('[video] Camera access denied');
+ setState('idle');
+ return false;
+ }
+
+ let stream: MediaStream | null = null;
+ try {
+ stream = await navigator.mediaDevices.getUserMedia({
+ video: { width: { ideal: 1280 }, height: { ideal: 720 }, facingMode: 'user' },
+ audio: false,
+ });
+ } catch (err) {
+ console.error('[video] Camera access failed:', err);
+ setState('idle');
+ return false;
+ }
+
+ streamRef.current = stream;
+ // Offscreen