added screen share

apps/x/apps/renderer/src/App.tsx

@@ -1221,6 +1221,16 @@ function App() {
     voiceRef.current.setPaused(activeIsProcessing || tts.state !== 'idle')
   }, [videoChatMode, activeIsProcessing, tts.state])
 
+  // Screen sharing (any video mode): frames of the shared screen ride along
+  // with each message next to the webcam frames.
+  const handleToggleScreenShare = useCallback(async () => {
+    if (video.screenState === 'live') {
+      video.stopScreenShare()
+    } else {
+      await video.startScreenShare()
+    }
+  }, [video])
+
   // Enter to submit voice input, Escape to cancel
   useEffect(() => {
     const handleKeyDown = (e: KeyboardEvent) => {
@@ -2717,7 +2727,7 @@ function App() {
           })),
           ...videoFrames.map((frame, index) => ({
             path: '',
-            filename: `camera-frame-${index + 1}.jpg`,
+            filename: `${frame.source}-frame-${index + 1}.jpg`,
             mimeType: frame.mediaType,
             thumbnailUrl: frame.dataUrl,
             isVideoFrame: true,
@@ -2800,7 +2810,7 @@ function App() {
               type: 'image'
               data: string
               mediaType: string
-              source: 'camera'
+              source: 'camera' | 'screen'
               capturedAt: string
             }
 
@@ -2839,7 +2849,7 @@ function App() {
             type: 'image',
             data: frame.data,
             mediaType: frame.mediaType,
-            source: 'camera',
+            source: frame.source,
             capturedAt: frame.capturedAt,
           })
         }
@@ -6599,12 +6609,17 @@ function App() {
                     : undefined
                 }
                 interimText={videoChatMode === 'call' ? voice.interimText : undefined}
+                isScreenSharing={video.screenState === 'live'}
+                onToggleScreenShare={handleToggleScreenShare}
               />
             )}
             {/* Full-screen Meet-style call: user tile + animated mascot tile */}
             {videoChatMode === 'meeting' && (
               <VideoCallView
                 streamRef={video.streamRef}
+                screenStreamRef={video.screenStreamRef}
+                isScreenSharing={video.screenState === 'live'}
+                onToggleScreenShare={handleToggleScreenShare}
                 ttsState={tts.state}
                 getTtsLevel={tts.getLevel}
                 status={

apps/x/apps/renderer/src/components/video-call-view.tsx

@@ -1,5 +1,5 @@
 import { useEffect, useRef, useState } from 'react'
-import { PhoneOff } from 'lucide-react'
+import { MonitorUp, PhoneOff } from 'lucide-react'
 
 import { MascotFaceIcon, TalkingHead } from '@/components/talking-head'
 import type { TTSState } from '@/hooks/useVoiceTTS'
@@ -10,6 +10,10 @@ export type VideoCallStatus = 'listening' | 'thinking' | 'speaking'
 interface VideoCallViewProps {
   /** Live camera stream from useVideoMode — attached to the user's tile. */
   streamRef: React.MutableRefObject<MediaStream | null>
+  /** Live screen-share stream — shown as the presentation tile when sharing. */
+  screenStreamRef: React.MutableRefObject<MediaStream | null>
+  isScreenSharing: boolean
+  onToggleScreenShare: () => void
   ttsState: TTSState
   /** Live TTS output level — drives the mascot's mouth animation. */
   getTtsLevel: () => number
@@ -27,23 +31,17 @@ const STATUS_DISPLAY: Record<VideoCallStatus, { label: string; dotClass: string
   speaking: { label: 'Speaking', dotClass: 'bg-sky-400 animate-pulse' },
 }
 
-/**
- * Full-screen hands-free call: a Meet-style two-tile layout with the user's
- * webcam on one side and the mascot as the other participant. The mascot
- * animates with the assistant's speech; dismissing it swaps in a Meet-style
- * letter avatar ("R"). Live captions run along the bottom.
- */
-export function VideoCallView({
+/** Attach a MediaStream ref to a <video> element for the lifetime of the mount. */
+function StreamVideo({
   streamRef,
-  ttsState,
-  getTtsLevel,
-  status,
-  interimText,
-  assistantCaption,
-  onLeave,
-}: VideoCallViewProps) {
+  mirrored,
+  className,
+}: {
+  streamRef: React.MutableRefObject<MediaStream | null>
+  mirrored?: boolean
+  className?: string
+}) {
   const videoRef = useRef<HTMLVideoElement | null>(null)
-  const [mascotVisible, setMascotVisible] = useState(true)
 
   useEffect(() => {
     const videoEl = videoRef.current
@@ -55,6 +53,38 @@ export function VideoCallView({
     }
   }, [streamRef])
 
+  return (
+    <video
+      ref={videoRef}
+      muted
+      playsInline
+      className={className}
+      style={mirrored ? { transform: 'scaleX(-1)' } : undefined}
+    />
+  )
+}
+
+/**
+ * Full-screen hands-free call: a Meet-style layout with the user's webcam on
+ * one side and the mascot as the other participant. While presenting, the
+ * shared screen becomes the big tile and the participants shrink into a side
+ * rail. The mascot animates with the assistant's speech; dismissing it swaps
+ * in a Meet-style letter avatar ("R"). Live captions run along the bottom.
+ */
+export function VideoCallView({
+  streamRef,
+  screenStreamRef,
+  isScreenSharing,
+  onToggleScreenShare,
+  ttsState,
+  getTtsLevel,
+  status,
+  interimText,
+  assistantCaption,
+  onLeave,
+}: VideoCallViewProps) {
+  const [mascotVisible, setMascotVisible] = useState(true)
+
   const userSpeaking = status === 'listening' && Boolean(interimText)
   const assistantSpeaking = ttsState === 'speaking'
 
@@ -64,58 +94,77 @@ export function VideoCallView({
       ? { who: 'You', text: interimText }
       : null
 
+  const userTile = (
+    <div
+      className={cn(
+        'relative flex items-center justify-center overflow-hidden rounded-2xl bg-neutral-900 transition-shadow',
+        userSpeaking && 'ring-2 ring-green-500/80',
+        isScreenSharing && 'aspect-video w-full'
+      )}
+    >
+      <StreamVideo streamRef={streamRef} mirrored className="h-full w-full object-cover" />
+      <span className="absolute bottom-3 left-3 rounded-md bg-black/50 px-2 py-0.5 text-sm text-white">
+        You
+      </span>
+    </div>
+  )
+
+  const assistantTile = (
+    <div
+      className={cn(
+        'group relative flex items-center justify-center overflow-hidden rounded-2xl bg-neutral-900 transition-shadow',
+        assistantSpeaking && 'ring-2 ring-sky-400/80',
+        isScreenSharing && 'aspect-video w-full'
+      )}
+    >
+      {mascotVisible ? (
+        <TalkingHead ttsState={ttsState} getLevel={getTtsLevel} size={isScreenSharing ? 96 : 220} />
+      ) : (
+        <span
+          className={cn(
+            'flex items-center justify-center rounded-full bg-sky-600 font-medium text-white',
+            isScreenSharing ? 'h-16 w-16 text-3xl' : 'h-40 w-40 text-7xl'
+          )}
+          aria-label="Rowboat"
+        >
+          R
+        </span>
+      )}
+      <span className="absolute bottom-3 left-3 rounded-md bg-black/50 px-2 py-0.5 text-sm text-white">
+        Rowboat
+      </span>
+      <button
+        type="button"
+        onClick={() => setMascotVisible((v) => !v)}
+        className="absolute right-3 top-3 rounded-md bg-black/50 px-2 py-1 text-xs text-white/80 opacity-0 transition-opacity hover:text-white group-hover:opacity-100"
+      >
+        {mascotVisible ? 'Hide mascot' : 'Show mascot'}
+      </button>
+    </div>
+  )
+
   return (
     <div className="fixed inset-0 z-[100] flex flex-col bg-neutral-950">
-      {/* Participant tiles */}
-      <div className="grid min-h-0 flex-1 grid-cols-1 gap-3 p-4 pb-2 md:grid-cols-2">
-        {/* User */}
-        <div
-          className={cn(
-            'relative flex items-center justify-center overflow-hidden rounded-2xl bg-neutral-900 transition-shadow',
-            userSpeaking && 'ring-2 ring-green-500/80'
-          )}
-        >
-          <video
-            ref={videoRef}
-            muted
-            playsInline
-            className="h-full w-full object-cover"
-            style={{ transform: 'scaleX(-1)' }}
-          />
-          <span className="absolute bottom-3 left-3 rounded-md bg-black/50 px-2 py-0.5 text-sm text-white">
-            You
-          </span>
-        </div>
-
-        {/* Assistant */}
-        <div
-          className={cn(
-            'group relative flex items-center justify-center overflow-hidden rounded-2xl bg-neutral-900 transition-shadow',
-            assistantSpeaking && 'ring-2 ring-sky-400/80'
-          )}
-        >
-          {mascotVisible ? (
-            <TalkingHead ttsState={ttsState} getLevel={getTtsLevel} size={220} />
-          ) : (
-            <span
-              className="flex h-40 w-40 items-center justify-center rounded-full bg-sky-600 text-7xl font-medium text-white"
-              aria-label="Rowboat"
-            >
-              R
+      {/* Participant tiles — Meet-style presentation layout while sharing */}
+      {isScreenSharing ? (
+        <div className="flex min-h-0 flex-1 gap-3 p-4 pb-2">
+          <div className="relative flex flex-1 items-center justify-center overflow-hidden rounded-2xl bg-neutral-900">
+            <StreamVideo streamRef={screenStreamRef} className="h-full w-full object-contain" />
+            <span className="absolute bottom-3 left-3 rounded-md bg-black/50 px-2 py-0.5 text-sm text-white">
+              Your screen
             </span>
-          )}
-          <span className="absolute bottom-3 left-3 rounded-md bg-black/50 px-2 py-0.5 text-sm text-white">
-            Rowboat
-          </span>
-          <button
-            type="button"
-            onClick={() => setMascotVisible((v) => !v)}
-            className="absolute right-3 top-3 rounded-md bg-black/50 px-2 py-1 text-xs text-white/80 opacity-0 transition-opacity hover:text-white group-hover:opacity-100"
-          >
-            {mascotVisible ? 'Hide mascot' : 'Show mascot'}
-          </button>
+          </div>
+          <div className="flex w-52 shrink-0 flex-col gap-3">
+            {userTile}
+            {assistantTile}
+          </div>
         </div>
-      </div>
+      ) : (
+        <div className="grid min-h-0 flex-1 grid-cols-1 gap-3 p-4 pb-2 md:grid-cols-2">
+          {userTile}
+          {assistantTile}
+        </div>
+      )}
 
       {/* Captions */}
       <div className="flex h-14 items-center justify-center px-6">
@@ -135,10 +184,22 @@ export function VideoCallView({
         </span>
         <button
           type="button"
-          onClick={() => setMascotVisible((v) => !v)}
+          onClick={onToggleScreenShare}
           className={cn(
-            'relative flex h-10 w-10 items-center justify-center rounded-full bg-neutral-800 text-white/90 transition-colors hover:bg-neutral-700',
+            'flex h-10 w-10 items-center justify-center rounded-full transition-colors',
+            isScreenSharing
+              ? 'bg-sky-600 text-white hover:bg-sky-500'
+              : 'bg-neutral-800 text-white/90 hover:bg-neutral-700'
           )}
+          aria-label={isScreenSharing ? 'Stop presenting' : 'Present your screen'}
+          title={isScreenSharing ? 'Stop presenting' : 'Present your screen'}
+        >
+          <MonitorUp className="h-5 w-5" />
+        </button>
+        <button
+          type="button"
+          onClick={() => setMascotVisible((v) => !v)}
+          className="relative flex h-10 w-10 items-center justify-center rounded-full bg-neutral-800 text-white/90 transition-colors hover:bg-neutral-700"
           aria-label={mascotVisible ? 'Hide mascot' : 'Show mascot'}
         >
           <MascotFaceIcon />

apps/x/apps/renderer/src/components/video-preview-overlay.tsx

@@ -1,5 +1,5 @@
 import { useEffect, useRef } from 'react'
-import { X } from 'lucide-react'
+import { MonitorUp, X } from 'lucide-react'
 
 interface VideoPreviewOverlayProps {
   /** Live camera stream from useVideoMode — attached to the preview element. */
@@ -9,6 +9,8 @@ interface VideoPreviewOverlayProps {
   callStatus?: 'listening' | 'thinking' | 'speaking'
   /** Hands-free call mode: live transcript of the in-progress utterance. */
   interimText?: string
+  isScreenSharing?: boolean
+  onToggleScreenShare?: () => void
 }
 
 const CALL_STATUS_DISPLAY: Record<NonNullable<VideoPreviewOverlayProps['callStatus']>, { label: string; dotClass: string }> = {
@@ -22,7 +24,7 @@ const CALL_STATUS_DISPLAY: Record<NonNullable<VideoPreviewOverlayProps['callStat
  * on. Mirrored like a selfie camera so the user's movements feel natural.
  * Sits above the composer dock, mirroring the talking-head overlay's corner.
  */
-export function VideoPreviewOverlay({ streamRef, onTurnOff, callStatus, interimText }: VideoPreviewOverlayProps) {
+export function VideoPreviewOverlay({ streamRef, onTurnOff, callStatus, interimText, isScreenSharing, onToggleScreenShare }: VideoPreviewOverlayProps) {
   const videoRef = useRef<HTMLVideoElement | null>(null)
 
   useEffect(() => {
@@ -54,6 +56,21 @@ export function VideoPreviewOverlay({ streamRef, onTurnOff, callStatus, interimT
       >
         <X className="h-3 w-3" />
       </button>
+      {onToggleScreenShare && (
+        <button
+          type="button"
+          onClick={onToggleScreenShare}
+          className={`absolute -left-1.5 -top-1.5 z-10 flex h-5 w-5 items-center justify-center rounded-full border shadow-sm transition-opacity ${
+            isScreenSharing
+              ? 'border-sky-500 bg-sky-600 text-white opacity-100'
+              : 'border-border bg-background text-muted-foreground opacity-0 hover:text-foreground group-hover:opacity-100'
+          }`}
+          aria-label={isScreenSharing ? 'Stop sharing screen' : 'Share your screen'}
+          title={isScreenSharing ? 'Stop sharing screen' : 'Share your screen'}
+        >
+          <MonitorUp className="h-3 w-3" />
+        </button>
+      )}
       <video
         ref={videoRef}
         muted

apps/x/apps/renderer/src/hooks/useVideoMode.ts

@@ -1,6 +1,7 @@
 import { useCallback, useEffect, useRef, useState } from 'react';
 
 export type VideoModeState = 'idle' | 'starting' | 'live';
+export type ScreenShareState = 'idle' | 'starting' | 'live';
 
 export interface CapturedVideoFrame {
     /** base64-encoded JPEG bytes (no data: prefix) — shape of the UserImagePart wire format */
@@ -9,21 +10,28 @@ export interface CapturedVideoFrame {
     capturedAt: string; // ISO timestamp
     /** data: URL of the same frame, for direct display in the transcript */
     dataUrl: string;
+    source: 'camera' | 'screen';
 }
 
 // Frames are grabbed once per second — dense enough to catch expression and
 // posture changes while the user talks. Per message we attach at most
-// MAX_FRAMES_PER_MESSAGE frames, evenly sampled across the window since the
+// MAX_*_FRAMES_PER_MESSAGE frames, evenly sampled across the window since the
 // last send, so long monologues don't balloon the request.
 const CAPTURE_INTERVAL_MS = 1000;
-const MAX_FRAMES_PER_MESSAGE = 12;
+const MAX_CAMERA_FRAMES_PER_MESSAGE = 12;
+// Screen frames are ~4x the resolution (and tokens) of camera frames, and the
+// latest view matters far more than the trajectory — keep the cap small.
+const MAX_SCREEN_FRAMES_PER_MESSAGE = 4;
 // Rolling buffer bound (~2 minutes). The buffer only needs to cover the gap
 // between two sends; anything older is stale context anyway.
 const MAX_BUFFERED_FRAMES = 120;
-// Downscale target. 512px wide JPEG keeps a frame around 20-40KB — cheap
-// enough to inline a dozen per message as multimodal image parts.
-const FRAME_WIDTH = 512;
-const JPEG_QUALITY = 0.65;
+// Downscale targets. 512px wide JPEG keeps a webcam frame around 20-40KB —
+// cheap enough to inline a dozen per message as multimodal image parts.
+// Screen captures keep 1280px so on-screen text stays legible to the model.
+const CAMERA_FRAME_WIDTH = 512;
+const SCREEN_FRAME_WIDTH = 1280;
+const CAMERA_JPEG_QUALITY = 0.65;
+const SCREEN_JPEG_QUALITY = 0.7;
 
 interface BufferedFrame {
     dataUrl: string;
@@ -31,58 +39,148 @@ interface BufferedFrame {
     ts: number;
 }
 
+// One capture pipeline: stream → offscreen <video> → canvas JPEG → ring buffer.
+interface CapturePipe {
+    stream: MediaStream | null;
+    videoEl: HTMLVideoElement | null;
+    canvas: HTMLCanvasElement | null;
+    interval: ReturnType<typeof setInterval> | null;
+    frames: BufferedFrame[];
+    lastCollectTs: number;
+}
+
+const emptyPipe = (): CapturePipe => ({
+    stream: null,
+    videoEl: null,
+    canvas: null,
+    interval: null,
+    frames: [],
+    lastCollectTs: 0,
+});
+
+function capturePipeFrame(pipe: CapturePipe, width: number, quality: number) {
+    const videoEl = pipe.videoEl;
+    if (!videoEl || videoEl.readyState < 2 || videoEl.videoWidth === 0) return;
+    if (!pipe.canvas) {
+        pipe.canvas = document.createElement('canvas');
+    }
+    const canvas = pipe.canvas;
+    const scale = Math.min(1, width / videoEl.videoWidth);
+    canvas.width = Math.round(videoEl.videoWidth * scale);
+    canvas.height = Math.round(videoEl.videoHeight * scale);
+    const ctx = canvas.getContext('2d');
+    if (!ctx) return;
+    ctx.drawImage(videoEl, 0, 0, canvas.width, canvas.height);
+    const dataUrl = canvas.toDataURL('image/jpeg', quality);
+    // A near-empty data URL means the frame was blank (source still warming up)
+    if (dataUrl.length < 100) return;
+    pipe.frames.push({ dataUrl, capturedAt: new Date().toISOString(), ts: Date.now() });
+    if (pipe.frames.length > MAX_BUFFERED_FRAMES) {
+        pipe.frames.splice(0, pipe.frames.length - MAX_BUFFERED_FRAMES);
+    }
+}
+
+function attachPipeSource(pipe: CapturePipe, stream: MediaStream, grab: () => void) {
+    pipe.stream = stream;
+    // Offscreen <video> that feeds the capture canvas; any visible preview
+    // attaches to the same MediaStream separately.
+    const videoEl = document.createElement('video');
+    videoEl.muted = true;
+    videoEl.playsInline = true;
+    videoEl.srcObject = stream;
+    pipe.videoEl = videoEl;
+    videoEl.play().catch(() => {});
+    // First frame as soon as the source delivers data, then steady-state cadence.
+    videoEl.addEventListener('loadeddata', () => grab(), { once: true });
+    pipe.interval = setInterval(grab, CAPTURE_INTERVAL_MS);
+}
+
+function teardownPipe(pipe: CapturePipe) {
+    if (pipe.interval) {
+        clearInterval(pipe.interval);
+        pipe.interval = null;
+    }
+    if (pipe.videoEl) {
+        pipe.videoEl.srcObject = null;
+        pipe.videoEl = null;
+    }
+    if (pipe.stream) {
+        pipe.stream.getTracks().forEach((t) => t.stop());
+        pipe.stream = null;
+    }
+    pipe.frames = [];
+    pipe.lastCollectTs = 0;
+}
+
+/**
+ * Drain frames captured since the previous collection, evenly sampled down to
+ * `max` (always keeping the newest). Falls back to the single most recent
+ * frame when nothing new accumulated (rapid-fire messages), so every message
+ * carries at least one frame once the source has warmed up.
+ */
+function drainPipe(pipe: CapturePipe, max: number, source: CapturedVideoFrame['source']): CapturedVideoFrame[] {
+    const all = pipe.frames;
+    if (all.length === 0) return [];
+
+    let window_ = all.filter((f) => f.ts > pipe.lastCollectTs);
+    if (window_.length === 0) {
+        window_ = [all[all.length - 1]];
+    }
+    pipe.lastCollectTs = window_[window_.length - 1].ts;
+
+    let sampled: BufferedFrame[];
+    if (window_.length <= max) {
+        sampled = window_;
+    } else {
+        sampled = [];
+        const step = (window_.length - 1) / (max - 1);
+        for (let i = 0; i < max; i++) {
+            sampled.push(window_[Math.round(i * step)]);
+        }
+    }
+
+    return sampled.map((f) => ({
+        data: f.dataUrl.slice(f.dataUrl.indexOf(',') + 1),
+        mediaType: 'image/jpeg',
+        capturedAt: f.capturedAt,
+        dataUrl: f.dataUrl,
+        source,
+    }));
+}
+
 export function useVideoMode() {
     const [state, setState] = useState<VideoModeState>('idle');
+    const [screenState, setScreenState] = useState<ScreenShareState>('idle');
+    const cameraPipeRef = useRef<CapturePipe>(emptyPipe());
+    const screenPipeRef = useRef<CapturePipe>(emptyPipe());
+    // Stable stream refs for preview components (<video srcObject>).
     const streamRef = useRef<MediaStream | null>(null);
-    const videoElRef = useRef<HTMLVideoElement | null>(null);
-    const canvasRef = useRef<HTMLCanvasElement | null>(null);
-    const intervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
-    const framesRef = useRef<BufferedFrame[]>([]);
-    const lastCollectTsRef = useRef(0);
+    const screenStreamRef = useRef<MediaStream | null>(null);
     const stateRef = useRef<VideoModeState>('idle');
     stateRef.current = state;
+    const screenStateRef = useRef<ScreenShareState>('idle');
+    screenStateRef.current = screenState;
 
-    const captureFrame = useCallback(() => {
-        const videoEl = videoElRef.current;
-        if (!videoEl || videoEl.readyState < 2 || videoEl.videoWidth === 0) return;
-        let canvas = canvasRef.current;
-        if (!canvas) {
-            canvas = document.createElement('canvas');
-            canvasRef.current = canvas;
-        }
-        const scale = FRAME_WIDTH / videoEl.videoWidth;
-        canvas.width = FRAME_WIDTH;
-        canvas.height = Math.round(videoEl.videoHeight * scale);
-        const ctx = canvas.getContext('2d');
-        if (!ctx) return;
-        ctx.drawImage(videoEl, 0, 0, canvas.width, canvas.height);
-        const dataUrl = canvas.toDataURL('image/jpeg', JPEG_QUALITY);
-        // A near-empty data URL means the frame was blank (camera still warming up)
-        if (dataUrl.length < 100) return;
-        const frames = framesRef.current;
-        frames.push({ dataUrl, capturedAt: new Date().toISOString(), ts: Date.now() });
-        if (frames.length > MAX_BUFFERED_FRAMES) {
-            frames.splice(0, frames.length - MAX_BUFFERED_FRAMES);
-        }
+    const captureCameraFrame = useCallback(() => {
+        capturePipeFrame(cameraPipeRef.current, CAMERA_FRAME_WIDTH, CAMERA_JPEG_QUALITY);
+    }, []);
+
+    const captureScreenFrame = useCallback(() => {
+        capturePipeFrame(screenPipeRef.current, SCREEN_FRAME_WIDTH, SCREEN_JPEG_QUALITY);
+    }, []);
+
+    const stopScreenShare = useCallback(() => {
+        teardownPipe(screenPipeRef.current);
+        screenStreamRef.current = null;
+        setScreenState('idle');
     }, []);
 
     const stop = useCallback(() => {
-        if (intervalRef.current) {
-            clearInterval(intervalRef.current);
-            intervalRef.current = null;
-        }
-        if (videoElRef.current) {
-            videoElRef.current.srcObject = null;
-            videoElRef.current = null;
-        }
-        if (streamRef.current) {
-            streamRef.current.getTracks().forEach((t) => t.stop());
-            streamRef.current = null;
-        }
-        framesRef.current = [];
-        lastCollectTsRef.current = 0;
+        teardownPipe(cameraPipeRef.current);
+        streamRef.current = null;
         setState('idle');
-    }, []);
+        stopScreenShare();
+    }, [stopScreenShare]);
 
     const start = useCallback(async (): Promise<boolean> => {
         if (stateRef.current !== 'idle') return true;
@@ -113,62 +211,64 @@ export function useVideoMode() {
         }
 
         streamRef.current = stream;
-        // Offscreen <video> that feeds the capture canvas; the visible preview
-        // attaches to the same MediaStream separately.
-        const videoEl = document.createElement('video');
-        videoEl.muted = true;
-        videoEl.playsInline = true;
-        videoEl.srcObject = stream;
-        videoElRef.current = videoEl;
-        videoEl.play().catch(() => {});
-        // First frame as soon as the camera delivers data, then steady-state cadence.
-        videoEl.addEventListener('loadeddata', () => captureFrame(), { once: true });
-        intervalRef.current = setInterval(captureFrame, CAPTURE_INTERVAL_MS);
+        attachPipeSource(cameraPipeRef.current, stream, captureCameraFrame);
         setState('live');
         return true;
-    }, [captureFrame]);
+    }, [captureCameraFrame]);
 
     /**
-     * Drain frames captured since the previous collection, evenly sampled down
-     * to MAX_FRAMES_PER_MESSAGE (always keeping the newest). Falls back to the
-     * single most recent frame when nothing new accumulated (rapid-fire
-     * messages), so every video-mode message carries at least one frame once
-     * the camera has warmed up.
+     * Share the screen. The main process auto-approves getDisplayMedia with
+     * the primary screen (see setDisplayMediaRequestHandler in main.ts), so
+     * no source picker appears. Returns false if capture couldn't start
+     * (usually the macOS Screen Recording permission).
+     */
+    const startScreenShare = useCallback(async (): Promise<boolean> => {
+        if (screenStateRef.current !== 'idle') return true;
+        setScreenState('starting');
+
+        // Surfaces the macOS Screen Recording permission state and, on first
+        // use, registers the app in System Settings (same flow meetings use).
+        await window.ipc.invoke('meeting:checkScreenPermission', null).catch(() => null);
+
+        let stream: MediaStream | null = null;
+        try {
+            stream = await navigator.mediaDevices.getDisplayMedia({
+                video: { frameRate: { ideal: 5 } },
+                audio: false,
+            });
+        } catch (err) {
+            console.error('[video] Screen share failed:', err);
+            setScreenState('idle');
+            return false;
+        }
+
+        screenStreamRef.current = stream;
+        // The capture can end outside our UI (display unplugged, OS revokes) —
+        // tear down cleanly so the UI doesn't show a dead share.
+        stream.getVideoTracks()[0]?.addEventListener('ended', () => stopScreenShare(), { once: true });
+        attachPipeSource(screenPipeRef.current, stream, captureScreenFrame);
+        setScreenState('live');
+        return true;
+    }, [captureScreenFrame, stopScreenShare]);
+
+    /**
+     * Drain webcam + screen-share frames buffered since the last send, tagged
+     * by source. Webcam frames come first, then screen frames.
      */
     const collectFrames = useCallback((): CapturedVideoFrame[] => {
         if (stateRef.current !== 'live') return [];
         // Grab a frame right now so the message always includes the moment of send.
-        captureFrame();
-        const all = framesRef.current;
-        if (all.length === 0) return [];
-
-        let window_ = all.filter((f) => f.ts > lastCollectTsRef.current);
-        if (window_.length === 0) {
-            window_ = [all[all.length - 1]];
+        captureCameraFrame();
+        const frames = drainPipe(cameraPipeRef.current, MAX_CAMERA_FRAMES_PER_MESSAGE, 'camera');
+        if (screenStateRef.current === 'live') {
+            captureScreenFrame();
+            frames.push(...drainPipe(screenPipeRef.current, MAX_SCREEN_FRAMES_PER_MESSAGE, 'screen'));
         }
-        lastCollectTsRef.current = window_[window_.length - 1].ts;
+        return frames;
+    }, [captureCameraFrame, captureScreenFrame]);
 
-        let sampled: BufferedFrame[];
-        if (window_.length <= MAX_FRAMES_PER_MESSAGE) {
-            sampled = window_;
-        } else {
-            sampled = [];
-            const step = (window_.length - 1) / (MAX_FRAMES_PER_MESSAGE - 1);
-            for (let i = 0; i < MAX_FRAMES_PER_MESSAGE; i++) {
-                sampled.push(window_[Math.round(i * step)]);
-            }
-        }
-
-        return sampled.map((f) => ({
-            data: f.dataUrl.slice(f.dataUrl.indexOf(',') + 1),
-            mediaType: 'image/jpeg',
-            capturedAt: f.capturedAt,
-            dataUrl: f.dataUrl,
-        }));
-    }, [captureFrame]);
-
-    // Release the camera if the component unmounts with video mode on.
+    // Release the camera/screen if the component unmounts with video mode on.
     useEffect(() => stop, [stop]);
 
-    return { state, streamRef, start, stop, collectFrames };
+    return { state, screenState, streamRef, screenStreamRef, start, stop, startScreenShare, stopScreenShare, collectFrames };
 }

apps/x/apps/renderer/src/lib/run-to-conversation.ts

@@ -43,6 +43,7 @@ export function runLogToConversation(log: RunLog): ConversationItem[] {
             size?: number
             data?: string
             mediaType?: string
+            source?: string
             toolCallId?: string
             toolName?: string
             arguments?: unknown
@@ -66,7 +67,7 @@ export function runLogToConversation(log: RunLog): ConversationItem[] {
               })),
               ...imageParts.map((p, index) => ({
                 path: '',
-                filename: `camera-frame-${index + 1}.jpg`,
+                filename: `${p.source === 'screen' ? 'screen' : 'camera'}-frame-${index + 1}.jpg`,
                 mimeType: p.mediaType || 'image/jpeg',
                 thumbnailUrl: `data:${p.mediaType || 'image/jpeg'};base64,${p.data}`,
                 isVideoFrame: true,