initial commit of video mode

apps/x/apps/main/src/ipc.ts

@@ -1713,6 +1713,21 @@ export function setupIpcHandlers() {
         return { granted: false };
       }
     },
+    'voice:ensureCameraAccess': async () => {
+      if (process.platform !== 'darwin') return { granted: true };
+      const status = systemPreferences.getMediaAccessStatus('camera');
+      console.log('[video] Camera permission status:', status);
+      if (status === 'granted') return { granted: true };
+      // Same flow as the microphone: settle the native TCC prompt before the
+      // renderer's getUserMedia so the first video click doesn't silently fail.
+      try {
+        const granted = await systemPreferences.askForMediaAccess('camera');
+        console.log('[video] Camera permission after prompt:', granted);
+        return { granted };
+      } catch {
+        return { granted: false };
+      }
+    },
     // Live-note handlers
     'live-note:run': async (_event, args) => {
       const result = await runLiveNoteAgent(args.filePath, 'manual', args.context);

apps/x/apps/renderer/src/App.tsx

@@ -118,8 +118,10 @@ import { AgentScheduleConfig } from '@x/shared/dist/agent-schedule.js'
 import { AgentScheduleState } from '@x/shared/dist/agent-schedule-state.js'
 import { toast } from "sonner"
 import { useVoiceMode } from '@/hooks/useVoiceMode'
+import { useVideoMode } from '@/hooks/useVideoMode'
 import { useVoiceTTS } from '@/hooks/useVoiceTTS'
 import { TalkingHeadOverlay } from '@/components/talking-head'
+import { VideoPreviewOverlay } from '@/components/video-preview-overlay'
 import { ProductTour, type TourNavTarget } from '@/components/product-tour'
 import { useMeetingTranscription, type CalendarEventMeta } from '@/hooks/useMeetingTranscription'
 import { useAnalyticsIdentity } from '@/hooks/useAnalyticsIdentity'
@@ -1010,6 +1012,23 @@ function App() {
   const voiceRef = useRef(voice)
   voiceRef.current = voice
 
+  // Video chat mode: while on, the webcam runs and frames are attached to
+  // every outgoing message so the assistant can see the user.
+  const video = useVideoMode()
+  const [videoEnabled, setVideoEnabled] = useState(false)
+  const videoEnabledRef = useRef(false)
+  const handleToggleVideo = useCallback(async () => {
+    if (videoEnabledRef.current) {
+      video.stop()
+      videoEnabledRef.current = false
+      setVideoEnabled(false)
+    } else {
+      const ok = await video.start()
+      videoEnabledRef.current = ok
+      setVideoEnabled(ok)
+    }
+  }, [video])
+
   const handleToggleMeetingRef = useRef<(() => void) | undefined>(undefined)
   const meetingTranscription = useMeetingTranscription(() => {
     handleToggleMeetingRef.current?.()
@@ -2608,15 +2627,28 @@ function App() {
 
     setMessage('')
 
+    // Video chat mode: drain the webcam frames buffered since the last send
+    // so they ride along with this message as inline image parts.
+    const videoFrames = videoEnabledRef.current ? video.collectFrames() : []
+
     const userMessageId = `user-${Date.now()}`
-    const displayAttachments: ChatMessage['attachments'] = hasAttachments
-      ? stagedAttachments.map((attachment) => ({
-          path: attachment.path,
-          filename: attachment.filename,
-          mimeType: attachment.mimeType,
-          size: attachment.size,
-          thumbnailUrl: attachment.thumbnailUrl,
-        }))
+    const displayAttachments: ChatMessage['attachments'] = hasAttachments || videoFrames.length > 0
+      ? [
+          ...stagedAttachments.map((attachment) => ({
+            path: attachment.path,
+            filename: attachment.filename,
+            mimeType: attachment.mimeType,
+            size: attachment.size,
+            thumbnailUrl: attachment.thumbnailUrl,
+          })),
+          ...videoFrames.map((frame, index) => ({
+            path: '',
+            filename: `camera-frame-${index + 1}.jpg`,
+            mimeType: frame.mediaType,
+            thumbnailUrl: frame.dataUrl,
+            isVideoFrame: true,
+          })),
+        ]
       : undefined
     setConversation((prev) => [...prev, {
       id: userMessageId,
@@ -2668,6 +2700,7 @@ function App() {
               ...(ttsEnabledRef.current ? { voiceOutput: ttsModeRef.current } : {}),
               ...(searchEnabled ? { searchEnabled: true } : {}),
               ...(codeMode ? { codeMode } : {}),
+              ...(videoEnabledRef.current ? { videoMode: true } : {}),
             },
           },
         },
@@ -2678,7 +2711,7 @@ function App() {
         middlePane: middlePane ?? { kind: 'empty' as const },
       })
 
-      if (hasAttachments || hasMentions) {
+      if (hasAttachments || hasMentions || videoFrames.length > 0) {
         type ContentPart =
           | { type: 'text'; text: string }
           | {
@@ -2689,6 +2722,13 @@ function App() {
               size?: number
               lineNumber?: number
             }
+          | {
+              type: 'image'
+              data: string
+              mediaType: string
+              source: 'camera'
+              capturedAt: string
+            }
 
         const contentParts: ContentPart[] = []
 
@@ -2720,6 +2760,16 @@ function App() {
           titleSource = stagedAttachments[0]?.filename ?? mentions?.[0]?.displayName ?? mentions?.[0]?.path ?? ''
         }
 
+        for (const frame of videoFrames) {
+          contentParts.push({
+            type: 'image',
+            data: frame.data,
+            mediaType: frame.mediaType,
+            source: 'camera',
+            capturedAt: frame.capturedAt,
+          })
+        }
+
         const middlePaneContext = await buildMiddlePaneContext()
         await window.ipc.invoke('sessions:sendMessage', {
           sessionId: currentRunId,
@@ -6337,6 +6387,8 @@ function App() {
                             onTtsModeChange={isActive ? handleTtsModeChange : undefined}
                             ttsAvatarEnabled={ttsAvatarEnabled}
                             onToggleTtsAvatar={isActive ? handleToggleTtsAvatar : undefined}
+                            videoEnabled={videoEnabled}
+                            onToggleVideo={isActive ? handleToggleVideo : undefined}
                           />
                         </div>
                       )
@@ -6450,9 +6502,15 @@ function App() {
                 onTtsModeChange={handleTtsModeChange}
                 ttsAvatarEnabled={ttsAvatarEnabled}
                 onToggleTtsAvatar={handleToggleTtsAvatar}
+                videoEnabled={videoEnabled}
+                onToggleVideo={handleToggleVideo}
                 onComposioConnected={handleComposioConnected}
               />
             )}
+            {/* Webcam PiP preview while video chat mode is on */}
+            {videoEnabled && (
+              <VideoPreviewOverlay streamRef={video.streamRef} onTurnOff={handleToggleVideo} />
+            )}
             {/* Talking head hovers over the active view while avatar voice mode is
                 on (hidden during the tour, which shows its own mascot) */}
             {ttsAvatarEnabled && !tourActive && (

apps/x/apps/renderer/src/components/chat-input-with-mentions.tsx

@@ -25,6 +25,7 @@ import {
   ShieldCheck,
   Square,
   Terminal,
+  Video,
   X,
 } from 'lucide-react'
 
@@ -238,6 +239,9 @@ interface ChatInputInnerProps {
   onTtsModeChange?: (mode: 'summary' | 'full') => void
   ttsAvatarEnabled?: boolean
   onToggleTtsAvatar?: () => void
+  /** Video chat mode: live webcam frames are attached to each message. */
+  videoEnabled?: boolean
+  onToggleVideo?: () => void
   /** Fired when the user picks a different model in the dropdown (only when no run exists yet). */
   onSelectedModelChange?: (model: SelectedModel | null) => void
   /** Work directory for this chat (per-chat). Null when none is set. */
@@ -278,6 +282,8 @@ function ChatInputInner({
   onTtsModeChange,
   ttsAvatarEnabled,
   onToggleTtsAvatar,
+  videoEnabled,
+  onToggleVideo,
   onSelectedModelChange,
   workDir = null,
   onWorkDirChange,
@@ -1350,6 +1356,33 @@ function ChatInputInner({
             </TooltipContent>
           </Tooltip>
         )}
+        {onToggleVideo && (
+          <Tooltip delayDuration={CHAT_INPUT_TOOLTIP_DELAY_MS}>
+            <TooltipTrigger asChild>
+              <button
+                type="button"
+                onClick={onToggleVideo}
+                className={cn(
+                  'relative flex h-7 w-7 shrink-0 items-center justify-center rounded-full transition-colors',
+                  videoEnabled
+                    ? 'text-red-500 hover:bg-muted'
+                    : 'text-muted-foreground hover:bg-muted hover:text-foreground'
+                )}
+                aria-label={videoEnabled ? 'Turn off video chat' : 'Turn on video chat'}
+              >
+                <Video className="h-4 w-4" />
+                {!videoEnabled && (
+                  <span className="absolute inset-0 flex items-center justify-center pointer-events-none">
+                    <span className="block h-[1.5px] w-5 -rotate-45 rounded-full bg-muted-foreground" />
+                  </span>
+                )}
+              </button>
+            </TooltipTrigger>
+            <TooltipContent side="top">
+              {videoEnabled ? 'Video chat on — frames are shared with the assistant' : 'Video chat off'}
+            </TooltipContent>
+          </Tooltip>
+        )}
         {voiceAvailable && onStartRecording && (
           <button
             type="button"
@@ -1526,6 +1559,8 @@ export interface ChatInputWithMentionsProps {
   onTtsModeChange?: (mode: 'summary' | 'full') => void
   ttsAvatarEnabled?: boolean
   onToggleTtsAvatar?: () => void
+  videoEnabled?: boolean
+  onToggleVideo?: () => void
   onSelectedModelChange?: (model: SelectedModel | null) => void
   workDir?: string | null
   onWorkDirChange?: (value: string | null) => void
@@ -1562,6 +1597,8 @@ export function ChatInputWithMentions({
   onTtsModeChange,
   ttsAvatarEnabled,
   onToggleTtsAvatar,
+  videoEnabled,
+  onToggleVideo,
   onSelectedModelChange,
   workDir,
   onWorkDirChange,
@@ -1595,6 +1632,8 @@ export function ChatInputWithMentions({
         onTtsModeChange={onTtsModeChange}
         ttsAvatarEnabled={ttsAvatarEnabled}
         onToggleTtsAvatar={onToggleTtsAvatar}
+        videoEnabled={videoEnabled}
+        onToggleVideo={onToggleVideo}
         onSelectedModelChange={onSelectedModelChange}
         workDir={workDir}
         onWorkDirChange={onWorkDirChange}

apps/x/apps/renderer/src/components/chat-message-attachments.tsx

@@ -91,11 +91,28 @@ interface ChatMessageAttachmentsProps {
 export function ChatMessageAttachments({ attachments, className }: ChatMessageAttachmentsProps) {
   if (attachments.length === 0) return null
 
-  const imageAttachments = attachments.filter((attachment) => isImageMime(attachment.mimeType))
-  const fileAttachments = attachments.filter((attachment) => !isImageMime(attachment.mimeType))
+  const videoFrames = attachments.filter((attachment) => attachment.isVideoFrame)
+  const imageAttachments = attachments.filter(
+    (attachment) => !attachment.isVideoFrame && isImageMime(attachment.mimeType)
+  )
+  const fileAttachments = attachments.filter(
+    (attachment) => !attachment.isVideoFrame && !isImageMime(attachment.mimeType)
+  )
 
   return (
     <div className={cn('flex flex-col items-end gap-2', className)}>
+      {videoFrames.length > 0 && (
+        <div className="flex max-w-[340px] flex-wrap justify-end gap-1">
+          {videoFrames.map((frame, index) => (
+            <img
+              key={`frame-${index}`}
+              src={frame.thumbnailUrl}
+              alt={`Camera frame ${index + 1}`}
+              className="h-12 w-auto rounded-md border border-border/60 bg-muted object-cover"
+            />
+          ))}
+        </div>
+      )}
       {imageAttachments.length > 0 && (
         <div className="flex flex-wrap justify-end gap-2">
           {imageAttachments.map((attachment, index) => (

apps/x/apps/renderer/src/components/chat-sidebar.tsx

@@ -194,6 +194,8 @@ interface ChatSidebarProps {
   onTtsModeChange?: (mode: 'summary' | 'full') => void
   ttsAvatarEnabled?: boolean
   onToggleTtsAvatar?: () => void
+  videoEnabled?: boolean
+  onToggleVideo?: () => void
   onComposioConnected?: (toolkitSlug: string) => void
 }
 
@@ -260,6 +262,8 @@ export function ChatSidebar({
   onTtsModeChange,
   ttsAvatarEnabled,
   onToggleTtsAvatar,
+  videoEnabled,
+  onToggleVideo,
   onComposioConnected,
 }: ChatSidebarProps) {
   const { state: sidebarState } = useSidebar()
@@ -836,6 +840,8 @@ export function ChatSidebar({
                           onTtsModeChange={isActive ? onTtsModeChange : undefined}
                           ttsAvatarEnabled={ttsAvatarEnabled}
                           onToggleTtsAvatar={isActive ? onToggleTtsAvatar : undefined}
+                          videoEnabled={videoEnabled}
+                          onToggleVideo={isActive ? onToggleVideo : undefined}
                         />
                       </div>
                     )

apps/x/apps/renderer/src/components/video-preview-overlay.tsx

@@ -0,0 +1,60 @@
+import { useEffect, useRef } from 'react'
+import { X } from 'lucide-react'
+
+interface VideoPreviewOverlayProps {
+  /** Live camera stream from useVideoMode — attached to the preview element. */
+  streamRef: React.MutableRefObject<MediaStream | null>
+  onTurnOff: () => void
+}
+
+/**
+ * Floating picture-in-picture webcam preview shown while video chat mode is
+ * on. Mirrored like a selfie camera so the user's movements feel natural.
+ * Sits above the composer dock, mirroring the talking-head overlay's corner.
+ */
+export function VideoPreviewOverlay({ streamRef, onTurnOff }: VideoPreviewOverlayProps) {
+  const videoRef = useRef<HTMLVideoElement | null>(null)
+
+  useEffect(() => {
+    const videoEl = videoRef.current
+    if (!videoEl) return
+    videoEl.srcObject = streamRef.current
+    videoEl.play().catch(() => {})
+    return () => {
+      videoEl.srcObject = null
+    }
+  }, [streamRef])
+
+  return (
+    <div
+      className="group fixed bottom-28 left-8 z-50"
+      style={{ animation: 'video-preview-pop 0.35s cubic-bezier(0.34, 1.56, 0.64, 1)' }}
+    >
+      <style>{`
+        @keyframes video-preview-pop {
+          0% { opacity: 0; scale: 0.4; }
+          100% { opacity: 1; scale: 1; }
+        }
+      `}</style>
+      <button
+        type="button"
+        onClick={onTurnOff}
+        className="absolute -right-1.5 -top-1.5 z-10 flex h-5 w-5 items-center justify-center rounded-full border border-border bg-background text-muted-foreground opacity-0 shadow-sm transition-opacity hover:text-foreground group-hover:opacity-100"
+        aria-label="Turn off video chat"
+      >
+        <X className="h-3 w-3" />
+      </button>
+      <video
+        ref={videoRef}
+        muted
+        playsInline
+        className="h-32 w-auto rounded-xl border border-border/70 bg-black shadow-lg"
+        style={{ transform: 'scaleX(-1)' }}
+      />
+      <span className="pointer-events-none absolute bottom-1.5 left-1.5 flex items-center gap-1 rounded-full bg-black/60 px-1.5 py-0.5 text-[10px] font-medium text-white">
+        <span className="block h-1.5 w-1.5 animate-pulse rounded-full bg-red-500" />
+        Video on
+      </span>
+    </div>
+  )
+}

apps/x/apps/renderer/src/hooks/useVideoMode.ts

@@ -0,0 +1,174 @@
+import { useCallback, useEffect, useRef, useState } from 'react';
+
+export type VideoModeState = 'idle' | 'starting' | 'live';
+
+export interface CapturedVideoFrame {
+    /** base64-encoded JPEG bytes (no data: prefix) — shape of the UserImagePart wire format */
+    data: string;
+    mediaType: string;
+    capturedAt: string; // ISO timestamp
+    /** data: URL of the same frame, for direct display in the transcript */
+    dataUrl: string;
+}
+
+// Frames are grabbed once per second — dense enough to catch expression and
+// posture changes while the user talks. Per message we attach at most
+// MAX_FRAMES_PER_MESSAGE frames, evenly sampled across the window since the
+// last send, so long monologues don't balloon the request.
+const CAPTURE_INTERVAL_MS = 1000;
+const MAX_FRAMES_PER_MESSAGE = 12;
+// Rolling buffer bound (~2 minutes). The buffer only needs to cover the gap
+// between two sends; anything older is stale context anyway.
+const MAX_BUFFERED_FRAMES = 120;
+// Downscale target. 512px wide JPEG keeps a frame around 20-40KB — cheap
+// enough to inline a dozen per message as multimodal image parts.
+const FRAME_WIDTH = 512;
+const JPEG_QUALITY = 0.65;
+
+interface BufferedFrame {
+    dataUrl: string;
+    capturedAt: string;
+    ts: number;
+}
+
+export function useVideoMode() {
+    const [state, setState] = useState<VideoModeState>('idle');
+    const streamRef = useRef<MediaStream | null>(null);
+    const videoElRef = useRef<HTMLVideoElement | null>(null);
+    const canvasRef = useRef<HTMLCanvasElement | null>(null);
+    const intervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
+    const framesRef = useRef<BufferedFrame[]>([]);
+    const lastCollectTsRef = useRef(0);
+    const stateRef = useRef<VideoModeState>('idle');
+    stateRef.current = state;
+
+    const captureFrame = useCallback(() => {
+        const videoEl = videoElRef.current;
+        if (!videoEl || videoEl.readyState < 2 || videoEl.videoWidth === 0) return;
+        let canvas = canvasRef.current;
+        if (!canvas) {
+            canvas = document.createElement('canvas');
+            canvasRef.current = canvas;
+        }
+        const scale = FRAME_WIDTH / videoEl.videoWidth;
+        canvas.width = FRAME_WIDTH;
+        canvas.height = Math.round(videoEl.videoHeight * scale);
+        const ctx = canvas.getContext('2d');
+        if (!ctx) return;
+        ctx.drawImage(videoEl, 0, 0, canvas.width, canvas.height);
+        const dataUrl = canvas.toDataURL('image/jpeg', JPEG_QUALITY);
+        // A near-empty data URL means the frame was blank (camera still warming up)
+        if (dataUrl.length < 100) return;
+        const frames = framesRef.current;
+        frames.push({ dataUrl, capturedAt: new Date().toISOString(), ts: Date.now() });
+        if (frames.length > MAX_BUFFERED_FRAMES) {
+            frames.splice(0, frames.length - MAX_BUFFERED_FRAMES);
+        }
+    }, []);
+
+    const stop = useCallback(() => {
+        if (intervalRef.current) {
+            clearInterval(intervalRef.current);
+            intervalRef.current = null;
+        }
+        if (videoElRef.current) {
+            videoElRef.current.srcObject = null;
+            videoElRef.current = null;
+        }
+        if (streamRef.current) {
+            streamRef.current.getTracks().forEach((t) => t.stop());
+            streamRef.current = null;
+        }
+        framesRef.current = [];
+        lastCollectTsRef.current = 0;
+        setState('idle');
+    }, []);
+
+    const start = useCallback(async (): Promise<boolean> => {
+        if (stateRef.current !== 'idle') return true;
+        setState('starting');
+
+        // Settle the macOS TCC camera permission before getUserMedia, same as
+        // voice mode does for the mic — otherwise the first click silently
+        // fails while the native prompt is still up.
+        const access = await window.ipc
+            .invoke('voice:ensureCameraAccess', null)
+            .catch(() => ({ granted: true }));
+        if (!access.granted) {
+            console.error('[video] Camera access denied');
+            setState('idle');
+            return false;
+        }
+
+        let stream: MediaStream | null = null;
+        try {
+            stream = await navigator.mediaDevices.getUserMedia({
+                video: { width: { ideal: 1280 }, height: { ideal: 720 }, facingMode: 'user' },
+                audio: false,
+            });
+        } catch (err) {
+            console.error('[video] Camera access failed:', err);
+            setState('idle');
+            return false;
+        }
+
+        streamRef.current = stream;
+        // Offscreen <video> that feeds the capture canvas; the visible preview
+        // attaches to the same MediaStream separately.
+        const videoEl = document.createElement('video');
+        videoEl.muted = true;
+        videoEl.playsInline = true;
+        videoEl.srcObject = stream;
+        videoElRef.current = videoEl;
+        videoEl.play().catch(() => {});
+        // First frame as soon as the camera delivers data, then steady-state cadence.
+        videoEl.addEventListener('loadeddata', () => captureFrame(), { once: true });
+        intervalRef.current = setInterval(captureFrame, CAPTURE_INTERVAL_MS);
+        setState('live');
+        return true;
+    }, [captureFrame]);
+
+    /**
+     * Drain frames captured since the previous collection, evenly sampled down
+     * to MAX_FRAMES_PER_MESSAGE (always keeping the newest). Falls back to the
+     * single most recent frame when nothing new accumulated (rapid-fire
+     * messages), so every video-mode message carries at least one frame once
+     * the camera has warmed up.
+     */
+    const collectFrames = useCallback((): CapturedVideoFrame[] => {
+        if (stateRef.current !== 'live') return [];
+        // Grab a frame right now so the message always includes the moment of send.
+        captureFrame();
+        const all = framesRef.current;
+        if (all.length === 0) return [];
+
+        let window_ = all.filter((f) => f.ts > lastCollectTsRef.current);
+        if (window_.length === 0) {
+            window_ = [all[all.length - 1]];
+        }
+        lastCollectTsRef.current = window_[window_.length - 1].ts;
+
+        let sampled: BufferedFrame[];
+        if (window_.length <= MAX_FRAMES_PER_MESSAGE) {
+            sampled = window_;
+        } else {
+            sampled = [];
+            const step = (window_.length - 1) / (MAX_FRAMES_PER_MESSAGE - 1);
+            for (let i = 0; i < MAX_FRAMES_PER_MESSAGE; i++) {
+                sampled.push(window_[Math.round(i * step)]);
+            }
+        }
+
+        return sampled.map((f) => ({
+            data: f.dataUrl.slice(f.dataUrl.indexOf(',') + 1),
+            mediaType: 'image/jpeg',
+            capturedAt: f.capturedAt,
+            dataUrl: f.dataUrl,
+        }));
+    }, [captureFrame]);
+
+    // Release the camera if the component unmounts with video mode on.
+    useEffect(() => stop, [stop]);
+
+    return { state, streamRef, start, stop, collectFrames };
+}

apps/x/apps/renderer/src/lib/chat-conversation.ts

@@ -10,6 +10,9 @@ export interface MessageAttachment {
   mimeType: string
   size?: number
   thumbnailUrl?: string
+  /** Live webcam frame from video chat mode — rendered as a compact filmstrip.
+   *  Carries no path; thumbnailUrl holds the frame as a data: URL. */
+  isVideoFrame?: boolean
 }
 
 export interface ChatMessage {

apps/x/apps/renderer/src/lib/run-to-conversation.ts

@@ -41,6 +41,8 @@ export function runLogToConversation(log: RunLog): ConversationItem[] {
             filename?: string
             mimeType?: string
             size?: number
+            data?: string
+            mediaType?: string
             toolCallId?: string
             toolName?: string
             arguments?: unknown
@@ -52,13 +54,24 @@ export function runLogToConversation(log: RunLog): ConversationItem[] {
             .join('')
 
           const attachmentParts = parts.filter((p) => p.type === 'attachment' && p.path)
-          if (attachmentParts.length > 0) {
-            msgAttachments = attachmentParts.map((p) => ({
-              path: p.path!,
-              filename: p.filename || p.path!.split('/').pop() || p.path!,
-              mimeType: p.mimeType || 'application/octet-stream',
-              size: p.size,
-            }))
+          // Video-mode webcam frames — inline base64 image parts, shown as a filmstrip
+          const imageParts = parts.filter((p) => p.type === 'image' && p.data)
+          if (attachmentParts.length > 0 || imageParts.length > 0) {
+            msgAttachments = [
+              ...attachmentParts.map((p) => ({
+                path: p.path!,
+                filename: p.filename || p.path!.split('/').pop() || p.path!,
+                mimeType: p.mimeType || 'application/octet-stream',
+                size: p.size,
+              })),
+              ...imageParts.map((p, index) => ({
+                path: '',
+                filename: `camera-frame-${index + 1}.jpg`,
+                mimeType: p.mediaType || 'image/jpeg',
+                thumbnailUrl: `data:${p.mediaType || 'image/jpeg'};base64,${p.data}`,
+                isVideoFrame: true,
+              })),
+            ]
           }
 
           if (msg.role === 'assistant') {

apps/x/packages/core/src/agents/runtime.ts

@@ -337,6 +337,8 @@ export interface ComposeSystemInstructionsInput {
     searchEnabled: boolean;
     codeMode: 'claude' | 'codex' | null;
     codeCwd: string | null;
+    // Optional so legacy callers (old streamAgent path) are unaffected.
+    videoMode?: boolean;
 }
 
 // System-prompt assembly, extracted verbatim from streamAgent so the new turn
@@ -351,6 +353,7 @@ export function composeSystemInstructions({
     searchEnabled,
     codeMode,
     codeCwd,
+    videoMode,
 }: ComposeSystemInstructionsInput): string {
     let instructionsWithDateTime = `${instructions}\n\n${USER_CONTEXT_SYSTEM_INSTRUCTIONS}`;
         if (agentNotesContext) {
@@ -379,6 +382,21 @@ Do not announce the work directory unless it's relevant. Just use it.`;
         if (voiceInput) {
             instructionsWithDateTime += `\n\n# Voice Input\nThe user's message was transcribed from speech. Be aware that:\n- There may be transcription errors. Silently correct obvious ones (e.g. homophones, misheard words). If an error is genuinely ambiguous, briefly mention your interpretation (e.g. "I'm assuming you meant X").\n- Spoken messages are often long-winded. The user may ramble, repeat themselves, or correct something they said earlier in the same message. Focus on their final intent, not every word verbatim.`;
         }
+        if (videoMode) {
+            instructionsWithDateTime += `\n\n# Video Mode (Live Camera)
+The user has turned on video mode: their webcam is on, and their messages arrive with a series of live webcam frames (ordered oldest to newest) captured while they were speaking or typing. The frames are a live view of the user themselves — not a document or file to analyze.
+
+How to use the frames:
+- Use them for what visual awareness genuinely adds: the user's expressions, body language, posture, gestures, eye contact with the camera, visible energy, anything they hold up to the camera, and their surroundings when relevant.
+- Compare across frames to notice change over time within a message (e.g. increasingly slouched, started smiling, looked away for most of the message).
+- When the user practices something performative (a pitch, presentation, interview, talk) or asks for delivery feedback, give specific, actionable coaching grounded in what you actually see — cite concrete observations ("in the last few frames you looked down and away from the camera") rather than generic advice. Cover posture, eye contact, facial expressiveness, gesturing, and visible energy.
+- If they show something to the camera (an object, document, whiteboard), read or describe it and respond accordingly.
+
+Etiquette:
+- Do not narrate or list what you see in every response. Bring up visual observations only when relevant to the user's request or clearly worth mentioning.
+- Never comment on the user's physical appearance, attractiveness, or personal attributes — visual feedback is strictly about delivery, expression, and body language.
+- Frames are periodic snapshots, not continuous video; moments between frames are missing. Don't claim certainty about motion you couldn't have seen.`;
+        }
         if (voiceOutput === 'summary') {
             instructionsWithDateTime += `\n\n# Voice Output (MANDATORY — READ THIS FIRST)\nThe user has voice output enabled. THIS IS YOUR #1 PRIORITY: you MUST start your response with <voice></voice> tags. If your response does not begin with <voice> tags, the user will hear nothing — which is a broken experience. NEVER skip this.\n\nRules:\n1. YOUR VERY FIRST OUTPUT MUST BE A <voice> TAG. No exceptions. Do not start with markdown, headings, or any other text. The literal first characters of your response must be "<voice>".\n2. Place ALL <voice> tags at the BEGINNING of your response, before any detailed content. Do NOT intersperse <voice> tags throughout the response.\n3. Wrap EACH spoken sentence in its own separate <voice> tag so it can be spoken incrementally. Do NOT wrap everything in a single <voice> block.\n4. Use voice as a TL;DR and navigation aid — do NOT read the entire response aloud.\n5. After all <voice> tags, you may include detailed written content (markdown, tables, code, etc.) that will be shown visually but not spoken.\n\n## Examples\n\nExample 1 — User asks: "what happened in my meeting with Alex yesterday?"\n\n<voice>Your meeting with Alex covered three main things: the Q2 roadmap timeline, hiring for the backend role, and the client demo next week.</voice>\n<voice>I've pulled out the key details and action items below — the demo prep notes are at the end.</voice>\n\n## Meeting with Alex — March 11\n### Roadmap\n- Agreed to push Q2 launch to April 15...\n(detailed written content continues)\n\nExample 2 — User asks: "summarize my emails"\n\n<voice>You have five new emails since this morning.</voice>\n<voice>Two are from your team — Jordan sent the RFC you requested and Taylor flagged a contract issue.</voice>\n<voice>There's also a warm intro from a VC partner connecting you with someone at a prospective customer.</voice>\n<voice>I've drafted responses for three of them. The details and drafts are below.</voice>\n\n(email blocks, tables, and detailed content follow)\n\nExample 3 — User asks: "what's on my calendar today?"\n\n<voice>You've got a pretty packed day — seven meetings starting with standup at 9.</voice>\n<voice>The big ones are your investor call at 11, lunch with a partner from your lead VC at 12:30, and a customer call at 4.</voice>\n<voice>Your only free block for deep work is 2:30 to 4.</voice>\n\n(calendar block with full event details follows)\n\nExample 4 — User asks: "draft an email to Sam with our metrics"\n\n<voice>Done — I've drafted the email to Sam with your latest WAU and churn numbers.</voice>\n<voice>Take a look at the draft below and send it when you're ready.</voice>\n\n(email block with draft follows)\n\nREMEMBER: If you do not start with <voice> tags, the user hears silence. Always speak first, then write.`;
         } else if (voiceOutput === 'full') {
@@ -942,15 +960,22 @@ export function convertFromMessages(messages: z.infer<typeof Message>[]): ModelM
                         providerOptions,
                     });
                 } else {
-                    // New content parts array — collapse to text for LLM
+                    // New content parts array — collapse text/attachments to text
+                    // for the LLM; inline image parts (video-mode webcam frames)
+                    // are passed through as real multimodal image parts.
                     const textSegments: string[] = userMessageContextPrefix ? [userMessageContextPrefix] : [];
                     const attachmentLines: string[] = [];
+                    const imageParts: Array<{ type: "image"; image: string; mediaType: string }> = [];
+                    const frameTimes: string[] = [];
 
                     for (const part of msg.content) {
                         if (part.type === "attachment") {
                             const sizeStr = part.size ? `, ${formatBytes(part.size)}` : '';
                             const lineStr = part.lineNumber ? ` (line ${part.lineNumber})` : '';
                             attachmentLines.push(`- ${part.filename} (${part.mimeType}${sizeStr}) at ${part.path}${lineStr}`);
+                        } else if (part.type === "image") {
+                            imageParts.push({ type: "image", image: part.data, mediaType: part.mediaType });
+                            if (part.capturedAt) frameTimes.push(part.capturedAt);
                         } else {
                             textSegments.push(part.text);
                         }
@@ -964,11 +989,28 @@ export function convertFromMessages(messages: z.infer<typeof Message>[]): ModelM
                         }
                     }
 
-                    result.push({
-                        role: "user",
-                        content: textSegments.join("\n"),
-                        providerOptions,
-                    });
+                    if (imageParts.length > 0) {
+                        const span = frameTimes.length >= 2
+                            ? ` spanning ${frameTimes[0]} to ${frameTimes[frameTimes.length - 1]}`
+                            : frameTimes.length === 1
+                                ? ` captured at ${frameTimes[0]}`
+                                : '';
+                        textSegments.push(`[Video mode: ${imageParts.length} live webcam frame${imageParts.length === 1 ? '' : 's'} of the user attached below, oldest to newest,${span ? span + ',' : ''} recorded while they composed this message.]`);
+                        result.push({
+                            role: "user",
+                            content: [
+                                { type: "text", text: textSegments.join("\n") },
+                                ...imageParts,
+                            ],
+                            providerOptions,
+                        });
+                    } else {
+                        result.push({
+                            role: "user",
+                            content: textSegments.join("\n"),
+                            providerOptions,
+                        });
+                    }
                 }
                 break;
             }

apps/x/packages/core/src/turns/bridges/real-agent-resolver.ts

@@ -54,6 +54,7 @@ const CompositionOverrides = z.object({
     searchEnabled: z.boolean().optional(),
     codeMode: z.enum(["claude", "codex"]).nullable().optional(),
     codeCwd: z.string().nullable().optional(),
+    videoMode: z.boolean().optional(),
 });
 
 export interface RealAgentResolverDeps {
@@ -121,6 +122,7 @@ export class RealAgentResolver implements IAgentResolver {
             searchEnabled: composition.searchEnabled ?? false,
             codeMode: composition.codeMode ?? null,
             codeCwd: composition.codeCwd ?? null,
+            videoMode: composition.videoMode ?? false,
         });
 
         const tools = await this.resolveTools(agent);

apps/x/packages/shared/src/ipc.ts

@@ -1372,6 +1372,14 @@ const ipcSchemas = {
       granted: z.boolean(),
     }),
   },
+  // Same as ensureMicAccess but for the camera — settles the macOS TCC
+  // permission before video mode calls getUserMedia({ video: true }).
+  'voice:ensureCameraAccess': {
+    req: z.null(),
+    res: z.object({
+      granted: z.boolean(),
+    }),
+  },
   'meeting:checkScreenPermission': {
     req: z.null(),
     res: z.object({

apps/x/packages/shared/src/message.ts

@@ -44,8 +44,20 @@ export const UserAttachmentPart = z.object({
     lineNumber: z.number().int().min(1).optional(),  // 1-indexed line in source file (for editor-context references)
 });
 
-// Any single part of a user message (text or attachment)
-export const UserContentPart = z.union([UserTextPart, UserAttachmentPart]);
+// An inline image within a content array (e.g. a live webcam frame from
+// video mode). Unlike attachments, image parts carry their data inline as
+// base64 and are sent to the model as real multimodal image parts rather
+// than a file-path reference.
+export const UserImagePart = z.object({
+    type: z.literal("image"),
+    data: z.string(),                    // base64-encoded image bytes (no data: prefix)
+    mediaType: z.string(),               // MIME type ("image/jpeg")
+    source: z.enum(["camera"]).optional(),
+    capturedAt: z.string().optional(),   // ISO timestamp of capture
+});
+
+// Any single part of a user message (text, attachment, or inline image)
+export const UserContentPart = z.union([UserTextPart, UserAttachmentPart, UserImagePart]);
 
 // Named type for user message content — used everywhere instead of repeating the union
 export const UserMessageContent = z.union([z.string(), z.array(UserContentPart)]);