apunkt/rowboat
1
0
Fork 0
mirror of https://github.com/rowboatlabs/rowboat.git synced 2026-07-12 21:02:17 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

initial commit of video mode

Browse source
This commit is contained in:
Arjun 2026-07-03 22:29:10 +05:30
parent 3ba94402d3
commit 6f901095ff
13 changed files with 475 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

15
apps/x/apps/main/src/ipc.ts
View file

@ -1713,6 +1713,21 @@ export function setupIpcHandlers() {
return { granted: false };
}
},
'voice:ensureCameraAccess': async () => {
if (process.platform !== 'darwin') return { granted: true };
const status = systemPreferences.getMediaAccessStatus('camera');
console.log('[video] Camera permission status:', status);
if (status === 'granted') return { granted: true };
// Same flow as the microphone: settle the native TCC prompt before the
// renderer's getUserMedia so the first video click doesn't silently fail.
try {
const granted = await systemPreferences.askForMediaAccess('camera');
console.log('[video] Camera permission after prompt:', granted);
return { granted };
} catch {
return { granted: false };
}
},
// Live-note handlers
'live-note:run': async (_event, args) => {
const result = await runLiveNoteAgent(args.filePath, 'manual', args.context);

76
apps/x/apps/renderer/src/App.tsx
View file

@ -118,8 +118,10 @@ import { AgentScheduleConfig } from '@x/shared/dist/agent-schedule.js'
import { AgentScheduleState } from '@x/shared/dist/agent-schedule-state.js'
import { toast } from "sonner"
import { useVoiceMode } from '@/hooks/useVoiceMode'
import { useVideoMode } from '@/hooks/useVideoMode'
import { useVoiceTTS } from '@/hooks/useVoiceTTS'
import { TalkingHeadOverlay } from '@/components/talking-head'
import { VideoPreviewOverlay } from '@/components/video-preview-overlay'
import { ProductTour, type TourNavTarget } from '@/components/product-tour'
import { useMeetingTranscription, type CalendarEventMeta } from '@/hooks/useMeetingTranscription'
import { useAnalyticsIdentity } from '@/hooks/useAnalyticsIdentity'
@ -1010,6 +1012,23 @@ function App() {
const voiceRef = useRef(voice)
voiceRef.current = voice
// Video chat mode: while on, the webcam runs and frames are attached to
// every outgoing message so the assistant can see the user.
const video = useVideoMode()
const [videoEnabled, setVideoEnabled] = useState(false)
const videoEnabledRef = useRef(false)
const handleToggleVideo = useCallback(async () => {
if (videoEnabledRef.current) {
video.stop()
videoEnabledRef.current = false
setVideoEnabled(false)
} else {
const ok = await video.start()
videoEnabledRef.current = ok
setVideoEnabled(ok)
}
}, [video])
const handleToggleMeetingRef = useRef<(() => void) | undefined>(undefined)
const meetingTranscription = useMeetingTranscription(() => {
handleToggleMeetingRef.current?.()
@ -2608,15 +2627,28 @@ function App() {
setMessage('')
// Video chat mode: drain the webcam frames buffered since the last send
// so they ride along with this message as inline image parts.
const videoFrames = videoEnabledRef.current ? video.collectFrames() : []
const userMessageId = `user-${Date.now()}`
const displayAttachments: ChatMessage['attachments'] = hasAttachments
? stagedAttachments.map((attachment) => ({
path: attachment.path,
filename: attachment.filename,
mimeType: attachment.mimeType,
size: attachment.size,
thumbnailUrl: attachment.thumbnailUrl,
}))
const displayAttachments: ChatMessage['attachments'] = hasAttachments || videoFrames.length > 0
? [
...stagedAttachments.map((attachment) => ({
path: attachment.path,
filename: attachment.filename,
mimeType: attachment.mimeType,
size: attachment.size,
thumbnailUrl: attachment.thumbnailUrl,
})),
...videoFrames.map((frame, index) => ({
path: '',
filename: `camera-frame-${index + 1}.jpg`,
mimeType: frame.mediaType,
thumbnailUrl: frame.dataUrl,
isVideoFrame: true,
})),
]
: undefined
setConversation((prev) => [...prev, {
id: userMessageId,
@ -2668,6 +2700,7 @@ function App() {
...(ttsEnabledRef.current ? { voiceOutput: ttsModeRef.current } : {}),
...(searchEnabled ? { searchEnabled: true } : {}),
...(codeMode ? { codeMode } : {}),
...(videoEnabledRef.current ? { videoMode: true } : {}),
},
},
},
@ -2678,7 +2711,7 @@ function App() {
middlePane: middlePane ?? { kind: 'empty' as const },
})
if (hasAttachments || hasMentions) {
if (hasAttachments || hasMentions || videoFrames.length > 0) {
type ContentPart =
| { type: 'text'; text: string }
| {
@ -2689,6 +2722,13 @@ function App() {
size?: number
lineNumber?: number
}
| {
type: 'image'
data: string
mediaType: string
source: 'camera'
capturedAt: string
}
const contentParts: ContentPart[] = []
@ -2720,6 +2760,16 @@ function App() {
titleSource = stagedAttachments[0]?.filename ?? mentions?.[0]?.displayName ?? mentions?.[0]?.path ?? ''
}
for (const frame of videoFrames) {
contentParts.push({
type: 'image',
data: frame.data,
mediaType: frame.mediaType,
source: 'camera',
capturedAt: frame.capturedAt,
})
}
const middlePaneContext = await buildMiddlePaneContext()
await window.ipc.invoke('sessions:sendMessage', {
sessionId: currentRunId,
@ -6337,6 +6387,8 @@ function App() {
onTtsModeChange={isActive ? handleTtsModeChange : undefined}
ttsAvatarEnabled={ttsAvatarEnabled}
onToggleTtsAvatar={isActive ? handleToggleTtsAvatar : undefined}
videoEnabled={videoEnabled}
onToggleVideo={isActive ? handleToggleVideo : undefined}
/>
</div>
)
@ -6450,9 +6502,15 @@ function App() {
onTtsModeChange={handleTtsModeChange}
ttsAvatarEnabled={ttsAvatarEnabled}
onToggleTtsAvatar={handleToggleTtsAvatar}
videoEnabled={videoEnabled}
onToggleVideo={handleToggleVideo}
onComposioConnected={handleComposioConnected}
/>
)}
{/* Webcam PiP preview while video chat mode is on */}
{videoEnabled && (
<VideoPreviewOverlay streamRef={video.streamRef} onTurnOff={handleToggleVideo} />
)}
{/* Talking head hovers over the active view while avatar voice mode is
on (hidden during the tour, which shows its own mascot) */}
{ttsAvatarEnabled && !tourActive && (

39
apps/x/apps/renderer/src/components/chat-input-with-mentions.tsx
View file

@ -25,6 +25,7 @@ import {
ShieldCheck,
Square,
Terminal,
Video,
X,
} from 'lucide-react'
@ -238,6 +239,9 @@ interface ChatInputInnerProps {
onTtsModeChange?: (mode: 'summary' | 'full') => void
ttsAvatarEnabled?: boolean
onToggleTtsAvatar?: () => void
/** Video chat mode: live webcam frames are attached to each message. */
videoEnabled?: boolean
onToggleVideo?: () => void
/** Fired when the user picks a different model in the dropdown (only when no run exists yet). */
onSelectedModelChange?: (model: SelectedModel | null) => void
/** Work directory for this chat (per-chat). Null when none is set. */
@ -278,6 +282,8 @@ function ChatInputInner({
onTtsModeChange,
ttsAvatarEnabled,
onToggleTtsAvatar,
videoEnabled,
onToggleVideo,
onSelectedModelChange,
workDir = null,
onWorkDirChange,
@ -1350,6 +1356,33 @@ function ChatInputInner({
</TooltipContent>
</Tooltip>
)}
{onToggleVideo && (
<Tooltip delayDuration={CHAT_INPUT_TOOLTIP_DELAY_MS}>
<TooltipTrigger asChild>
<button
type="button"
onClick={onToggleVideo}
className={cn(
'relative flex h-7 w-7 shrink-0 items-center justify-center rounded-full transition-colors',
videoEnabled
? 'text-red-500 hover:bg-muted'
: 'text-muted-foreground hover:bg-muted hover:text-foreground'
)}
aria-label={videoEnabled ? 'Turn off video chat' : 'Turn on video chat'}
>
<Video className="h-4 w-4" />
{!videoEnabled && (
<span className="absolute inset-0 flex items-center justify-center pointer-events-none">
<span className="block h-[1.5px] w-5 -rotate-45 rounded-full bg-muted-foreground" />
</span>
)}
</button>
</TooltipTrigger>
<TooltipContent side="top">
{videoEnabled ? 'Video chat on — frames are shared with the assistant' : 'Video chat off'}
</TooltipContent>
</Tooltip>
)}
{voiceAvailable && onStartRecording && (
<button
type="button"
@ -1526,6 +1559,8 @@ export interface ChatInputWithMentionsProps {
onTtsModeChange?: (mode: 'summary' | 'full') => void
ttsAvatarEnabled?: boolean
onToggleTtsAvatar?: () => void
videoEnabled?: boolean
onToggleVideo?: () => void
onSelectedModelChange?: (model: SelectedModel | null) => void
workDir?: string | null
onWorkDirChange?: (value: string | null) => void
@ -1562,6 +1597,8 @@ export function ChatInputWithMentions({
onTtsModeChange,
ttsAvatarEnabled,
onToggleTtsAvatar,
videoEnabled,
onToggleVideo,
onSelectedModelChange,
workDir,
onWorkDirChange,
@ -1595,6 +1632,8 @@ export function ChatInputWithMentions({
onTtsModeChange={onTtsModeChange}
ttsAvatarEnabled={ttsAvatarEnabled}
onToggleTtsAvatar={onToggleTtsAvatar}
videoEnabled={videoEnabled}
onToggleVideo={onToggleVideo}
onSelectedModelChange={onSelectedModelChange}
workDir={workDir}
onWorkDirChange={onWorkDirChange}

21
apps/x/apps/renderer/src/components/chat-message-attachments.tsx
View file

@ -91,11 +91,28 @@ interface ChatMessageAttachmentsProps {
export function ChatMessageAttachments({ attachments, className }: ChatMessageAttachmentsProps) {
if (attachments.length === 0) return null
const imageAttachments = attachments.filter((attachment) => isImageMime(attachment.mimeType))
const fileAttachments = attachments.filter((attachment) => !isImageMime(attachment.mimeType))
const videoFrames = attachments.filter((attachment) => attachment.isVideoFrame)
const imageAttachments = attachments.filter(
(attachment) => !attachment.isVideoFrame && isImageMime(attachment.mimeType)
)
const fileAttachments = attachments.filter(
(attachment) => !attachment.isVideoFrame && !isImageMime(attachment.mimeType)
)
return (
<div className={cn('flex flex-col items-end gap-2', className)}>
{videoFrames.length > 0 && (
<div className="flex max-w-[340px] flex-wrap justify-end gap-1">
{videoFrames.map((frame, index) => (
<img
key={`frame-${index}`}
src={frame.thumbnailUrl}
alt={`Camera frame ${index + 1}`}
className="h-12 w-auto rounded-md border border-border/60 bg-muted object-cover"
/>
))}
</div>
)}
{imageAttachments.length > 0 && (
<div className="flex flex-wrap justify-end gap-2">
{imageAttachments.map((attachment, index) => (

6
apps/x/apps/renderer/src/components/chat-sidebar.tsx
View file

@ -194,6 +194,8 @@ interface ChatSidebarProps {
onTtsModeChange?: (mode: 'summary' | 'full') => void
ttsAvatarEnabled?: boolean
onToggleTtsAvatar?: () => void
videoEnabled?: boolean
onToggleVideo?: () => void
onComposioConnected?: (toolkitSlug: string) => void
}
@ -260,6 +262,8 @@ export function ChatSidebar({
onTtsModeChange,
ttsAvatarEnabled,
onToggleTtsAvatar,
videoEnabled,
onToggleVideo,
onComposioConnected,
}: ChatSidebarProps) {
const { state: sidebarState } = useSidebar()
@ -836,6 +840,8 @@ export function ChatSidebar({
onTtsModeChange={isActive ? onTtsModeChange : undefined}
ttsAvatarEnabled={ttsAvatarEnabled}
onToggleTtsAvatar={isActive ? onToggleTtsAvatar : undefined}
videoEnabled={videoEnabled}
onToggleVideo={isActive ? onToggleVideo : undefined}
/>
</div>
)

60
apps/x/apps/renderer/src/components/video-preview-overlay.tsx Normal file
View file

@ -0,0 +1,60 @@
import { useEffect, useRef } from 'react'
import { X } from 'lucide-react'
interface VideoPreviewOverlayProps {
/** Live camera stream from useVideoMode — attached to the preview element. */
streamRef: React.MutableRefObject<MediaStream | null>
onTurnOff: () => void
}
/**
* Floating picture-in-picture webcam preview shown while video chat mode is
* on. Mirrored like a selfie camera so the user's movements feel natural.
* Sits above the composer dock, mirroring the talking-head overlay's corner.
*/
export function VideoPreviewOverlay({ streamRef, onTurnOff }: VideoPreviewOverlayProps) {
const videoRef = useRef<HTMLVideoElement | null>(null)
useEffect(() => {
const videoEl = videoRef.current
if (!videoEl) return
videoEl.srcObject = streamRef.current
videoEl.play().catch(() => {})
return () => {
videoEl.srcObject = null
}
}, [streamRef])
return (
<div
className="group fixed bottom-28 left-8 z-50"
style={{ animation: 'video-preview-pop 0.35s cubic-bezier(0.34, 1.56, 0.64, 1)' }}
>
<style>{`
@keyframes video-preview-pop {
0% { opacity: 0; scale: 0.4; }
100% { opacity: 1; scale: 1; }
}
`}</style>
<button
type="button"
onClick={onTurnOff}
className="absolute -right-1.5 -top-1.5 z-10 flex h-5 w-5 items-center justify-center rounded-full border border-border bg-background text-muted-foreground opacity-0 shadow-sm transition-opacity hover:text-foreground group-hover:opacity-100"
aria-label="Turn off video chat"
>
<X className="h-3 w-3" />
</button>
<video
ref={videoRef}
muted
playsInline
className="h-32 w-auto rounded-xl border border-border/70 bg-black shadow-lg"
style={{ transform: 'scaleX(-1)' }}
/>
<span className="pointer-events-none absolute bottom-1.5 left-1.5 flex items-center gap-1 rounded-full bg-black/60 px-1.5 py-0.5 text-[10px] font-medium text-white">
<span className="block h-1.5 w-1.5 animate-pulse rounded-full bg-red-500" />
Video on
</span>
</div>
)
}

174
apps/x/apps/renderer/src/hooks/useVideoMode.ts Normal file
View file

@ -0,0 +1,174 @@
import { useCallback, useEffect, useRef, useState } from 'react';
export type VideoModeState = 'idle' | 'starting' | 'live';
export interface CapturedVideoFrame {
/** base64-encoded JPEG bytes (no data: prefix) — shape of the UserImagePart wire format */
data: string;
mediaType: string;
capturedAt: string; // ISO timestamp
/** data: URL of the same frame, for direct display in the transcript */
dataUrl: string;
}
// Frames are grabbed once per second — dense enough to catch expression and
// posture changes while the user talks. Per message we attach at most
// MAX_FRAMES_PER_MESSAGE frames, evenly sampled across the window since the
// last send, so long monologues don't balloon the request.
const CAPTURE_INTERVAL_MS = 1000;
const MAX_FRAMES_PER_MESSAGE = 12;
// Rolling buffer bound (~2 minutes). The buffer only needs to cover the gap
// between two sends; anything older is stale context anyway.
const MAX_BUFFERED_FRAMES = 120;
// Downscale target. 512px wide JPEG keeps a frame around 20-40KB — cheap
// enough to inline a dozen per message as multimodal image parts.
const FRAME_WIDTH = 512;
const JPEG_QUALITY = 0.65;
interface BufferedFrame {
dataUrl: string;
capturedAt: string;
ts: number;
}
export function useVideoMode() {
const [state, setState] = useState<VideoModeState>('idle');
const streamRef = useRef<MediaStream | null>(null);
const videoElRef = useRef<HTMLVideoElement | null>(null);
const canvasRef = useRef<HTMLCanvasElement | null>(null);
const intervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
const framesRef = useRef<BufferedFrame[]>([]);
const lastCollectTsRef = useRef(0);
const stateRef = useRef<VideoModeState>('idle');
stateRef.current = state;
const captureFrame = useCallback(() => {
const videoEl = videoElRef.current;
if (!videoEl || videoEl.readyState < 2 || videoEl.videoWidth === 0) return;
let canvas = canvasRef.current;
if (!canvas) {
canvas = document.createElement('canvas');
canvasRef.current = canvas;
}
const scale = FRAME_WIDTH / videoEl.videoWidth;
canvas.width = FRAME_WIDTH;
canvas.height = Math.round(videoEl.videoHeight * scale);
const ctx = canvas.getContext('2d');
if (!ctx) return;
ctx.drawImage(videoEl, 0, 0, canvas.width, canvas.height);
const dataUrl = canvas.toDataURL('image/jpeg', JPEG_QUALITY);
// A near-empty data URL means the frame was blank (camera still warming up)
if (dataUrl.length < 100) return;
const frames = framesRef.current;
frames.push({ dataUrl, capturedAt: new Date().toISOString(), ts: Date.now() });
if (frames.length > MAX_BUFFERED_FRAMES) {
frames.splice(0, frames.length - MAX_BUFFERED_FRAMES);
}
}, []);
const stop = useCallback(() => {
if (intervalRef.current) {
clearInterval(intervalRef.current);
intervalRef.current = null;
}
if (videoElRef.current) {
videoElRef.current.srcObject = null;
videoElRef.current = null;
}
if (streamRef.current) {
streamRef.current.getTracks().forEach((t) => t.stop());
streamRef.current = null;
}
framesRef.current = [];
lastCollectTsRef.current = 0;
setState('idle');
}, []);
const start = useCallback(async (): Promise<boolean> => {
if (stateRef.current !== 'idle') return true;
setState('starting');
// Settle the macOS TCC camera permission before getUserMedia, same as
// voice mode does for the mic — otherwise the first click silently
// fails while the native prompt is still up.
const access = await window.ipc
.invoke('voice:ensureCameraAccess', null)
.catch(() => ({ granted: true }));
if (!access.granted) {
console.error('[video] Camera access denied');
setState('idle');
return false;
}
let stream: MediaStream | null = null;
try {
stream = await navigator.mediaDevices.getUserMedia({
video: { width: { ideal: 1280 }, height: { ideal: 720 }, facingMode: 'user' },
audio: false,
});
} catch (err) {
console.error('[video] Camera access failed:', err);
setState('idle');
return false;
}
streamRef.current = stream;
// Offscreen <video> that feeds the capture canvas; the visible preview
// attaches to the same MediaStream separately.
const videoEl = document.createElement('video');
videoEl.muted = true;
videoEl.playsInline = true;
videoEl.srcObject = stream;
videoElRef.current = videoEl;
videoEl.play().catch(() => {});
// First frame as soon as the camera delivers data, then steady-state cadence.
videoEl.addEventListener('loadeddata', () => captureFrame(), { once: true });
intervalRef.current = setInterval(captureFrame, CAPTURE_INTERVAL_MS);
setState('live');
return true;
}, [captureFrame]);
/**
* Drain frames captured since the previous collection, evenly sampled down
* to MAX_FRAMES_PER_MESSAGE (always keeping the newest). Falls back to the
* single most recent frame when nothing new accumulated (rapid-fire
* messages), so every video-mode message carries at least one frame once
* the camera has warmed up.
*/
const collectFrames = useCallback((): CapturedVideoFrame[] => {
if (stateRef.current !== 'live') return [];
// Grab a frame right now so the message always includes the moment of send.
captureFrame();
const all = framesRef.current;
if (all.length === 0) return [];
let window_ = all.filter((f) => f.ts > lastCollectTsRef.current);
if (window_.length === 0) {
window_ = [all[all.length - 1]];
}
lastCollectTsRef.current = window_[window_.length - 1].ts;
let sampled: BufferedFrame[];
if (window_.length <= MAX_FRAMES_PER_MESSAGE) {
sampled = window_;
} else {
sampled = [];
const step = (window_.length - 1) / (MAX_FRAMES_PER_MESSAGE - 1);
for (let i = 0; i < MAX_FRAMES_PER_MESSAGE; i++) {
sampled.push(window_[Math.round(i * step)]);
}
}
return sampled.map((f) => ({
data: f.dataUrl.slice(f.dataUrl.indexOf(',') + 1),
mediaType: 'image/jpeg',
capturedAt: f.capturedAt,
dataUrl: f.dataUrl,
}));
}, [captureFrame]);
// Release the camera if the component unmounts with video mode on.
useEffect(() => stop, [stop]);
return { state, streamRef, start, stop, collectFrames };
}

3
apps/x/apps/renderer/src/lib/chat-conversation.ts
View file

@ -10,6 +10,9 @@ export interface MessageAttachment {
mimeType: string
size?: number
thumbnailUrl?: string
/** Live webcam frame from video chat mode — rendered as a compact filmstrip.
* Carries no path; thumbnailUrl holds the frame as a data: URL. */
isVideoFrame?: boolean
}
export interface ChatMessage {

27
apps/x/apps/renderer/src/lib/run-to-conversation.ts
View file

@ -41,6 +41,8 @@ export function runLogToConversation(log: RunLog): ConversationItem[] {
filename?: string
mimeType?: string
size?: number
data?: string
mediaType?: string
toolCallId?: string
toolName?: string
arguments?: unknown
@ -52,13 +54,24 @@ export function runLogToConversation(log: RunLog): ConversationItem[] {
.join('')
const attachmentParts = parts.filter((p) => p.type === 'attachment' && p.path)
if (attachmentParts.length > 0) {
msgAttachments = attachmentParts.map((p) => ({
path: p.path!,
filename: p.filename || p.path!.split('/').pop() || p.path!,
mimeType: p.mimeType || 'application/octet-stream',
size: p.size,
}))
// Video-mode webcam frames — inline base64 image parts, shown as a filmstrip
const imageParts = parts.filter((p) => p.type === 'image' && p.data)
if (attachmentParts.length > 0 || imageParts.length > 0) {
msgAttachments = [
...attachmentParts.map((p) => ({
path: p.path!,
filename: p.filename || p.path!.split('/').pop() || p.path!,
mimeType: p.mimeType || 'application/octet-stream',
size: p.size,
})),
...imageParts.map((p, index) => ({
path: '',
filename: `camera-frame-${index + 1}.jpg`,
mimeType: p.mediaType || 'image/jpeg',
thumbnailUrl: `data:${p.mediaType || 'image/jpeg'};base64,${p.data}`,
isVideoFrame: true,
})),
]
}
if (msg.role === 'assistant') {