feat(x): browser screen-share picker, web notifications, Chrome extensions

Three capability upgrades to the embedded browser pane:

- Screen share: getDisplayMedia() in the browser partition now shows a
  Chrome-style source picker (screens/windows with thumbnails, optional
  system audio for screens) instead of silently capturing the primary
  screen. Requests are denied when the pane is hidden. The app's own
  session keeps its auto-loopback handler for meeting transcription.

- Notifications: the browser partition grants the notifications
  permission so sites (WhatsApp Web, Gmail, ...) can post native OS
  notifications while loaded. Background Web Push remains unavailable.

- Extensions: electron-chrome-extensions wired to the browser session.
  Unpacked extensions load from ~/.rowboat/extensions/<name>/; tab
  lifecycle is mirrored into chrome.tabs, and a <browser-action-list>
  toolbar hosts action icons/popups next to the address bar. NOTE: the
  library is GPL-3.0/Patron dual-licensed — licensing decision required
  before shipping in a release build.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Arjun 2026-07-16 09:18:40 +05:30
parent f68f546223
commit 781b16d881
12 changed files with 586 additions and 14 deletions

View file

@ -91,6 +91,17 @@ if (!fs.existsSync(stagedBinary)) {
}
console.log('✅ node-pty staged in .package/node_modules');
// electron-chrome-extensions injects a preload script into browser tabs to
// implement the chrome.* extension APIs. It resolves that file at runtime:
// via require.resolve when node_modules is present (dev), falling back to a
// file next to the running bundle (packaged app, where node_modules is
// gone). Stage it next to main.cjs for the packaged case.
const crxPreloadSrc = fs.realpathSync(
path.join(here, 'node_modules', 'electron-chrome-extensions', 'dist', 'chrome-extension-api.preload.js'),
);
fs.copyFileSync(crxPreloadSrc, path.join(here, '.package', 'dist', 'chrome-extension-api.preload.js'));
console.log('✅ electron-chrome-extensions preload staged');
// Compile the mic-monitor helper (ambient meeting detection) on macOS.
// Best-effort: without swiftc — or on other platforms — the app still works,
// ad-hoc meeting detection just stays off (main checks the binary exists).

View file

@ -19,6 +19,7 @@
"@x/shared": "workspace:*",
"agent-slack": "0.9.3",
"chokidar": "^4.0.3",
"electron-chrome-extensions": "^4.9.0",
"electron-squirrel-startup": "^1.0.1",
"html-to-docx": "^1.8.0",
"mammoth": "^1.11.0",

View file

@ -0,0 +1,129 @@
import path from 'node:path';
import fs from 'node:fs/promises';
import { session, type BrowserWindow, type Session, type WebContents } from 'electron';
import { ElectronChromeExtensions } from 'electron-chrome-extensions';
import { WorkDir } from '@x/core/dist/config/config.js';
import { browserViewManager } from './view.js';
/**
* Chrome extension support for the embedded browser, via
* electron-chrome-extensions (GPL-3.0 / Patron dual-licensed see the
* library's LICENSE.md before shipping this in a release build).
*
* Extensions are loaded unpacked from ~/.rowboat/extensions/<name>/ either
* a directory containing manifest.json directly, or (Chrome Web Store
* unpacked layout) a directory whose single versioned subdirectory contains
* it. There is no install UI; drop a folder there and restart the app.
*
* Known limits:
* - The browser session's client-hint spoofing uses Electron's webRequest
* API, which prevents extensions' chrome.webRequest listeners from firing
* (Electron allows one consumer per session). Blockers that rely on
* webRequest (uBlock MV2) won't block; content-script-based extensions
* (Dark Reader, password managers) work.
* - declarativeNetRequest is not implemented by Electron.
*/
const EXTENSIONS_DIR = path.join(WorkDir, 'extensions');
let extensions: ElectronChromeExtensions | null = null;
/**
* Called once at startup (before any browser use). Binds the extension
* system to the browser session when the BrowserViewManager creates it, and
* mirrors the manager's tab lifecycle into chrome.tabs.
*/
export function setupBrowserExtensions(): void {
browserViewManager.on('session-created', (browserSession: Session) => {
initExtensions(browserSession);
});
browserViewManager.on('tab-created', (wc: WebContents, win: BrowserWindow) => {
try {
extensions?.addTab(wc, win);
} catch (error) {
console.error('[Extensions] addTab failed:', error);
}
});
browserViewManager.on('tab-selected', (wc: WebContents) => {
try {
extensions?.selectTab(wc);
} catch (error) {
console.error('[Extensions] selectTab failed:', error);
}
});
// Serves crx:// (extension action icons) to the app renderer, which hosts
// the <browser-action-list> element on the default session; the element's
// partition attribute routes state queries to the browser session.
ElectronChromeExtensions.handleCRXProtocol(session.defaultSession);
}
function initExtensions(browserSession: Session): void {
if (extensions) return;
extensions = new ElectronChromeExtensions({
license: 'GPL-3.0',
session: browserSession,
async createTab(details) {
const result = await browserViewManager.newTab(details.url);
if (!result.ok || !result.tabId) {
throw new Error(result.error ?? 'Failed to create tab');
}
const wc = browserViewManager.getTabWebContents(result.tabId);
const win = browserViewManager.getWindow();
if (!wc || !win) throw new Error('Browser window is not available');
return [wc, win];
},
selectTab(wc) {
const tabId = browserViewManager.getTabIdForWebContents(wc);
if (tabId) browserViewManager.switchTab(tabId);
},
removeTab(wc) {
const tabId = browserViewManager.getTabIdForWebContents(wc);
if (tabId) browserViewManager.closeTab(tabId);
},
});
void loadUnpackedExtensions(browserSession);
}
async function resolveExtensionRoot(dir: string): Promise<string | null> {
const hasManifest = async (candidate: string) =>
fs.access(path.join(candidate, 'manifest.json')).then(() => true, () => false);
if (await hasManifest(dir)) return dir;
// Chrome Web Store unpacked layout: <id>/<version>/manifest.json. Pick the
// lexically-latest version directory.
const entries = await fs.readdir(dir, { withFileTypes: true }).catch(() => []);
const versionDirs = entries
.filter((entry) => entry.isDirectory())
.map((entry) => entry.name)
.sort()
.reverse();
for (const name of versionDirs) {
const candidate = path.join(dir, name);
if (await hasManifest(candidate)) return candidate;
}
return null;
}
async function loadUnpackedExtensions(browserSession: Session): Promise<void> {
await fs.mkdir(EXTENSIONS_DIR, { recursive: true });
const entries = await fs.readdir(EXTENSIONS_DIR, { withFileTypes: true }).catch(() => []);
for (const entry of entries) {
if (!entry.isDirectory()) continue;
const root = await resolveExtensionRoot(path.join(EXTENSIONS_DIR, entry.name));
if (!root) {
console.warn(`[Extensions] No manifest.json under ${path.join(EXTENSIONS_DIR, entry.name)} — skipped`);
continue;
}
try {
const extension = await browserSession.extensions.loadExtension(root);
console.log(`[Extensions] Loaded ${extension.name}@${extension.version} (${root})`);
} catch (error) {
console.error(`[Extensions] Failed to load ${root}:`, error);
}
}
}

View file

@ -1,6 +1,6 @@
import { BrowserWindow } from 'electron';
import { ipc } from '@x/shared';
import { browserViewManager, type BrowserState, type HttpAuthRequest } from './view.js';
import { browserViewManager, type BrowserState, type DisplayMediaRequest, type HttpAuthRequest } from './view.js';
type IPCChannels = ipc.IPCChannels;
@ -21,6 +21,7 @@ type BrowserHandlers = {
'browser:reload': InvokeHandler<'browser:reload'>;
'browser:getState': InvokeHandler<'browser:getState'>;
'browser:httpAuthResponse': InvokeHandler<'browser:httpAuthResponse'>;
'browser:displayMediaResponse': InvokeHandler<'browser:displayMediaResponse'>;
};
/**
@ -66,6 +67,9 @@ export const browserIpcHandlers: BrowserHandlers = {
'browser:httpAuthResponse': async (_event, args) => {
return browserViewManager.respondToHttpAuth(args);
},
'browser:displayMediaResponse': async (_event, args) => {
return browserViewManager.respondToDisplayMedia(args);
},
};
/**
@ -96,4 +100,12 @@ export function setupBrowserEventForwarding(): void {
browserViewManager.on('http-auth-resolved', (requestId: string) => {
broadcast('browser:httpAuthResolved', { requestId });
});
browserViewManager.on('display-media-request', (request: DisplayMediaRequest) => {
broadcast('browser:displayMediaRequest', request);
});
browserViewManager.on('display-media-resolved', (requestId: string) => {
broadcast('browser:displayMediaResolved', { requestId });
});
}

View file

@ -1,11 +1,12 @@
import { randomUUID } from 'node:crypto';
import { EventEmitter } from 'node:events';
import { BrowserWindow, WebContentsView, session, shell, type Session, type WebContents } from 'electron';
import { BrowserWindow, WebContentsView, desktopCapturer, session, shell, type Session, type WebContents } from 'electron';
import type {
BrowserPageElement,
BrowserPageSnapshot,
BrowserState,
BrowserTabState,
DisplayMediaRequest,
HttpAuthRequest,
} from '@x/shared/dist/browser-control.js';
import { normalizeNavigationTarget } from './navigation.js';
@ -21,7 +22,7 @@ import {
type RawBrowserPageSnapshot,
} from './page-scripts.js';
export type { BrowserPageSnapshot, BrowserState, BrowserTabState, HttpAuthRequest };
export type { BrowserPageSnapshot, BrowserState, BrowserTabState, DisplayMediaRequest, HttpAuthRequest };
/**
* Embedded browser pane implementation.
@ -64,6 +65,8 @@ const SPOOF_UA = buildChromeUserAgent();
const HOME_URL = 'https://www.google.com';
const NAVIGATION_TIMEOUT_MS = 10000;
const HTTP_AUTH_TIMEOUT_MS = 120000;
const DISPLAY_MEDIA_TIMEOUT_MS = 120000;
const DISPLAY_MEDIA_THUMBNAIL_SIZE = { width: 320, height: 180 };
const POST_ACTION_IDLE_MS = 400;
const POST_ACTION_MAX_ELEMENTS = 25;
const POST_ACTION_MAX_TEXT_LENGTH = 4000;
@ -96,6 +99,14 @@ type PendingHttpAuth = {
webContents: WebContents;
};
type PendingDisplayMedia = {
callback: (streams: Electron.Streams) => void;
timer: NodeJS.Timeout;
// The picker answers with a source id; the native callback needs the full
// DesktopCapturerSource, so keep the gathered sources until resolution.
sources: Map<string, Electron.DesktopCapturerSource>;
};
const EMPTY_STATE: BrowserState = {
activeTabId: null,
tabs: [],
@ -138,6 +149,7 @@ export class BrowserViewManager extends EventEmitter {
private bounds: BrowserBounds = { x: 0, y: 0, width: 0, height: 0 };
private snapshotCache = new Map<string, CachedSnapshot>();
private pendingHttpAuth = new Map<string, PendingHttpAuth>();
private pendingDisplayMedia = new Map<string, PendingDisplayMedia>();
// Child windows created by page window.open() (OAuth/SSO popups). Tracked so
// they can be closed when the host window goes away — otherwise an orphaned
// popup keeps BrowserWindow.getAllWindows() non-empty and, on macOS, blocks
@ -188,6 +200,9 @@ export class BrowserViewManager extends EventEmitter {
for (const requestId of [...this.pendingHttpAuth.keys()]) {
this.finishHttpAuth(requestId);
}
for (const requestId of [...this.pendingDisplayMedia.keys()]) {
this.finishDisplayMedia(requestId);
}
// Close any OAuth/SSO popups so they don't outlive the app window.
for (const popup of popups) {
if (!popup.isDestroyed()) popup.close();
@ -244,10 +259,108 @@ export class BrowserViewManager extends EventEmitter {
callback({ requestHeaders });
});
// getDisplayMedia() from pages (e.g. Google Meet "Present now"). When the
// browser pane is on screen, forward a source picker to the renderer and
// resolve with the user's choice. Registered here (not in main.ts's
// configureSessionPermissions) so the picker plumbing lives with the rest
// of the pane's request/response wiring; the app's own session keeps its
// auto-approve loopback handler for meeting transcription.
browserSession.setDisplayMediaRequestHandler((_request, callback) => {
this.handleDisplayMediaRequest(callback).catch(() => callback({}));
});
this.browserSession = browserSession;
// Synchronous on purpose: the extension system (browser/extensions.ts)
// must bind to the session — registering its tab-API preload — before the
// first tab's WebContentsView is constructed right after this returns.
this.emit('session-created', browserSession);
return browserSession;
}
/**
* Resolve a page's getDisplayMedia() request. With the pane visible, gather
* shareable sources and ask the renderer to show a picker (denied after a
* timeout if unanswered). With the pane hidden, deny outright: nobody can
* answer a picker, and auto-granting would hand a live screen capture to an
* arbitrary page without consent. `visible` is also false whenever the
* renderer hides the view behind a blocking overlay including this
* picker's own dialog so a page re-requesting mid-picker lands here and
* must be denied, not silently granted.
*/
private async handleDisplayMediaRequest(callback: (streams: Electron.Streams) => void): Promise<void> {
if (!this.visible || !this.window) {
callback({});
return;
}
const sources = await desktopCapturer.getSources({
types: ['screen', 'window'],
thumbnailSize: DISPLAY_MEDIA_THUMBNAIL_SIZE,
fetchWindowIcons: true,
});
if (sources.length === 0) {
callback({});
return;
}
const requestId = randomUUID();
const timer = setTimeout(() => {
this.finishDisplayMedia(requestId);
}, DISPLAY_MEDIA_TIMEOUT_MS);
this.pendingDisplayMedia.set(requestId, {
callback,
timer,
sources: new Map(sources.map((source) => [source.id, source])),
});
const request: DisplayMediaRequest = {
requestId,
sources: sources.map((source) => ({
id: source.id,
name: source.name,
kind: source.id.startsWith('screen:') ? 'screen' as const : 'window' as const,
thumbnailDataUrl: source.thumbnail.isEmpty() ? '' : source.thumbnail.toDataURL(),
// appIcon is typed non-null but is absent for screen sources.
appIconDataUrl: source.appIcon && !source.appIcon.isEmpty() ? source.appIcon.toDataURL() : null,
})),
};
this.emit('display-media-request', request);
}
/**
* Resolve a pending display-media request. `sourceId === undefined` (or an
* id no longer in the gathered set) denies it. Always notifies the renderer
* so a picker it may still be showing (e.g. after a timeout) is pruned.
*/
private finishDisplayMedia(requestId: string, sourceId?: string, audio?: boolean): boolean {
const pending = this.pendingDisplayMedia.get(requestId);
if (!pending) return false;
this.pendingDisplayMedia.delete(requestId);
clearTimeout(pending.timer);
const source = sourceId != null ? pending.sources.get(sourceId) : undefined;
try {
if (!source) {
pending.callback({});
} else if (audio) {
pending.callback({ video: source, audio: 'loopback' });
} else {
pending.callback({ video: source });
}
} catch {
// The requesting webContents may already be destroyed.
}
this.emit('display-media-resolved', requestId);
return true;
}
respondToDisplayMedia(input: {
requestId: string;
sourceId?: string;
audio?: boolean;
}): { ok: boolean } {
return { ok: this.finishDisplayMedia(input.requestId, input.sourceId, input.audio) };
}
private emitState(): void {
this.emit('state-updated', this.snapshotState());
}
@ -560,6 +673,10 @@ export class BrowserViewManager extends EventEmitter {
this.invalidateSnapshot(tabId);
this.syncAttachedView();
this.emitState();
// Register with the extension system (chrome.tabs) before the initial
// load below so extensions observe the tab's first navigation.
this.emit('tab-created', tab.view.webContents, this.window);
this.emit('tab-selected', tab.view.webContents);
const targetUrl =
initialUrl === 'about:blank'
@ -733,11 +850,13 @@ export class BrowserViewManager extends EventEmitter {
}
switchTab(tabId: string): { ok: boolean } {
if (!this.tabs.has(tabId)) return { ok: false };
const tab = this.tabs.get(tabId);
if (!tab) return { ok: false };
if (this.activeTabId === tabId) return { ok: true };
this.activeTabId = tabId;
this.syncAttachedView();
this.emitState();
this.emit('tab-selected', tab.view.webContents);
return { ok: true };
}
@ -1048,6 +1167,24 @@ export class BrowserViewManager extends EventEmitter {
return this.snapshotState();
}
// Accessors for the extension system's chrome.tabs bridge
// (browser/extensions.ts), which speaks in WebContents while the manager
// speaks in tab ids.
getWindow(): BrowserWindow | null {
return this.window;
}
getTabWebContents(tabId: string): WebContents | null {
return this.getTab(tabId)?.view.webContents ?? null;
}
getTabIdForWebContents(wc: WebContents): string | null {
for (const tab of this.tabs.values()) {
if (tab.view.webContents === wc) return tab.id;
}
return null;
}
private snapshotState(): BrowserState {
if (this.tabOrder.length === 0) return { ...EMPTY_STATE };
return {

View file

@ -55,6 +55,7 @@ import type { CodeModeManager } from "@x/core/dist/code-mode/acp/manager.js";
import type { ISessions } from "@x/core/dist/runtime/sessions/index.js";
import { browserViewManager, BROWSER_PARTITION } from "./browser/view.js";
import { setupBrowserEventForwarding } from "./browser/ipc.js";
import { setupBrowserExtensions } from "./browser/extensions.js";
import { ElectronBrowserControlService } from "./browser/control-service.js";
import { ElectronNotificationService } from "./notification/electron-notification-service.js";
import {
@ -226,18 +227,34 @@ protocol.registerSchemesAsPrivileged([
const ALLOWED_SESSION_PERMISSIONS = new Set(["media", "display-capture", "clipboard-read", "clipboard-sanitized-write"]);
function configureSessionPermissions(targetSession: Session): void {
// Granted to the embedded browser partition on top of the base set.
// `notifications` lets sites (WhatsApp Web, Gmail, Slack, ...) show native OS
// notifications via the HTML5 Notification API — Electron renders these
// through the system notification center once the permission resolves to
// granted. Background Web Push is still unavailable (Electron has no FCM),
// so notifications only fire while the site is loaded in a tab. The app's
// own renderer keeps the base set; it notifies through the main-process
// notification service instead.
const BROWSER_EXTRA_PERMISSIONS = ["notifications"] as const;
function configureSessionPermissions(targetSession: Session, extraPermissions: readonly string[] = []): void {
const allowed = new Set([...ALLOWED_SESSION_PERMISSIONS, ...extraPermissions]);
targetSession.setPermissionCheckHandler((_webContents, permission) => {
return ALLOWED_SESSION_PERMISSIONS.has(permission);
return allowed.has(permission);
});
targetSession.setPermissionRequestHandler((_webContents, permission, callback) => {
callback(ALLOWED_SESSION_PERMISSIONS.has(permission));
callback(allowed.has(permission));
});
}
// Auto-approve display media requests and route system audio as loopback.
// Electron requires a video source in the callback even if we only want audio.
// We pass the first available screen source; the renderer discards the video track.
// Auto-approve display media requests and route system audio as loopback.
// Electron requires a video source in the callback even if we only want audio.
// We pass the first available screen source; the renderer discards the video track.
// App session only — the embedded browser partition registers its own handler
// (a user-facing source picker) in BrowserViewManager.
function configureAppDisplayMediaHandler(targetSession: Session): void {
targetSession.setDisplayMediaRequestHandler(async (_request, callback) => {
const sources = await desktopCapturer.getSources({ types: ['screen'] });
if (sources.length === 0) {
@ -346,7 +363,8 @@ function createWindow(options: { startHidden?: boolean } = {}) {
});
configureSessionPermissions(session.defaultSession);
configureSessionPermissions(session.fromPartition(BROWSER_PARTITION));
configureAppDisplayMediaHandler(session.defaultSession);
configureSessionPermissions(session.fromPartition(BROWSER_PARTITION), BROWSER_EXTRA_PERMISSIONS);
mainWindow = win;
setMainWindowForDeepLinks(win);
@ -471,6 +489,7 @@ app.whenReady().then(async () => {
setupIpcHandlers();
setupBrowserEventForwarding();
setupBrowserExtensions();
// Start the Rowboat Apps server (per-app origins on 127.0.0.1:3210) BEFORE
// the window and the long service-init chain below. The Apps view is

View file

@ -7,7 +7,8 @@
"build": "rm -rf dist && tsc && esbuild dist/preload.js --bundle --platform=node --format=cjs --external:electron --outfile=dist/preload.bundle.js && mv dist/preload.bundle.js dist/preload.js"
},
"dependencies": {
"@x/shared": "workspace:*"
"@x/shared": "workspace:*",
"electron-chrome-extensions": "^4.9.0"
},
"devDependencies": {
"electron": "^39.2.7",

View file

@ -1,6 +1,19 @@
import { contextBridge, ipcRenderer, webFrame, webUtils } from 'electron';
import { injectBrowserAction } from 'electron-chrome-extensions/browser-action';
import { ipc as ipcShared } from '@x/shared';
// Expose the <browser-action-list> custom element (extension action icons +
// popups for the embedded browser pane). App documents only — this preload
// is attached solely to the app window, but guard against it ever being
// reused for remote content.
if (location.protocol === 'app:' || location.origin === 'http://localhost:5173') {
try {
injectBrowserAction();
} catch (error) {
console.error('[preload] injectBrowserAction failed:', error);
}
}
type InvokeChannels = ipcShared.InvokeChannels;
type IPCChannels = ipcShared.IPCChannels;
type SendChannels = ipcShared.SendChannels;

View file

@ -1,7 +1,25 @@
import { useCallback, useEffect, useRef, useState } from 'react'
import { ArrowLeft, ArrowRight, Loader2, Plus, RotateCw, X } from 'lucide-react'
import type { HttpAuthRequest } from '@x/shared/dist/browser-control.js'
// Custom element provided by electron-chrome-extensions (injected via the
// preload script): a row of extension action icons for the given session
// partition, with popup handling built in. Type names resolve against the
// react module's own scope inside this augmentation.
declare module 'react' {
namespace JSX {
interface IntrinsicElements {
'browser-action-list': import('react').DetailedHTMLProps<
import('react').HTMLAttributes<HTMLElement>,
HTMLElement
> & {
partition?: string
alignment?: string
}
}
}
}
import type { DisplayMediaRequest, DisplayMediaSource, HttpAuthRequest } from '@x/shared/dist/browser-control.js'
import { TabBar } from '@/components/tab-bar'
import { Button } from '@/components/ui/button'
@ -168,10 +186,127 @@ function BrowserHttpAuthDialog({
)
}
/**
* Source picker for getDisplayMedia() requests raised by pages in the
* embedded browser (e.g. Google Meet "Present now"). Mirrors Chrome's share
* dialog: pick a screen or window, optionally include system audio (screens
* only loopback capture is system-wide, so it isn't offered per-window).
*/
function BrowserScreenSharePickerDialog({
request,
onSubmit,
onCancel,
}: {
request: DisplayMediaRequest
onSubmit: (sourceId: string, audio: boolean) => void
onCancel: () => void
}) {
const [selectedId, setSelectedId] = useState<string | null>(null)
const [shareAudio, setShareAudio] = useState(false)
const screens = request.sources.filter((source) => source.kind === 'screen')
const windows = request.sources.filter((source) => source.kind === 'window')
const selected = request.sources.find((source) => source.id === selectedId) ?? null
const audioAvailable = selected?.kind === 'screen'
const renderSources = (sources: DisplayMediaSource[]) => (
<div className="grid grid-cols-3 gap-2">
{sources.map((source) => (
<button
key={source.id}
type="button"
onClick={() => setSelectedId(source.id)}
className={cn(
'flex min-w-0 flex-col gap-1.5 rounded-md border p-1.5 text-left transition-colors',
source.id === selectedId
? 'border-ring bg-accent'
: 'border-border hover:bg-accent/50',
)}
>
<div className="flex h-20 items-center justify-center overflow-hidden rounded bg-muted">
{source.thumbnailDataUrl ? (
<img
src={source.thumbnailDataUrl}
alt=""
className="max-h-full max-w-full object-contain"
/>
) : null}
</div>
<div className="flex min-w-0 items-center gap-1">
{source.appIconDataUrl ? (
<img src={source.appIconDataUrl} alt="" className="size-3.5 shrink-0" />
) : null}
<span className="truncate text-xs text-foreground">{source.name}</span>
</div>
</button>
))}
</div>
)
return (
<Dialog open onOpenChange={(open) => { if (!open) onCancel() }}>
<DialogContent className="w-[min(36rem,calc(100%-2rem))] max-w-xl">
<DialogHeader>
<DialogTitle>Share your screen</DialogTitle>
<DialogDescription>
Choose what to share with this site.
</DialogDescription>
</DialogHeader>
<div className="flex max-h-[50vh] flex-col gap-3 overflow-y-auto pr-1">
{screens.length > 0 && (
<div className="flex flex-col gap-1.5">
<span className="text-xs font-medium text-muted-foreground">Screens</span>
{renderSources(screens)}
</div>
)}
{windows.length > 0 && (
<div className="flex flex-col gap-1.5">
<span className="text-xs font-medium text-muted-foreground">Windows</span>
{renderSources(windows)}
</div>
)}
</div>
<DialogFooter className="items-center sm:justify-between">
<label
className={cn(
'flex items-center gap-2 text-xs',
audioAvailable ? 'text-foreground' : 'text-muted-foreground/60',
)}
>
<input
type="checkbox"
checked={audioAvailable && shareAudio}
disabled={!audioAvailable}
onChange={(e) => setShareAudio(e.target.checked)}
/>
Also share system audio
</label>
<div className="flex gap-2">
<Button type="button" variant="outline" onClick={onCancel}>
Cancel
</Button>
<Button
type="button"
disabled={!selected}
onClick={() => {
if (!selected) return
onSubmit(selected.id, audioAvailable && shareAudio)
}}
>
Share
</Button>
</div>
</DialogFooter>
</DialogContent>
</Dialog>
)
}
export function BrowserPane({ onClose, forceHidden = false }: BrowserPaneProps) {
const [state, setState] = useState<BrowserState>(EMPTY_STATE)
const [addressValue, setAddressValue] = useState('')
const [authQueue, setAuthQueue] = useState<HttpAuthRequest[]>([])
const [displayMediaQueue, setDisplayMediaQueue] = useState<DisplayMediaRequest[]>([])
const activeTabIdRef = useRef<string | null>(null)
const addressFocusedRef = useRef(false)
@ -249,6 +384,47 @@ export function BrowserPane({ onClose, forceHidden = false }: BrowserPaneProps)
const activeAuthRequest = authQueue[0] ?? null
// Same lifecycle as the auth queue: push on request, prune on main-side
// resolution (timeout / window teardown), cancel leftovers on unmount so
// the main-process callbacks and timers are freed immediately.
const displayMediaQueueRef = useRef<DisplayMediaRequest[]>([])
useEffect(() => {
displayMediaQueueRef.current = displayMediaQueue
}, [displayMediaQueue])
useEffect(() => {
const offRequest = window.ipc.on('browser:displayMediaRequest', (incoming) => {
setDisplayMediaQueue((queue) => [...queue, incoming as DisplayMediaRequest])
})
const offResolved = window.ipc.on('browser:displayMediaResolved', (incoming) => {
const { requestId } = incoming as { requestId: string }
setDisplayMediaQueue((queue) => queue.filter((request) => request.requestId !== requestId))
})
return () => {
offRequest()
offResolved()
for (const request of displayMediaQueueRef.current) {
void window.ipc.invoke('browser:displayMediaResponse', { requestId: request.requestId })
}
}
}, [])
const respondToDisplayMedia = useCallback(
(requestId: string, choice: { sourceId: string; audio: boolean } | null) => {
setDisplayMediaQueue((queue) => queue.filter((request) => request.requestId !== requestId))
// Omit sourceId to cancel; include it to share the chosen source.
void window.ipc.invoke(
'browser:displayMediaResponse',
choice
? { requestId, sourceId: choice.sourceId, audio: choice.audio }
: { requestId },
)
},
[],
)
const activeDisplayMediaRequest = displayMediaQueue[0] ?? null
const setViewVisible = useCallback((visible: boolean) => {
if (viewVisibleRef.current === visible) return
viewVisibleRef.current = visible
@ -533,6 +709,11 @@ export function BrowserPane({ onClose, forceHidden = false }: BrowserPaneProps)
autoCapitalize="off"
/>
</form>
<browser-action-list
partition="persist:rowboat-browser"
alignment="bottom right"
className="ml-1 flex shrink-0 items-center"
/>
<button
type="button"
onClick={onClose}
@ -559,6 +740,17 @@ export function BrowserPane({ onClose, forceHidden = false }: BrowserPaneProps)
onCancel={() => respondToAuth(activeAuthRequest.requestId, null)}
/>
)}
{!activeAuthRequest && activeDisplayMediaRequest && (
<BrowserScreenSharePickerDialog
key={activeDisplayMediaRequest.requestId}
request={activeDisplayMediaRequest}
onSubmit={(sourceId, audio) =>
respondToDisplayMedia(activeDisplayMediaRequest.requestId, { sourceId, audio })
}
onCancel={() => respondToDisplayMedia(activeDisplayMediaRequest.requestId, null)}
/>
)}
</div>
)
}

View file

@ -23,6 +23,22 @@ export const HttpAuthRequestSchema = z.object({
realm: z.string().optional(),
});
// A screen or window offered to the user when a page in the embedded browser
// calls getDisplayMedia() (e.g. "Present now" in Google Meet). Main gathers
// these via desktopCapturer and the renderer shows a picker dialog.
export const DisplayMediaSourceSchema = z.object({
id: z.string(),
name: z.string(),
kind: z.enum(['screen', 'window']),
thumbnailDataUrl: z.string(),
appIconDataUrl: z.string().nullable(),
});
export const DisplayMediaRequestSchema = z.object({
requestId: z.string(),
sources: z.array(DisplayMediaSourceSchema),
});
export const BrowserPageElementSchema = z.object({
index: z.number().int().positive(),
tagName: z.string(),
@ -144,6 +160,8 @@ export const BrowserControlResultSchema = z.object({
export type BrowserTabState = z.infer<typeof BrowserTabStateSchema>;
export type BrowserState = z.infer<typeof BrowserStateSchema>;
export type HttpAuthRequest = z.infer<typeof HttpAuthRequestSchema>;
export type DisplayMediaSource = z.infer<typeof DisplayMediaSourceSchema>;
export type DisplayMediaRequest = z.infer<typeof DisplayMediaRequestSchema>;
export type BrowserPageElement = z.infer<typeof BrowserPageElementSchema>;
export type BrowserPageSnapshot = z.infer<typeof BrowserPageSnapshotSchema>;
export type BrowserControlAction = z.infer<typeof BrowserControlActionSchema>;

View file

@ -19,7 +19,7 @@ import type { SessionBusEvent, SessionIndexEntry, SessionState } from './session
import { RowboatApiConfig } from './rowboat-account.js';
import { ZListToolkitsResponse } from './composio.js';
import { AppSummarySchema, RegistryRecordSchema, RowboatAppManifestSchema } from './rowboat-app.js';
import { BrowserStateSchema, HttpAuthRequestSchema } from './browser-control.js';
import { BrowserStateSchema, DisplayMediaRequestSchema, HttpAuthRequestSchema } from './browser-control.js';
import { BillingInfoSchema } from './billing.js';
import { EmailBlockSchema, GmailThreadSchema } from './blocks.js';
import { PermissionDecision, ApprovalPolicy, CodingAgent, type CodeRunFeedEvent } from './code-mode.js';
@ -2303,6 +2303,30 @@ const ipcSchemas = {
}),
res: z.object({ ok: z.boolean() }),
},
// Screen-share picker for pages calling getDisplayMedia() in the embedded
// browser (main → renderer push). The renderer shows a source picker and
// answers via browser:displayMediaResponse.
'browser:displayMediaRequest': {
req: DisplayMediaRequestSchema,
res: z.null(),
},
// Main → renderer: a pending display-media request was resolved without the
// renderer answering (timed out, or the window went away), so the renderer
// must drop the corresponding picker dialog.
'browser:displayMediaResolved': {
req: z.object({ requestId: z.string() }),
res: z.null(),
},
// Renderer → main. Omit sourceId to cancel the request; `audio` asks for
// system-audio loopback alongside the shared screen.
'browser:displayMediaResponse': {
req: z.object({
requestId: z.string(),
sourceId: z.string().optional(),
audio: z.boolean().optional(),
}),
res: z.object({ ok: z.boolean() }),
},
// Billing channels
'billing:getInfo': {
req: z.null(),

15
apps/x/pnpm-lock.yaml generated
View file

@ -70,6 +70,9 @@ importers:
chokidar:
specifier: ^4.0.3
version: 4.0.3
electron-chrome-extensions:
specifier: ^4.9.0
version: 4.9.0
electron-squirrel-startup:
specifier: ^1.0.1
version: 1.0.1
@ -143,6 +146,9 @@ importers:
'@x/shared':
specifier: workspace:*
version: link:../../packages/shared
electron-chrome-extensions:
specifier: ^4.9.0
version: 4.9.0
devDependencies:
electron:
specifier: ^39.2.7
@ -5410,6 +5416,9 @@ packages:
ee-first@1.1.1:
resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==}
electron-chrome-extensions@4.9.0:
resolution: {integrity: sha512-4kLlh4sPPF0ieOy/Gw7A7KrmqB07megDLqOWAWXdFLAYu+2AeOGME6uOBa80NZLbKgTqBvrsVDZ+epdszRm/EQ==}
electron-installer-common@0.10.4:
resolution: {integrity: sha512-8gMNPXfAqUE5CfXg8RL0vXpLE9HAaPkgLXVoHE3BMUzogMWenf4LmwQ27BdCUrEhkjrKl+igs2IHJibclR3z3Q==}
engines: {node: '>= 10.0.0'}
@ -15065,6 +15074,12 @@ snapshots:
ee-first@1.1.1: {}
electron-chrome-extensions@4.9.0:
dependencies:
debug: 4.4.3
transitivePeerDependencies:
- supports-color
electron-installer-common@0.10.4:
dependencies:
'@electron/asar': 3.4.1