mirror of
https://github.com/rowboatlabs/rowboat.git
synced 2026-07-21 21:31:12 +02:00
feat(x): browser screen-share picker, web notifications, Chrome extensions
Three capability upgrades to the embedded browser pane: - Screen share: getDisplayMedia() in the browser partition now shows a Chrome-style source picker (screens/windows with thumbnails, optional system audio for screens) instead of silently capturing the primary screen. Requests are denied when the pane is hidden. The app's own session keeps its auto-loopback handler for meeting transcription. - Notifications: the browser partition grants the notifications permission so sites (WhatsApp Web, Gmail, ...) can post native OS notifications while loaded. Background Web Push remains unavailable. - Extensions: electron-chrome-extensions wired to the browser session. Unpacked extensions load from ~/.rowboat/extensions/<name>/; tab lifecycle is mirrored into chrome.tabs, and a <browser-action-list> toolbar hosts action icons/popups next to the address bar. NOTE: the library is GPL-3.0/Patron dual-licensed — licensing decision required before shipping in a release build. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
f68f546223
commit
781b16d881
12 changed files with 586 additions and 14 deletions
|
|
@ -91,6 +91,17 @@ if (!fs.existsSync(stagedBinary)) {
|
|||
}
|
||||
console.log('✅ node-pty staged in .package/node_modules');
|
||||
|
||||
// electron-chrome-extensions injects a preload script into browser tabs to
|
||||
// implement the chrome.* extension APIs. It resolves that file at runtime:
|
||||
// via require.resolve when node_modules is present (dev), falling back to a
|
||||
// file next to the running bundle (packaged app, where node_modules is
|
||||
// gone). Stage it next to main.cjs for the packaged case.
|
||||
const crxPreloadSrc = fs.realpathSync(
|
||||
path.join(here, 'node_modules', 'electron-chrome-extensions', 'dist', 'chrome-extension-api.preload.js'),
|
||||
);
|
||||
fs.copyFileSync(crxPreloadSrc, path.join(here, '.package', 'dist', 'chrome-extension-api.preload.js'));
|
||||
console.log('✅ electron-chrome-extensions preload staged');
|
||||
|
||||
// Compile the mic-monitor helper (ambient meeting detection) on macOS.
|
||||
// Best-effort: without swiftc — or on other platforms — the app still works,
|
||||
// ad-hoc meeting detection just stays off (main checks the binary exists).
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@
|
|||
"@x/shared": "workspace:*",
|
||||
"agent-slack": "0.9.3",
|
||||
"chokidar": "^4.0.3",
|
||||
"electron-chrome-extensions": "^4.9.0",
|
||||
"electron-squirrel-startup": "^1.0.1",
|
||||
"html-to-docx": "^1.8.0",
|
||||
"mammoth": "^1.11.0",
|
||||
|
|
|
|||
129
apps/x/apps/main/src/browser/extensions.ts
Normal file
129
apps/x/apps/main/src/browser/extensions.ts
Normal file
|
|
@ -0,0 +1,129 @@
|
|||
import path from 'node:path';
|
||||
import fs from 'node:fs/promises';
|
||||
import { session, type BrowserWindow, type Session, type WebContents } from 'electron';
|
||||
import { ElectronChromeExtensions } from 'electron-chrome-extensions';
|
||||
import { WorkDir } from '@x/core/dist/config/config.js';
|
||||
import { browserViewManager } from './view.js';
|
||||
|
||||
/**
|
||||
* Chrome extension support for the embedded browser, via
|
||||
* electron-chrome-extensions (GPL-3.0 / Patron dual-licensed — see the
|
||||
* library's LICENSE.md before shipping this in a release build).
|
||||
*
|
||||
* Extensions are loaded unpacked from ~/.rowboat/extensions/<name>/ — either
|
||||
* a directory containing manifest.json directly, or (Chrome Web Store
|
||||
* unpacked layout) a directory whose single versioned subdirectory contains
|
||||
* it. There is no install UI; drop a folder there and restart the app.
|
||||
*
|
||||
* Known limits:
|
||||
* - The browser session's client-hint spoofing uses Electron's webRequest
|
||||
* API, which prevents extensions' chrome.webRequest listeners from firing
|
||||
* (Electron allows one consumer per session). Blockers that rely on
|
||||
* webRequest (uBlock MV2) won't block; content-script-based extensions
|
||||
* (Dark Reader, password managers) work.
|
||||
* - declarativeNetRequest is not implemented by Electron.
|
||||
*/
|
||||
|
||||
const EXTENSIONS_DIR = path.join(WorkDir, 'extensions');
|
||||
|
||||
let extensions: ElectronChromeExtensions | null = null;
|
||||
|
||||
/**
|
||||
* Called once at startup (before any browser use). Binds the extension
|
||||
* system to the browser session when the BrowserViewManager creates it, and
|
||||
* mirrors the manager's tab lifecycle into chrome.tabs.
|
||||
*/
|
||||
export function setupBrowserExtensions(): void {
|
||||
browserViewManager.on('session-created', (browserSession: Session) => {
|
||||
initExtensions(browserSession);
|
||||
});
|
||||
browserViewManager.on('tab-created', (wc: WebContents, win: BrowserWindow) => {
|
||||
try {
|
||||
extensions?.addTab(wc, win);
|
||||
} catch (error) {
|
||||
console.error('[Extensions] addTab failed:', error);
|
||||
}
|
||||
});
|
||||
browserViewManager.on('tab-selected', (wc: WebContents) => {
|
||||
try {
|
||||
extensions?.selectTab(wc);
|
||||
} catch (error) {
|
||||
console.error('[Extensions] selectTab failed:', error);
|
||||
}
|
||||
});
|
||||
|
||||
// Serves crx:// (extension action icons) to the app renderer, which hosts
|
||||
// the <browser-action-list> element on the default session; the element's
|
||||
// partition attribute routes state queries to the browser session.
|
||||
ElectronChromeExtensions.handleCRXProtocol(session.defaultSession);
|
||||
}
|
||||
|
||||
function initExtensions(browserSession: Session): void {
|
||||
if (extensions) return;
|
||||
|
||||
extensions = new ElectronChromeExtensions({
|
||||
license: 'GPL-3.0',
|
||||
session: browserSession,
|
||||
async createTab(details) {
|
||||
const result = await browserViewManager.newTab(details.url);
|
||||
if (!result.ok || !result.tabId) {
|
||||
throw new Error(result.error ?? 'Failed to create tab');
|
||||
}
|
||||
const wc = browserViewManager.getTabWebContents(result.tabId);
|
||||
const win = browserViewManager.getWindow();
|
||||
if (!wc || !win) throw new Error('Browser window is not available');
|
||||
return [wc, win];
|
||||
},
|
||||
selectTab(wc) {
|
||||
const tabId = browserViewManager.getTabIdForWebContents(wc);
|
||||
if (tabId) browserViewManager.switchTab(tabId);
|
||||
},
|
||||
removeTab(wc) {
|
||||
const tabId = browserViewManager.getTabIdForWebContents(wc);
|
||||
if (tabId) browserViewManager.closeTab(tabId);
|
||||
},
|
||||
});
|
||||
|
||||
void loadUnpackedExtensions(browserSession);
|
||||
}
|
||||
|
||||
async function resolveExtensionRoot(dir: string): Promise<string | null> {
|
||||
const hasManifest = async (candidate: string) =>
|
||||
fs.access(path.join(candidate, 'manifest.json')).then(() => true, () => false);
|
||||
|
||||
if (await hasManifest(dir)) return dir;
|
||||
|
||||
// Chrome Web Store unpacked layout: <id>/<version>/manifest.json. Pick the
|
||||
// lexically-latest version directory.
|
||||
const entries = await fs.readdir(dir, { withFileTypes: true }).catch(() => []);
|
||||
const versionDirs = entries
|
||||
.filter((entry) => entry.isDirectory())
|
||||
.map((entry) => entry.name)
|
||||
.sort()
|
||||
.reverse();
|
||||
for (const name of versionDirs) {
|
||||
const candidate = path.join(dir, name);
|
||||
if (await hasManifest(candidate)) return candidate;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function loadUnpackedExtensions(browserSession: Session): Promise<void> {
|
||||
await fs.mkdir(EXTENSIONS_DIR, { recursive: true });
|
||||
const entries = await fs.readdir(EXTENSIONS_DIR, { withFileTypes: true }).catch(() => []);
|
||||
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory()) continue;
|
||||
const root = await resolveExtensionRoot(path.join(EXTENSIONS_DIR, entry.name));
|
||||
if (!root) {
|
||||
console.warn(`[Extensions] No manifest.json under ${path.join(EXTENSIONS_DIR, entry.name)} — skipped`);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const extension = await browserSession.extensions.loadExtension(root);
|
||||
console.log(`[Extensions] Loaded ${extension.name}@${extension.version} (${root})`);
|
||||
} catch (error) {
|
||||
console.error(`[Extensions] Failed to load ${root}:`, error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
import { BrowserWindow } from 'electron';
|
||||
import { ipc } from '@x/shared';
|
||||
import { browserViewManager, type BrowserState, type HttpAuthRequest } from './view.js';
|
||||
import { browserViewManager, type BrowserState, type DisplayMediaRequest, type HttpAuthRequest } from './view.js';
|
||||
|
||||
type IPCChannels = ipc.IPCChannels;
|
||||
|
||||
|
|
@ -21,6 +21,7 @@ type BrowserHandlers = {
|
|||
'browser:reload': InvokeHandler<'browser:reload'>;
|
||||
'browser:getState': InvokeHandler<'browser:getState'>;
|
||||
'browser:httpAuthResponse': InvokeHandler<'browser:httpAuthResponse'>;
|
||||
'browser:displayMediaResponse': InvokeHandler<'browser:displayMediaResponse'>;
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
@ -66,6 +67,9 @@ export const browserIpcHandlers: BrowserHandlers = {
|
|||
'browser:httpAuthResponse': async (_event, args) => {
|
||||
return browserViewManager.respondToHttpAuth(args);
|
||||
},
|
||||
'browser:displayMediaResponse': async (_event, args) => {
|
||||
return browserViewManager.respondToDisplayMedia(args);
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
@ -96,4 +100,12 @@ export function setupBrowserEventForwarding(): void {
|
|||
browserViewManager.on('http-auth-resolved', (requestId: string) => {
|
||||
broadcast('browser:httpAuthResolved', { requestId });
|
||||
});
|
||||
|
||||
browserViewManager.on('display-media-request', (request: DisplayMediaRequest) => {
|
||||
broadcast('browser:displayMediaRequest', request);
|
||||
});
|
||||
|
||||
browserViewManager.on('display-media-resolved', (requestId: string) => {
|
||||
broadcast('browser:displayMediaResolved', { requestId });
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,11 +1,12 @@
|
|||
import { randomUUID } from 'node:crypto';
|
||||
import { EventEmitter } from 'node:events';
|
||||
import { BrowserWindow, WebContentsView, session, shell, type Session, type WebContents } from 'electron';
|
||||
import { BrowserWindow, WebContentsView, desktopCapturer, session, shell, type Session, type WebContents } from 'electron';
|
||||
import type {
|
||||
BrowserPageElement,
|
||||
BrowserPageSnapshot,
|
||||
BrowserState,
|
||||
BrowserTabState,
|
||||
DisplayMediaRequest,
|
||||
HttpAuthRequest,
|
||||
} from '@x/shared/dist/browser-control.js';
|
||||
import { normalizeNavigationTarget } from './navigation.js';
|
||||
|
|
@ -21,7 +22,7 @@ import {
|
|||
type RawBrowserPageSnapshot,
|
||||
} from './page-scripts.js';
|
||||
|
||||
export type { BrowserPageSnapshot, BrowserState, BrowserTabState, HttpAuthRequest };
|
||||
export type { BrowserPageSnapshot, BrowserState, BrowserTabState, DisplayMediaRequest, HttpAuthRequest };
|
||||
|
||||
/**
|
||||
* Embedded browser pane implementation.
|
||||
|
|
@ -64,6 +65,8 @@ const SPOOF_UA = buildChromeUserAgent();
|
|||
const HOME_URL = 'https://www.google.com';
|
||||
const NAVIGATION_TIMEOUT_MS = 10000;
|
||||
const HTTP_AUTH_TIMEOUT_MS = 120000;
|
||||
const DISPLAY_MEDIA_TIMEOUT_MS = 120000;
|
||||
const DISPLAY_MEDIA_THUMBNAIL_SIZE = { width: 320, height: 180 };
|
||||
const POST_ACTION_IDLE_MS = 400;
|
||||
const POST_ACTION_MAX_ELEMENTS = 25;
|
||||
const POST_ACTION_MAX_TEXT_LENGTH = 4000;
|
||||
|
|
@ -96,6 +99,14 @@ type PendingHttpAuth = {
|
|||
webContents: WebContents;
|
||||
};
|
||||
|
||||
type PendingDisplayMedia = {
|
||||
callback: (streams: Electron.Streams) => void;
|
||||
timer: NodeJS.Timeout;
|
||||
// The picker answers with a source id; the native callback needs the full
|
||||
// DesktopCapturerSource, so keep the gathered sources until resolution.
|
||||
sources: Map<string, Electron.DesktopCapturerSource>;
|
||||
};
|
||||
|
||||
const EMPTY_STATE: BrowserState = {
|
||||
activeTabId: null,
|
||||
tabs: [],
|
||||
|
|
@ -138,6 +149,7 @@ export class BrowserViewManager extends EventEmitter {
|
|||
private bounds: BrowserBounds = { x: 0, y: 0, width: 0, height: 0 };
|
||||
private snapshotCache = new Map<string, CachedSnapshot>();
|
||||
private pendingHttpAuth = new Map<string, PendingHttpAuth>();
|
||||
private pendingDisplayMedia = new Map<string, PendingDisplayMedia>();
|
||||
// Child windows created by page window.open() (OAuth/SSO popups). Tracked so
|
||||
// they can be closed when the host window goes away — otherwise an orphaned
|
||||
// popup keeps BrowserWindow.getAllWindows() non-empty and, on macOS, blocks
|
||||
|
|
@ -188,6 +200,9 @@ export class BrowserViewManager extends EventEmitter {
|
|||
for (const requestId of [...this.pendingHttpAuth.keys()]) {
|
||||
this.finishHttpAuth(requestId);
|
||||
}
|
||||
for (const requestId of [...this.pendingDisplayMedia.keys()]) {
|
||||
this.finishDisplayMedia(requestId);
|
||||
}
|
||||
// Close any OAuth/SSO popups so they don't outlive the app window.
|
||||
for (const popup of popups) {
|
||||
if (!popup.isDestroyed()) popup.close();
|
||||
|
|
@ -244,10 +259,108 @@ export class BrowserViewManager extends EventEmitter {
|
|||
callback({ requestHeaders });
|
||||
});
|
||||
|
||||
// getDisplayMedia() from pages (e.g. Google Meet "Present now"). When the
|
||||
// browser pane is on screen, forward a source picker to the renderer and
|
||||
// resolve with the user's choice. Registered here (not in main.ts's
|
||||
// configureSessionPermissions) so the picker plumbing lives with the rest
|
||||
// of the pane's request/response wiring; the app's own session keeps its
|
||||
// auto-approve loopback handler for meeting transcription.
|
||||
browserSession.setDisplayMediaRequestHandler((_request, callback) => {
|
||||
this.handleDisplayMediaRequest(callback).catch(() => callback({}));
|
||||
});
|
||||
|
||||
this.browserSession = browserSession;
|
||||
// Synchronous on purpose: the extension system (browser/extensions.ts)
|
||||
// must bind to the session — registering its tab-API preload — before the
|
||||
// first tab's WebContentsView is constructed right after this returns.
|
||||
this.emit('session-created', browserSession);
|
||||
return browserSession;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a page's getDisplayMedia() request. With the pane visible, gather
|
||||
* shareable sources and ask the renderer to show a picker (denied after a
|
||||
* timeout if unanswered). With the pane hidden, deny outright: nobody can
|
||||
* answer a picker, and auto-granting would hand a live screen capture to an
|
||||
* arbitrary page without consent. `visible` is also false whenever the
|
||||
* renderer hides the view behind a blocking overlay — including this
|
||||
* picker's own dialog — so a page re-requesting mid-picker lands here and
|
||||
* must be denied, not silently granted.
|
||||
*/
|
||||
private async handleDisplayMediaRequest(callback: (streams: Electron.Streams) => void): Promise<void> {
|
||||
if (!this.visible || !this.window) {
|
||||
callback({});
|
||||
return;
|
||||
}
|
||||
|
||||
const sources = await desktopCapturer.getSources({
|
||||
types: ['screen', 'window'],
|
||||
thumbnailSize: DISPLAY_MEDIA_THUMBNAIL_SIZE,
|
||||
fetchWindowIcons: true,
|
||||
});
|
||||
if (sources.length === 0) {
|
||||
callback({});
|
||||
return;
|
||||
}
|
||||
|
||||
const requestId = randomUUID();
|
||||
const timer = setTimeout(() => {
|
||||
this.finishDisplayMedia(requestId);
|
||||
}, DISPLAY_MEDIA_TIMEOUT_MS);
|
||||
this.pendingDisplayMedia.set(requestId, {
|
||||
callback,
|
||||
timer,
|
||||
sources: new Map(sources.map((source) => [source.id, source])),
|
||||
});
|
||||
|
||||
const request: DisplayMediaRequest = {
|
||||
requestId,
|
||||
sources: sources.map((source) => ({
|
||||
id: source.id,
|
||||
name: source.name,
|
||||
kind: source.id.startsWith('screen:') ? 'screen' as const : 'window' as const,
|
||||
thumbnailDataUrl: source.thumbnail.isEmpty() ? '' : source.thumbnail.toDataURL(),
|
||||
// appIcon is typed non-null but is absent for screen sources.
|
||||
appIconDataUrl: source.appIcon && !source.appIcon.isEmpty() ? source.appIcon.toDataURL() : null,
|
||||
})),
|
||||
};
|
||||
this.emit('display-media-request', request);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a pending display-media request. `sourceId === undefined` (or an
|
||||
* id no longer in the gathered set) denies it. Always notifies the renderer
|
||||
* so a picker it may still be showing (e.g. after a timeout) is pruned.
|
||||
*/
|
||||
private finishDisplayMedia(requestId: string, sourceId?: string, audio?: boolean): boolean {
|
||||
const pending = this.pendingDisplayMedia.get(requestId);
|
||||
if (!pending) return false;
|
||||
this.pendingDisplayMedia.delete(requestId);
|
||||
clearTimeout(pending.timer);
|
||||
const source = sourceId != null ? pending.sources.get(sourceId) : undefined;
|
||||
try {
|
||||
if (!source) {
|
||||
pending.callback({});
|
||||
} else if (audio) {
|
||||
pending.callback({ video: source, audio: 'loopback' });
|
||||
} else {
|
||||
pending.callback({ video: source });
|
||||
}
|
||||
} catch {
|
||||
// The requesting webContents may already be destroyed.
|
||||
}
|
||||
this.emit('display-media-resolved', requestId);
|
||||
return true;
|
||||
}
|
||||
|
||||
respondToDisplayMedia(input: {
|
||||
requestId: string;
|
||||
sourceId?: string;
|
||||
audio?: boolean;
|
||||
}): { ok: boolean } {
|
||||
return { ok: this.finishDisplayMedia(input.requestId, input.sourceId, input.audio) };
|
||||
}
|
||||
|
||||
private emitState(): void {
|
||||
this.emit('state-updated', this.snapshotState());
|
||||
}
|
||||
|
|
@ -560,6 +673,10 @@ export class BrowserViewManager extends EventEmitter {
|
|||
this.invalidateSnapshot(tabId);
|
||||
this.syncAttachedView();
|
||||
this.emitState();
|
||||
// Register with the extension system (chrome.tabs) before the initial
|
||||
// load below so extensions observe the tab's first navigation.
|
||||
this.emit('tab-created', tab.view.webContents, this.window);
|
||||
this.emit('tab-selected', tab.view.webContents);
|
||||
|
||||
const targetUrl =
|
||||
initialUrl === 'about:blank'
|
||||
|
|
@ -733,11 +850,13 @@ export class BrowserViewManager extends EventEmitter {
|
|||
}
|
||||
|
||||
switchTab(tabId: string): { ok: boolean } {
|
||||
if (!this.tabs.has(tabId)) return { ok: false };
|
||||
const tab = this.tabs.get(tabId);
|
||||
if (!tab) return { ok: false };
|
||||
if (this.activeTabId === tabId) return { ok: true };
|
||||
this.activeTabId = tabId;
|
||||
this.syncAttachedView();
|
||||
this.emitState();
|
||||
this.emit('tab-selected', tab.view.webContents);
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
|
|
@ -1048,6 +1167,24 @@ export class BrowserViewManager extends EventEmitter {
|
|||
return this.snapshotState();
|
||||
}
|
||||
|
||||
// Accessors for the extension system's chrome.tabs bridge
|
||||
// (browser/extensions.ts), which speaks in WebContents while the manager
|
||||
// speaks in tab ids.
|
||||
getWindow(): BrowserWindow | null {
|
||||
return this.window;
|
||||
}
|
||||
|
||||
getTabWebContents(tabId: string): WebContents | null {
|
||||
return this.getTab(tabId)?.view.webContents ?? null;
|
||||
}
|
||||
|
||||
getTabIdForWebContents(wc: WebContents): string | null {
|
||||
for (const tab of this.tabs.values()) {
|
||||
if (tab.view.webContents === wc) return tab.id;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private snapshotState(): BrowserState {
|
||||
if (this.tabOrder.length === 0) return { ...EMPTY_STATE };
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -55,6 +55,7 @@ import type { CodeModeManager } from "@x/core/dist/code-mode/acp/manager.js";
|
|||
import type { ISessions } from "@x/core/dist/runtime/sessions/index.js";
|
||||
import { browserViewManager, BROWSER_PARTITION } from "./browser/view.js";
|
||||
import { setupBrowserEventForwarding } from "./browser/ipc.js";
|
||||
import { setupBrowserExtensions } from "./browser/extensions.js";
|
||||
import { ElectronBrowserControlService } from "./browser/control-service.js";
|
||||
import { ElectronNotificationService } from "./notification/electron-notification-service.js";
|
||||
import {
|
||||
|
|
@ -226,18 +227,34 @@ protocol.registerSchemesAsPrivileged([
|
|||
|
||||
const ALLOWED_SESSION_PERMISSIONS = new Set(["media", "display-capture", "clipboard-read", "clipboard-sanitized-write"]);
|
||||
|
||||
function configureSessionPermissions(targetSession: Session): void {
|
||||
// Granted to the embedded browser partition on top of the base set.
|
||||
// `notifications` lets sites (WhatsApp Web, Gmail, Slack, ...) show native OS
|
||||
// notifications via the HTML5 Notification API — Electron renders these
|
||||
// through the system notification center once the permission resolves to
|
||||
// granted. Background Web Push is still unavailable (Electron has no FCM),
|
||||
// so notifications only fire while the site is loaded in a tab. The app's
|
||||
// own renderer keeps the base set; it notifies through the main-process
|
||||
// notification service instead.
|
||||
const BROWSER_EXTRA_PERMISSIONS = ["notifications"] as const;
|
||||
|
||||
function configureSessionPermissions(targetSession: Session, extraPermissions: readonly string[] = []): void {
|
||||
const allowed = new Set([...ALLOWED_SESSION_PERMISSIONS, ...extraPermissions]);
|
||||
|
||||
targetSession.setPermissionCheckHandler((_webContents, permission) => {
|
||||
return ALLOWED_SESSION_PERMISSIONS.has(permission);
|
||||
return allowed.has(permission);
|
||||
});
|
||||
|
||||
targetSession.setPermissionRequestHandler((_webContents, permission, callback) => {
|
||||
callback(ALLOWED_SESSION_PERMISSIONS.has(permission));
|
||||
callback(allowed.has(permission));
|
||||
});
|
||||
}
|
||||
|
||||
// Auto-approve display media requests and route system audio as loopback.
|
||||
// Electron requires a video source in the callback even if we only want audio.
|
||||
// We pass the first available screen source; the renderer discards the video track.
|
||||
// Auto-approve display media requests and route system audio as loopback.
|
||||
// Electron requires a video source in the callback even if we only want audio.
|
||||
// We pass the first available screen source; the renderer discards the video track.
|
||||
// App session only — the embedded browser partition registers its own handler
|
||||
// (a user-facing source picker) in BrowserViewManager.
|
||||
function configureAppDisplayMediaHandler(targetSession: Session): void {
|
||||
targetSession.setDisplayMediaRequestHandler(async (_request, callback) => {
|
||||
const sources = await desktopCapturer.getSources({ types: ['screen'] });
|
||||
if (sources.length === 0) {
|
||||
|
|
@ -346,7 +363,8 @@ function createWindow(options: { startHidden?: boolean } = {}) {
|
|||
});
|
||||
|
||||
configureSessionPermissions(session.defaultSession);
|
||||
configureSessionPermissions(session.fromPartition(BROWSER_PARTITION));
|
||||
configureAppDisplayMediaHandler(session.defaultSession);
|
||||
configureSessionPermissions(session.fromPartition(BROWSER_PARTITION), BROWSER_EXTRA_PERMISSIONS);
|
||||
|
||||
mainWindow = win;
|
||||
setMainWindowForDeepLinks(win);
|
||||
|
|
@ -471,6 +489,7 @@ app.whenReady().then(async () => {
|
|||
|
||||
setupIpcHandlers();
|
||||
setupBrowserEventForwarding();
|
||||
setupBrowserExtensions();
|
||||
|
||||
// Start the Rowboat Apps server (per-app origins on 127.0.0.1:3210) BEFORE
|
||||
// the window and the long service-init chain below. The Apps view is
|
||||
|
|
|
|||
|
|
@ -7,7 +7,8 @@
|
|||
"build": "rm -rf dist && tsc && esbuild dist/preload.js --bundle --platform=node --format=cjs --external:electron --outfile=dist/preload.bundle.js && mv dist/preload.bundle.js dist/preload.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"@x/shared": "workspace:*"
|
||||
"@x/shared": "workspace:*",
|
||||
"electron-chrome-extensions": "^4.9.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"electron": "^39.2.7",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,19 @@
|
|||
import { contextBridge, ipcRenderer, webFrame, webUtils } from 'electron';
|
||||
import { injectBrowserAction } from 'electron-chrome-extensions/browser-action';
|
||||
import { ipc as ipcShared } from '@x/shared';
|
||||
|
||||
// Expose the <browser-action-list> custom element (extension action icons +
|
||||
// popups for the embedded browser pane). App documents only — this preload
|
||||
// is attached solely to the app window, but guard against it ever being
|
||||
// reused for remote content.
|
||||
if (location.protocol === 'app:' || location.origin === 'http://localhost:5173') {
|
||||
try {
|
||||
injectBrowserAction();
|
||||
} catch (error) {
|
||||
console.error('[preload] injectBrowserAction failed:', error);
|
||||
}
|
||||
}
|
||||
|
||||
type InvokeChannels = ipcShared.InvokeChannels;
|
||||
type IPCChannels = ipcShared.IPCChannels;
|
||||
type SendChannels = ipcShared.SendChannels;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,25 @@
|
|||
import { useCallback, useEffect, useRef, useState } from 'react'
|
||||
import { ArrowLeft, ArrowRight, Loader2, Plus, RotateCw, X } from 'lucide-react'
|
||||
|
||||
import type { HttpAuthRequest } from '@x/shared/dist/browser-control.js'
|
||||
// Custom element provided by electron-chrome-extensions (injected via the
|
||||
// preload script): a row of extension action icons for the given session
|
||||
// partition, with popup handling built in. Type names resolve against the
|
||||
// react module's own scope inside this augmentation.
|
||||
declare module 'react' {
|
||||
namespace JSX {
|
||||
interface IntrinsicElements {
|
||||
'browser-action-list': import('react').DetailedHTMLProps<
|
||||
import('react').HTMLAttributes<HTMLElement>,
|
||||
HTMLElement
|
||||
> & {
|
||||
partition?: string
|
||||
alignment?: string
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
import type { DisplayMediaRequest, DisplayMediaSource, HttpAuthRequest } from '@x/shared/dist/browser-control.js'
|
||||
|
||||
import { TabBar } from '@/components/tab-bar'
|
||||
import { Button } from '@/components/ui/button'
|
||||
|
|
@ -168,10 +186,127 @@ function BrowserHttpAuthDialog({
|
|||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Source picker for getDisplayMedia() requests raised by pages in the
|
||||
* embedded browser (e.g. Google Meet "Present now"). Mirrors Chrome's share
|
||||
* dialog: pick a screen or window, optionally include system audio (screens
|
||||
* only — loopback capture is system-wide, so it isn't offered per-window).
|
||||
*/
|
||||
function BrowserScreenSharePickerDialog({
|
||||
request,
|
||||
onSubmit,
|
||||
onCancel,
|
||||
}: {
|
||||
request: DisplayMediaRequest
|
||||
onSubmit: (sourceId: string, audio: boolean) => void
|
||||
onCancel: () => void
|
||||
}) {
|
||||
const [selectedId, setSelectedId] = useState<string | null>(null)
|
||||
const [shareAudio, setShareAudio] = useState(false)
|
||||
|
||||
const screens = request.sources.filter((source) => source.kind === 'screen')
|
||||
const windows = request.sources.filter((source) => source.kind === 'window')
|
||||
const selected = request.sources.find((source) => source.id === selectedId) ?? null
|
||||
const audioAvailable = selected?.kind === 'screen'
|
||||
|
||||
const renderSources = (sources: DisplayMediaSource[]) => (
|
||||
<div className="grid grid-cols-3 gap-2">
|
||||
{sources.map((source) => (
|
||||
<button
|
||||
key={source.id}
|
||||
type="button"
|
||||
onClick={() => setSelectedId(source.id)}
|
||||
className={cn(
|
||||
'flex min-w-0 flex-col gap-1.5 rounded-md border p-1.5 text-left transition-colors',
|
||||
source.id === selectedId
|
||||
? 'border-ring bg-accent'
|
||||
: 'border-border hover:bg-accent/50',
|
||||
)}
|
||||
>
|
||||
<div className="flex h-20 items-center justify-center overflow-hidden rounded bg-muted">
|
||||
{source.thumbnailDataUrl ? (
|
||||
<img
|
||||
src={source.thumbnailDataUrl}
|
||||
alt=""
|
||||
className="max-h-full max-w-full object-contain"
|
||||
/>
|
||||
) : null}
|
||||
</div>
|
||||
<div className="flex min-w-0 items-center gap-1">
|
||||
{source.appIconDataUrl ? (
|
||||
<img src={source.appIconDataUrl} alt="" className="size-3.5 shrink-0" />
|
||||
) : null}
|
||||
<span className="truncate text-xs text-foreground">{source.name}</span>
|
||||
</div>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
)
|
||||
|
||||
return (
|
||||
<Dialog open onOpenChange={(open) => { if (!open) onCancel() }}>
|
||||
<DialogContent className="w-[min(36rem,calc(100%-2rem))] max-w-xl">
|
||||
<DialogHeader>
|
||||
<DialogTitle>Share your screen</DialogTitle>
|
||||
<DialogDescription>
|
||||
Choose what to share with this site.
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
<div className="flex max-h-[50vh] flex-col gap-3 overflow-y-auto pr-1">
|
||||
{screens.length > 0 && (
|
||||
<div className="flex flex-col gap-1.5">
|
||||
<span className="text-xs font-medium text-muted-foreground">Screens</span>
|
||||
{renderSources(screens)}
|
||||
</div>
|
||||
)}
|
||||
{windows.length > 0 && (
|
||||
<div className="flex flex-col gap-1.5">
|
||||
<span className="text-xs font-medium text-muted-foreground">Windows</span>
|
||||
{renderSources(windows)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<DialogFooter className="items-center sm:justify-between">
|
||||
<label
|
||||
className={cn(
|
||||
'flex items-center gap-2 text-xs',
|
||||
audioAvailable ? 'text-foreground' : 'text-muted-foreground/60',
|
||||
)}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={audioAvailable && shareAudio}
|
||||
disabled={!audioAvailable}
|
||||
onChange={(e) => setShareAudio(e.target.checked)}
|
||||
/>
|
||||
Also share system audio
|
||||
</label>
|
||||
<div className="flex gap-2">
|
||||
<Button type="button" variant="outline" onClick={onCancel}>
|
||||
Cancel
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
disabled={!selected}
|
||||
onClick={() => {
|
||||
if (!selected) return
|
||||
onSubmit(selected.id, audioAvailable && shareAudio)
|
||||
}}
|
||||
>
|
||||
Share
|
||||
</Button>
|
||||
</div>
|
||||
</DialogFooter>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
)
|
||||
}
|
||||
|
||||
export function BrowserPane({ onClose, forceHidden = false }: BrowserPaneProps) {
|
||||
const [state, setState] = useState<BrowserState>(EMPTY_STATE)
|
||||
const [addressValue, setAddressValue] = useState('')
|
||||
const [authQueue, setAuthQueue] = useState<HttpAuthRequest[]>([])
|
||||
const [displayMediaQueue, setDisplayMediaQueue] = useState<DisplayMediaRequest[]>([])
|
||||
|
||||
const activeTabIdRef = useRef<string | null>(null)
|
||||
const addressFocusedRef = useRef(false)
|
||||
|
|
@ -249,6 +384,47 @@ export function BrowserPane({ onClose, forceHidden = false }: BrowserPaneProps)
|
|||
|
||||
const activeAuthRequest = authQueue[0] ?? null
|
||||
|
||||
// Same lifecycle as the auth queue: push on request, prune on main-side
|
||||
// resolution (timeout / window teardown), cancel leftovers on unmount so
|
||||
// the main-process callbacks and timers are freed immediately.
|
||||
const displayMediaQueueRef = useRef<DisplayMediaRequest[]>([])
|
||||
useEffect(() => {
|
||||
displayMediaQueueRef.current = displayMediaQueue
|
||||
}, [displayMediaQueue])
|
||||
|
||||
useEffect(() => {
|
||||
const offRequest = window.ipc.on('browser:displayMediaRequest', (incoming) => {
|
||||
setDisplayMediaQueue((queue) => [...queue, incoming as DisplayMediaRequest])
|
||||
})
|
||||
const offResolved = window.ipc.on('browser:displayMediaResolved', (incoming) => {
|
||||
const { requestId } = incoming as { requestId: string }
|
||||
setDisplayMediaQueue((queue) => queue.filter((request) => request.requestId !== requestId))
|
||||
})
|
||||
return () => {
|
||||
offRequest()
|
||||
offResolved()
|
||||
for (const request of displayMediaQueueRef.current) {
|
||||
void window.ipc.invoke('browser:displayMediaResponse', { requestId: request.requestId })
|
||||
}
|
||||
}
|
||||
}, [])
|
||||
|
||||
const respondToDisplayMedia = useCallback(
|
||||
(requestId: string, choice: { sourceId: string; audio: boolean } | null) => {
|
||||
setDisplayMediaQueue((queue) => queue.filter((request) => request.requestId !== requestId))
|
||||
// Omit sourceId to cancel; include it to share the chosen source.
|
||||
void window.ipc.invoke(
|
||||
'browser:displayMediaResponse',
|
||||
choice
|
||||
? { requestId, sourceId: choice.sourceId, audio: choice.audio }
|
||||
: { requestId },
|
||||
)
|
||||
},
|
||||
[],
|
||||
)
|
||||
|
||||
const activeDisplayMediaRequest = displayMediaQueue[0] ?? null
|
||||
|
||||
const setViewVisible = useCallback((visible: boolean) => {
|
||||
if (viewVisibleRef.current === visible) return
|
||||
viewVisibleRef.current = visible
|
||||
|
|
@ -533,6 +709,11 @@ export function BrowserPane({ onClose, forceHidden = false }: BrowserPaneProps)
|
|||
autoCapitalize="off"
|
||||
/>
|
||||
</form>
|
||||
<browser-action-list
|
||||
partition="persist:rowboat-browser"
|
||||
alignment="bottom right"
|
||||
className="ml-1 flex shrink-0 items-center"
|
||||
/>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onClose}
|
||||
|
|
@ -559,6 +740,17 @@ export function BrowserPane({ onClose, forceHidden = false }: BrowserPaneProps)
|
|||
onCancel={() => respondToAuth(activeAuthRequest.requestId, null)}
|
||||
/>
|
||||
)}
|
||||
|
||||
{!activeAuthRequest && activeDisplayMediaRequest && (
|
||||
<BrowserScreenSharePickerDialog
|
||||
key={activeDisplayMediaRequest.requestId}
|
||||
request={activeDisplayMediaRequest}
|
||||
onSubmit={(sourceId, audio) =>
|
||||
respondToDisplayMedia(activeDisplayMediaRequest.requestId, { sourceId, audio })
|
||||
}
|
||||
onCancel={() => respondToDisplayMedia(activeDisplayMediaRequest.requestId, null)}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,6 +23,22 @@ export const HttpAuthRequestSchema = z.object({
|
|||
realm: z.string().optional(),
|
||||
});
|
||||
|
||||
// A screen or window offered to the user when a page in the embedded browser
|
||||
// calls getDisplayMedia() (e.g. "Present now" in Google Meet). Main gathers
|
||||
// these via desktopCapturer and the renderer shows a picker dialog.
|
||||
export const DisplayMediaSourceSchema = z.object({
|
||||
id: z.string(),
|
||||
name: z.string(),
|
||||
kind: z.enum(['screen', 'window']),
|
||||
thumbnailDataUrl: z.string(),
|
||||
appIconDataUrl: z.string().nullable(),
|
||||
});
|
||||
|
||||
export const DisplayMediaRequestSchema = z.object({
|
||||
requestId: z.string(),
|
||||
sources: z.array(DisplayMediaSourceSchema),
|
||||
});
|
||||
|
||||
export const BrowserPageElementSchema = z.object({
|
||||
index: z.number().int().positive(),
|
||||
tagName: z.string(),
|
||||
|
|
@ -144,6 +160,8 @@ export const BrowserControlResultSchema = z.object({
|
|||
export type BrowserTabState = z.infer<typeof BrowserTabStateSchema>;
|
||||
export type BrowserState = z.infer<typeof BrowserStateSchema>;
|
||||
export type HttpAuthRequest = z.infer<typeof HttpAuthRequestSchema>;
|
||||
export type DisplayMediaSource = z.infer<typeof DisplayMediaSourceSchema>;
|
||||
export type DisplayMediaRequest = z.infer<typeof DisplayMediaRequestSchema>;
|
||||
export type BrowserPageElement = z.infer<typeof BrowserPageElementSchema>;
|
||||
export type BrowserPageSnapshot = z.infer<typeof BrowserPageSnapshotSchema>;
|
||||
export type BrowserControlAction = z.infer<typeof BrowserControlActionSchema>;
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ import type { SessionBusEvent, SessionIndexEntry, SessionState } from './session
|
|||
import { RowboatApiConfig } from './rowboat-account.js';
|
||||
import { ZListToolkitsResponse } from './composio.js';
|
||||
import { AppSummarySchema, RegistryRecordSchema, RowboatAppManifestSchema } from './rowboat-app.js';
|
||||
import { BrowserStateSchema, HttpAuthRequestSchema } from './browser-control.js';
|
||||
import { BrowserStateSchema, DisplayMediaRequestSchema, HttpAuthRequestSchema } from './browser-control.js';
|
||||
import { BillingInfoSchema } from './billing.js';
|
||||
import { EmailBlockSchema, GmailThreadSchema } from './blocks.js';
|
||||
import { PermissionDecision, ApprovalPolicy, CodingAgent, type CodeRunFeedEvent } from './code-mode.js';
|
||||
|
|
@ -2303,6 +2303,30 @@ const ipcSchemas = {
|
|||
}),
|
||||
res: z.object({ ok: z.boolean() }),
|
||||
},
|
||||
// Screen-share picker for pages calling getDisplayMedia() in the embedded
|
||||
// browser (main → renderer push). The renderer shows a source picker and
|
||||
// answers via browser:displayMediaResponse.
|
||||
'browser:displayMediaRequest': {
|
||||
req: DisplayMediaRequestSchema,
|
||||
res: z.null(),
|
||||
},
|
||||
// Main → renderer: a pending display-media request was resolved without the
|
||||
// renderer answering (timed out, or the window went away), so the renderer
|
||||
// must drop the corresponding picker dialog.
|
||||
'browser:displayMediaResolved': {
|
||||
req: z.object({ requestId: z.string() }),
|
||||
res: z.null(),
|
||||
},
|
||||
// Renderer → main. Omit sourceId to cancel the request; `audio` asks for
|
||||
// system-audio loopback alongside the shared screen.
|
||||
'browser:displayMediaResponse': {
|
||||
req: z.object({
|
||||
requestId: z.string(),
|
||||
sourceId: z.string().optional(),
|
||||
audio: z.boolean().optional(),
|
||||
}),
|
||||
res: z.object({ ok: z.boolean() }),
|
||||
},
|
||||
// Billing channels
|
||||
'billing:getInfo': {
|
||||
req: z.null(),
|
||||
|
|
|
|||
15
apps/x/pnpm-lock.yaml
generated
15
apps/x/pnpm-lock.yaml
generated
|
|
@ -70,6 +70,9 @@ importers:
|
|||
chokidar:
|
||||
specifier: ^4.0.3
|
||||
version: 4.0.3
|
||||
electron-chrome-extensions:
|
||||
specifier: ^4.9.0
|
||||
version: 4.9.0
|
||||
electron-squirrel-startup:
|
||||
specifier: ^1.0.1
|
||||
version: 1.0.1
|
||||
|
|
@ -143,6 +146,9 @@ importers:
|
|||
'@x/shared':
|
||||
specifier: workspace:*
|
||||
version: link:../../packages/shared
|
||||
electron-chrome-extensions:
|
||||
specifier: ^4.9.0
|
||||
version: 4.9.0
|
||||
devDependencies:
|
||||
electron:
|
||||
specifier: ^39.2.7
|
||||
|
|
@ -5410,6 +5416,9 @@ packages:
|
|||
ee-first@1.1.1:
|
||||
resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==}
|
||||
|
||||
electron-chrome-extensions@4.9.0:
|
||||
resolution: {integrity: sha512-4kLlh4sPPF0ieOy/Gw7A7KrmqB07megDLqOWAWXdFLAYu+2AeOGME6uOBa80NZLbKgTqBvrsVDZ+epdszRm/EQ==}
|
||||
|
||||
electron-installer-common@0.10.4:
|
||||
resolution: {integrity: sha512-8gMNPXfAqUE5CfXg8RL0vXpLE9HAaPkgLXVoHE3BMUzogMWenf4LmwQ27BdCUrEhkjrKl+igs2IHJibclR3z3Q==}
|
||||
engines: {node: '>= 10.0.0'}
|
||||
|
|
@ -15065,6 +15074,12 @@ snapshots:
|
|||
|
||||
ee-first@1.1.1: {}
|
||||
|
||||
electron-chrome-extensions@4.9.0:
|
||||
dependencies:
|
||||
debug: 4.4.3
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
electron-installer-common@0.10.4:
|
||||
dependencies:
|
||||
'@electron/asar': 3.4.1
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue