feat(x): browser screen-share picker, web notifications, Chrome extensions

Three capability upgrades to the embedded browser pane:

- Screen share: getDisplayMedia() in the browser partition now shows a
  Chrome-style source picker (screens/windows with thumbnails, optional
  system audio for screens) instead of silently capturing the primary
  screen. Requests are denied when the pane is hidden. The app's own
  session keeps its auto-loopback handler for meeting transcription.

- Notifications: the browser partition grants the notifications
  permission so sites (WhatsApp Web, Gmail, ...) can post native OS
  notifications while loaded. Background Web Push remains unavailable.

- Extensions: electron-chrome-extensions wired to the browser session.
  Unpacked extensions load from ~/.rowboat/extensions/<name>/; tab
  lifecycle is mirrored into chrome.tabs, and a <browser-action-list>
  toolbar hosts action icons/popups next to the address bar. NOTE: the
  library is GPL-3.0/Patron dual-licensed — licensing decision required
  before shipping in a release build.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Arjun 2026-07-16 09:18:40 +05:30
parent f68f546223
commit 781b16d881
12 changed files with 586 additions and 14 deletions

View file

@ -23,6 +23,22 @@ export const HttpAuthRequestSchema = z.object({
realm: z.string().optional(),
});
// A screen or window offered to the user when a page in the embedded browser
// calls getDisplayMedia() (e.g. "Present now" in Google Meet). Main gathers
// these via desktopCapturer and the renderer shows a picker dialog.
export const DisplayMediaSourceSchema = z.object({
id: z.string(),
name: z.string(),
kind: z.enum(['screen', 'window']),
thumbnailDataUrl: z.string(),
appIconDataUrl: z.string().nullable(),
});
export const DisplayMediaRequestSchema = z.object({
requestId: z.string(),
sources: z.array(DisplayMediaSourceSchema),
});
export const BrowserPageElementSchema = z.object({
index: z.number().int().positive(),
tagName: z.string(),
@ -144,6 +160,8 @@ export const BrowserControlResultSchema = z.object({
export type BrowserTabState = z.infer<typeof BrowserTabStateSchema>;
export type BrowserState = z.infer<typeof BrowserStateSchema>;
export type HttpAuthRequest = z.infer<typeof HttpAuthRequestSchema>;
export type DisplayMediaSource = z.infer<typeof DisplayMediaSourceSchema>;
export type DisplayMediaRequest = z.infer<typeof DisplayMediaRequestSchema>;
export type BrowserPageElement = z.infer<typeof BrowserPageElementSchema>;
export type BrowserPageSnapshot = z.infer<typeof BrowserPageSnapshotSchema>;
export type BrowserControlAction = z.infer<typeof BrowserControlActionSchema>;

View file

@ -19,7 +19,7 @@ import type { SessionBusEvent, SessionIndexEntry, SessionState } from './session
import { RowboatApiConfig } from './rowboat-account.js';
import { ZListToolkitsResponse } from './composio.js';
import { AppSummarySchema, RegistryRecordSchema, RowboatAppManifestSchema } from './rowboat-app.js';
import { BrowserStateSchema, HttpAuthRequestSchema } from './browser-control.js';
import { BrowserStateSchema, DisplayMediaRequestSchema, HttpAuthRequestSchema } from './browser-control.js';
import { BillingInfoSchema } from './billing.js';
import { EmailBlockSchema, GmailThreadSchema } from './blocks.js';
import { PermissionDecision, ApprovalPolicy, CodingAgent, type CodeRunFeedEvent } from './code-mode.js';
@ -2303,6 +2303,30 @@ const ipcSchemas = {
}),
res: z.object({ ok: z.boolean() }),
},
// Screen-share picker for pages calling getDisplayMedia() in the embedded
// browser (main → renderer push). The renderer shows a source picker and
// answers via browser:displayMediaResponse.
'browser:displayMediaRequest': {
req: DisplayMediaRequestSchema,
res: z.null(),
},
// Main → renderer: a pending display-media request was resolved without the
// renderer answering (timed out, or the window went away), so the renderer
// must drop the corresponding picker dialog.
'browser:displayMediaResolved': {
req: z.object({ requestId: z.string() }),
res: z.null(),
},
// Renderer → main. Omit sourceId to cancel the request; `audio` asks for
// system-audio loopback alongside the shared screen.
'browser:displayMediaResponse': {
req: z.object({
requestId: z.string(),
sourceId: z.string().optional(),
audio: z.boolean().optional(),
}),
res: z.object({ ok: z.boolean() }),
},
// Billing channels
'billing:getInfo': {
req: z.null(),