From 7dbfcb72f41cbeebf48d609cf954b335e1889d28 Mon Sep 17 00:00:00 2001 From: arkml <6592213+arkml@users.noreply.github.com> Date: Wed, 15 Apr 2026 13:21:09 +0530 Subject: [PATCH] Browser2 (#495) Add tabbed embedded browser and assistant browser control --- .../apps/main/src/browser/control-service.ts | 243 ++++++ apps/x/apps/main/src/browser/ipc.ts | 81 ++ apps/x/apps/main/src/browser/navigation.ts | 41 + apps/x/apps/main/src/browser/page-scripts.ts | 546 ++++++++++++ apps/x/apps/main/src/browser/view.ts | 797 ++++++++++++++++++ apps/x/apps/main/src/ipc.ts | 3 + apps/x/apps/main/src/main.ts | 59 +- apps/x/apps/preload/src/preload.ts | 5 +- apps/x/apps/renderer/src/App.tsx | 115 ++- .../src/components/ai-elements/tool.tsx | 46 +- .../components/browser-pane/BrowserPane.tsx | 418 +++++++++ .../renderer/src/components/chat-sidebar.tsx | 9 +- apps/x/apps/renderer/src/global.d.ts | 3 +- .../renderer/src/lib/chat-conversation.ts | 191 +++++ .../src/application/assistant/instructions.ts | 9 + .../assistant/skills/browser-control/skill.ts | 106 +++ .../src/application/assistant/skills/index.ts | 7 + .../application/browser-control/service.ts | 8 + .../core/src/application/lib/builtin-tools.ts | 37 +- apps/x/packages/core/src/di/container.ts | 11 +- apps/x/packages/shared/src/browser-control.ts | 134 +++ apps/x/packages/shared/src/index.ts | 1 + apps/x/packages/shared/src/ipc.ts | 82 ++ 23 files changed, 2893 insertions(+), 59 deletions(-) create mode 100644 apps/x/apps/main/src/browser/control-service.ts create mode 100644 apps/x/apps/main/src/browser/ipc.ts create mode 100644 apps/x/apps/main/src/browser/navigation.ts create mode 100644 apps/x/apps/main/src/browser/page-scripts.ts create mode 100644 apps/x/apps/main/src/browser/view.ts create mode 100644 apps/x/apps/renderer/src/components/browser-pane/BrowserPane.tsx create mode 100644 apps/x/packages/core/src/application/assistant/skills/browser-control/skill.ts create mode 100644 apps/x/packages/core/src/application/browser-control/service.ts create mode 100644 apps/x/packages/shared/src/browser-control.ts diff --git a/apps/x/apps/main/src/browser/control-service.ts b/apps/x/apps/main/src/browser/control-service.ts new file mode 100644 index 00000000..b83ea7cb --- /dev/null +++ b/apps/x/apps/main/src/browser/control-service.ts @@ -0,0 +1,243 @@ +import type { IBrowserControlService } from '@x/core/dist/application/browser-control/service.js'; +import type { BrowserControlAction, BrowserControlInput, BrowserControlResult } from '@x/shared/dist/browser-control.js'; +import { browserViewManager } from './view.js'; +import { normalizeNavigationTarget } from './navigation.js'; + +function buildSuccessResult( + action: BrowserControlAction, + message: string, + page?: BrowserControlResult['page'], +): BrowserControlResult { + return { + success: true, + action, + message, + browser: browserViewManager.getState(), + ...(page ? { page } : {}), + }; +} + +function buildErrorResult(action: BrowserControlAction, error: string): BrowserControlResult { + return { + success: false, + action, + error, + browser: browserViewManager.getState(), + }; +} + +export class ElectronBrowserControlService implements IBrowserControlService { + async execute( + input: BrowserControlInput, + ctx?: { signal?: AbortSignal }, + ): Promise { + const signal = ctx?.signal; + + try { + switch (input.action) { + case 'open': { + await browserViewManager.ensureActiveTabReady(signal); + const page = await browserViewManager.readPageSummary(signal, { waitForReady: false }) ?? undefined; + return buildSuccessResult('open', 'Opened a browser session.', page); + } + + case 'get-state': + return buildSuccessResult('get-state', 'Read the current browser state.'); + + case 'new-tab': { + const target = input.target ? normalizeNavigationTarget(input.target) : undefined; + const result = await browserViewManager.newTab(target); + if (!result.ok) { + return buildErrorResult('new-tab', result.error ?? 'Failed to open a new tab.'); + } + await browserViewManager.ensureActiveTabReady(signal); + const page = await browserViewManager.readPageSummary(signal, { waitForReady: false }) ?? undefined; + return buildSuccessResult( + 'new-tab', + target ? `Opened a new tab for ${target}.` : 'Opened a new tab.', + page, + ); + } + + case 'switch-tab': { + const tabId = input.tabId; + if (!tabId) { + return buildErrorResult('switch-tab', 'tabId is required for switch-tab.'); + } + const result = browserViewManager.switchTab(tabId); + if (!result.ok) { + return buildErrorResult('switch-tab', `No browser tab exists with id ${tabId}.`); + } + await browserViewManager.ensureActiveTabReady(signal); + const page = await browserViewManager.readPageSummary(signal, { waitForReady: false }) ?? undefined; + return buildSuccessResult('switch-tab', `Switched to tab ${tabId}.`, page); + } + + case 'close-tab': { + const tabId = input.tabId; + if (!tabId) { + return buildErrorResult('close-tab', 'tabId is required for close-tab.'); + } + const result = browserViewManager.closeTab(tabId); + if (!result.ok) { + return buildErrorResult('close-tab', `Could not close tab ${tabId}.`); + } + await browserViewManager.ensureActiveTabReady(signal); + const page = await browserViewManager.readPageSummary(signal, { waitForReady: false }) ?? undefined; + return buildSuccessResult('close-tab', `Closed tab ${tabId}.`, page); + } + + case 'navigate': { + const rawTarget = input.target; + if (!rawTarget) { + return buildErrorResult('navigate', 'target is required for navigate.'); + } + const target = normalizeNavigationTarget(rawTarget); + const result = await browserViewManager.navigate(target); + if (!result.ok) { + return buildErrorResult('navigate', result.error ?? `Failed to navigate to ${target}.`); + } + await browserViewManager.ensureActiveTabReady(signal); + const page = await browserViewManager.readPageSummary(signal, { waitForReady: false }) ?? undefined; + return buildSuccessResult('navigate', `Navigated to ${target}.`, page); + } + + case 'back': { + const result = browserViewManager.back(); + if (!result.ok) { + return buildErrorResult('back', 'The active tab cannot go back.'); + } + await browserViewManager.ensureActiveTabReady(signal); + const page = await browserViewManager.readPageSummary(signal, { waitForReady: false }) ?? undefined; + return buildSuccessResult('back', 'Went back in the active tab.', page); + } + + case 'forward': { + const result = browserViewManager.forward(); + if (!result.ok) { + return buildErrorResult('forward', 'The active tab cannot go forward.'); + } + await browserViewManager.ensureActiveTabReady(signal); + const page = await browserViewManager.readPageSummary(signal, { waitForReady: false }) ?? undefined; + return buildSuccessResult('forward', 'Went forward in the active tab.', page); + } + + case 'reload': { + browserViewManager.reload(); + await browserViewManager.ensureActiveTabReady(signal); + const page = await browserViewManager.readPageSummary(signal, { waitForReady: false }) ?? undefined; + return buildSuccessResult('reload', 'Reloaded the active tab.', page); + } + + case 'read-page': { + const result = await browserViewManager.readPage( + { + maxElements: input.maxElements, + maxTextLength: input.maxTextLength, + }, + signal, + ); + if (!result.ok || !result.page) { + return buildErrorResult('read-page', result.error ?? 'Failed to read the current page.'); + } + return buildSuccessResult('read-page', 'Read the current page.', result.page); + } + + case 'click': { + const result = await browserViewManager.click( + { + index: input.index, + selector: input.selector, + snapshotId: input.snapshotId, + }, + signal, + ); + if (!result.ok) { + return buildErrorResult('click', result.error ?? 'Failed to click the requested element.'); + } + const page = await browserViewManager.readPageSummary(signal, { waitForReady: false }) ?? undefined; + return buildSuccessResult( + 'click', + result.description ? `Clicked ${result.description}.` : 'Clicked the requested element.', + page, + ); + } + + case 'type': { + const text = input.text; + if (text === undefined) { + return buildErrorResult('type', 'text is required for type.'); + } + const result = await browserViewManager.type( + { + index: input.index, + selector: input.selector, + snapshotId: input.snapshotId, + }, + text, + signal, + ); + if (!result.ok) { + return buildErrorResult('type', result.error ?? 'Failed to type into the requested element.'); + } + const page = await browserViewManager.readPageSummary(signal, { waitForReady: false }) ?? undefined; + return buildSuccessResult( + 'type', + result.description ? `Typed into ${result.description}.` : 'Typed into the requested element.', + page, + ); + } + + case 'press': { + const key = input.key; + if (!key) { + return buildErrorResult('press', 'key is required for press.'); + } + const result = await browserViewManager.press( + key, + { + index: input.index, + selector: input.selector, + snapshotId: input.snapshotId, + }, + signal, + ); + if (!result.ok) { + return buildErrorResult('press', result.error ?? `Failed to press ${key}.`); + } + const page = await browserViewManager.readPageSummary(signal, { waitForReady: false }) ?? undefined; + return buildSuccessResult( + 'press', + result.description ? `Pressed ${result.description}.` : `Pressed ${key}.`, + page, + ); + } + + case 'scroll': { + const result = await browserViewManager.scroll( + input.direction ?? 'down', + input.amount ?? 700, + signal, + ); + if (!result.ok) { + return buildErrorResult('scroll', result.error ?? 'Failed to scroll the page.'); + } + const page = await browserViewManager.readPageSummary(signal, { waitForReady: false }) ?? undefined; + return buildSuccessResult('scroll', `Scrolled ${input.direction ?? 'down'}.`, page); + } + + case 'wait': { + const duration = input.ms ?? 1000; + await browserViewManager.wait(duration, signal); + const page = await browserViewManager.readPageSummary(signal, { waitForReady: false }) ?? undefined; + return buildSuccessResult('wait', `Waited ${duration}ms for the page to settle.`, page); + } + } + } catch (error) { + return buildErrorResult( + input.action, + error instanceof Error ? error.message : 'Browser control failed unexpectedly.', + ); + } + } +} diff --git a/apps/x/apps/main/src/browser/ipc.ts b/apps/x/apps/main/src/browser/ipc.ts new file mode 100644 index 00000000..fa3b1ac1 --- /dev/null +++ b/apps/x/apps/main/src/browser/ipc.ts @@ -0,0 +1,81 @@ +import { BrowserWindow } from 'electron'; +import { ipc } from '@x/shared'; +import { browserViewManager, type BrowserState } from './view.js'; + +type IPCChannels = ipc.IPCChannels; + +type InvokeHandler = ( + event: Electron.IpcMainInvokeEvent, + args: IPCChannels[K]['req'], +) => IPCChannels[K]['res'] | Promise; + +type BrowserHandlers = { + 'browser:setBounds': InvokeHandler<'browser:setBounds'>; + 'browser:setVisible': InvokeHandler<'browser:setVisible'>; + 'browser:newTab': InvokeHandler<'browser:newTab'>; + 'browser:switchTab': InvokeHandler<'browser:switchTab'>; + 'browser:closeTab': InvokeHandler<'browser:closeTab'>; + 'browser:navigate': InvokeHandler<'browser:navigate'>; + 'browser:back': InvokeHandler<'browser:back'>; + 'browser:forward': InvokeHandler<'browser:forward'>; + 'browser:reload': InvokeHandler<'browser:reload'>; + 'browser:getState': InvokeHandler<'browser:getState'>; +}; + +/** + * Browser-specific IPC handlers, exported as a plain object so they can be + * spread into the main `registerIpcHandlers({...})` call in ipc.ts. This + * mirrors the convention of keeping feature handlers flat and namespaced by + * channel prefix (`browser:*`). + */ +export const browserIpcHandlers: BrowserHandlers = { + 'browser:setBounds': async (_event, args) => { + browserViewManager.setBounds(args); + return { ok: true }; + }, + 'browser:setVisible': async (_event, args) => { + browserViewManager.setVisible(args.visible); + return { ok: true }; + }, + 'browser:newTab': async (_event, args) => { + return browserViewManager.newTab(args.url); + }, + 'browser:switchTab': async (_event, args) => { + return browserViewManager.switchTab(args.tabId); + }, + 'browser:closeTab': async (_event, args) => { + return browserViewManager.closeTab(args.tabId); + }, + 'browser:navigate': async (_event, args) => { + return browserViewManager.navigate(args.url); + }, + 'browser:back': async () => { + return browserViewManager.back(); + }, + 'browser:forward': async () => { + return browserViewManager.forward(); + }, + 'browser:reload': async () => { + browserViewManager.reload(); + return { ok: true }; + }, + 'browser:getState': async () => { + return browserViewManager.getState(); + }, +}; + +/** + * Wire the BrowserViewManager's state-updated event to all renderer windows + * as a `browser:didUpdateState` push. Must be called once after the main + * window is created so the manager has a window to attach to. + */ +export function setupBrowserEventForwarding(): void { + browserViewManager.on('state-updated', (state: BrowserState) => { + const windows = BrowserWindow.getAllWindows(); + for (const win of windows) { + if (!win.isDestroyed() && win.webContents) { + win.webContents.send('browser:didUpdateState', state); + } + } + }); +} diff --git a/apps/x/apps/main/src/browser/navigation.ts b/apps/x/apps/main/src/browser/navigation.ts new file mode 100644 index 00000000..ac840956 --- /dev/null +++ b/apps/x/apps/main/src/browser/navigation.ts @@ -0,0 +1,41 @@ +const SEARCH_ENGINE_BASE_URL = 'https://www.google.com/search?q='; + +const HAS_SCHEME_RE = /^[a-z][a-z0-9+.-]*:/i; +const IPV4_HOST_RE = /^\d{1,3}(?:\.\d{1,3}){3}(?::\d+)?(?:\/.*)?$/; +const LOCALHOST_RE = /^localhost(?::\d+)?(?:\/.*)?$/i; +const DOMAIN_LIKE_RE = /^[\w.-]+\.[a-z]{2,}(?::\d+)?(?:\/.*)?$/i; + +export function normalizeNavigationTarget(target: string): string { + const trimmed = target.trim(); + if (!trimmed) { + throw new Error('Navigation target cannot be empty.'); + } + + const lower = trimmed.toLowerCase(); + if ( + lower.startsWith('javascript:') + || lower.startsWith('file://') + || lower.startsWith('chrome://') + || lower.startsWith('chrome-extension://') + ) { + throw new Error('That URL scheme is not allowed in the embedded browser.'); + } + + if (HAS_SCHEME_RE.test(trimmed)) { + return trimmed; + } + + const looksLikeHost = + LOCALHOST_RE.test(trimmed) + || DOMAIN_LIKE_RE.test(trimmed) + || IPV4_HOST_RE.test(trimmed); + + if (looksLikeHost && !/\s/.test(trimmed)) { + const scheme = LOCALHOST_RE.test(trimmed) || IPV4_HOST_RE.test(trimmed) + ? 'http://' + : 'https://'; + return `${scheme}${trimmed}`; + } + + return `${SEARCH_ENGINE_BASE_URL}${encodeURIComponent(trimmed)}`; +} diff --git a/apps/x/apps/main/src/browser/page-scripts.ts b/apps/x/apps/main/src/browser/page-scripts.ts new file mode 100644 index 00000000..fc079327 --- /dev/null +++ b/apps/x/apps/main/src/browser/page-scripts.ts @@ -0,0 +1,546 @@ +import type { BrowserPageElement } from '@x/shared/dist/browser-control.js'; + +const INTERACTABLE_SELECTORS = [ + 'a[href]', + 'button', + 'input', + 'textarea', + 'select', + 'summary', + '[role="button"]', + '[role="link"]', + '[role="tab"]', + '[role="menuitem"]', + '[role="option"]', + '[contenteditable="true"]', + '[tabindex]:not([tabindex="-1"])', +].join(', '); + +const CLICKABLE_TARGET_SELECTORS = [ + 'a[href]', + 'button', + 'summary', + 'label', + 'input', + 'textarea', + 'select', + '[role="button"]', + '[role="link"]', + '[role="tab"]', + '[role="menuitem"]', + '[role="option"]', + '[role="checkbox"]', + '[role="radio"]', + '[role="switch"]', + '[role="menuitemcheckbox"]', + '[role="menuitemradio"]', + '[aria-pressed]', + '[aria-expanded]', + '[aria-checked]', + '[contenteditable="true"]', + '[tabindex]:not([tabindex="-1"])', +].join(', '); + +const DOM_HELPERS_SOURCE = String.raw` +const truncateText = (value, max) => { + const normalized = String(value ?? '').replace(/\s+/g, ' ').trim(); + if (!normalized) return ''; + if (normalized.length <= max) return normalized; + const safeMax = Math.max(0, max - 3); + return normalized.slice(0, safeMax).trim() + '...'; +}; + +const cssEscapeValue = (value) => { + if (typeof CSS !== 'undefined' && typeof CSS.escape === 'function') { + return CSS.escape(value); + } + return String(value).replace(/[^a-zA-Z0-9_-]/g, (char) => '\\' + char); +}; + +const isVisibleElement = (element) => { + if (!(element instanceof Element)) return false; + const style = window.getComputedStyle(element); + if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') { + return false; + } + if (element.getAttribute('aria-hidden') === 'true') return false; + const rect = element.getBoundingClientRect(); + return rect.width > 0 && rect.height > 0; +}; + +const isDisabledElement = (element) => { + if (!(element instanceof Element)) return true; + if (element.getAttribute('aria-disabled') === 'true') return true; + return 'disabled' in element && Boolean(element.disabled); +}; + +const isUselessClickTarget = (element) => ( + element === document.body + || element === document.documentElement +); + +const getElementRole = (element) => { + const explicitRole = element.getAttribute('role'); + if (explicitRole) return explicitRole; + if (element instanceof HTMLAnchorElement) return 'link'; + if (element instanceof HTMLButtonElement) return 'button'; + if (element instanceof HTMLInputElement) return element.type === 'checkbox' ? 'checkbox' : 'input'; + if (element instanceof HTMLTextAreaElement) return 'textbox'; + if (element instanceof HTMLSelectElement) return 'combobox'; + if (element instanceof HTMLElement && element.isContentEditable) return 'textbox'; + return null; +}; + +const getElementType = (element) => { + if (element instanceof HTMLInputElement) return element.type || 'text'; + if (element instanceof HTMLTextAreaElement) return 'textarea'; + if (element instanceof HTMLSelectElement) return 'select'; + if (element instanceof HTMLButtonElement) return 'button'; + if (element instanceof HTMLElement && element.isContentEditable) return 'contenteditable'; + return null; +}; + +const getElementLabel = (element) => { + const ariaLabel = truncateText(element.getAttribute('aria-label') ?? '', 120); + if (ariaLabel) return ariaLabel; + + if ('labels' in element && element.labels && element.labels.length > 0) { + const labelText = truncateText( + Array.from(element.labels).map((label) => label.innerText || label.textContent || '').join(' '), + 120, + ); + if (labelText) return labelText; + } + + if (element.id) { + const label = document.querySelector('label[for="' + cssEscapeValue(element.id) + '"]'); + const labelText = truncateText(label?.textContent ?? '', 120); + if (labelText) return labelText; + } + + const placeholder = truncateText(element.getAttribute('placeholder') ?? '', 120); + if (placeholder) return placeholder; + + const text = truncateText( + element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement + ? element.value + : element.textContent ?? '', + 120, + ); + return text || null; +}; + +const describeElement = (element) => { + const role = getElementRole(element) || element.tagName.toLowerCase(); + const label = getElementLabel(element); + return label ? role + ' "' + label + '"' : role; +}; + +const clampNumber = (value, min, max) => Math.min(Math.max(value, min), max); + +const getAssociatedControl = (element) => { + if (!(element instanceof Element)) return null; + if (element instanceof HTMLLabelElement) return element.control; + const parentLabel = element.closest('label'); + return parentLabel instanceof HTMLLabelElement ? parentLabel.control : null; +}; + +const resolveClickTarget = (element) => { + if (!(element instanceof Element)) return null; + + const clickableAncestor = element.closest(${JSON.stringify(CLICKABLE_TARGET_SELECTORS)}); + const labelAncestor = element.closest('label'); + const associatedControl = getAssociatedControl(element); + const candidates = [clickableAncestor, labelAncestor, associatedControl, element]; + + for (const candidate of candidates) { + if (!(candidate instanceof Element)) continue; + if (isUselessClickTarget(candidate)) continue; + if (!isVisibleElement(candidate)) continue; + if (isDisabledElement(candidate)) continue; + return candidate; + } + + for (const candidate of candidates) { + if (candidate instanceof Element) return candidate; + } + + return null; +}; + +const getVerificationTargetState = (element) => { + if (!(element instanceof Element)) return null; + + const text = truncateText(element.innerText || element.textContent || '', 200); + const activeElement = document.activeElement; + const isActive = + activeElement instanceof Element + ? activeElement === element || element.contains(activeElement) + : false; + + return { + selector: buildUniqueSelector(element), + descriptor: describeElement(element), + text: text || null, + checked: + element instanceof HTMLInputElement && (element.type === 'checkbox' || element.type === 'radio') + ? element.checked + : null, + value: + element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement + ? truncateText(element.value ?? '', 200) + : element instanceof HTMLSelectElement + ? truncateText(element.value ?? '', 200) + : element instanceof HTMLElement && element.isContentEditable + ? truncateText(element.innerText || element.textContent || '', 200) + : null, + selectedIndex: element instanceof HTMLSelectElement ? element.selectedIndex : null, + open: + 'open' in element && typeof element.open === 'boolean' + ? element.open + : null, + disabled: isDisabledElement(element), + active: isActive, + ariaChecked: element.getAttribute('aria-checked'), + ariaPressed: element.getAttribute('aria-pressed'), + ariaExpanded: element.getAttribute('aria-expanded'), + }; +}; + +const getPageVerificationState = () => { + const activeElement = document.activeElement instanceof Element ? document.activeElement : null; + return { + url: window.location.href, + title: document.title || '', + textSample: truncateText(document.body?.innerText || document.body?.textContent || '', 2000), + activeSelector: activeElement ? buildUniqueSelector(activeElement) : null, + }; +}; + +const buildUniqueSelector = (element) => { + if (!(element instanceof Element)) return null; + + if (element.id) { + const idSelector = '#' + cssEscapeValue(element.id); + try { + if (document.querySelectorAll(idSelector).length === 1) return idSelector; + } catch {} + } + + const segments = []; + let current = element; + while (current && current instanceof Element && current !== document.documentElement) { + const tag = current.tagName.toLowerCase(); + if (!tag) break; + + let segment = tag; + const name = current.getAttribute('name'); + if (name) { + const nameSelector = tag + '[name="' + cssEscapeValue(name) + '"]'; + try { + if (document.querySelectorAll(nameSelector).length === 1) { + segments.unshift(nameSelector); + return segments.join(' > '); + } + } catch {} + } + + const parent = current.parentElement; + if (parent) { + const sameTagSiblings = Array.from(parent.children).filter((child) => child.tagName === current.tagName); + const position = sameTagSiblings.indexOf(current) + 1; + segment += ':nth-of-type(' + position + ')'; + } + + segments.unshift(segment); + const selector = segments.join(' > '); + try { + if (document.querySelectorAll(selector).length === 1) return selector; + } catch {} + + current = current.parentElement; + } + + return segments.length > 0 ? segments.join(' > ') : null; +}; +`; + +type RawBrowserPageElement = BrowserPageElement & { + selector: string; +}; + +export type RawBrowserPageSnapshot = { + url: string; + title: string; + loading: boolean; + text: string; + elements: RawBrowserPageElement[]; +}; + +export type ElementTarget = { + index?: number; + selector?: string; + snapshotId?: string; +}; + +export function buildReadPageScript(maxElements: number, maxTextLength: number): string { + return `(() => { + ${DOM_HELPERS_SOURCE} + const candidates = Array.from(document.querySelectorAll(${JSON.stringify(INTERACTABLE_SELECTORS)})); + const elements = []; + const seenSelectors = new Set(); + + for (const candidate of candidates) { + if (!(candidate instanceof Element)) continue; + if (!isVisibleElement(candidate)) continue; + + const selector = buildUniqueSelector(candidate); + if (!selector || seenSelectors.has(selector)) continue; + seenSelectors.add(selector); + + elements.push({ + index: elements.length + 1, + selector, + tagName: candidate.tagName.toLowerCase(), + role: getElementRole(candidate), + type: getElementType(candidate), + label: getElementLabel(candidate), + text: truncateText(candidate.innerText || candidate.textContent || '', 120) || null, + placeholder: truncateText(candidate.getAttribute('placeholder') ?? '', 120) || null, + href: candidate instanceof HTMLAnchorElement ? candidate.href : candidate.getAttribute('href'), + disabled: isDisabledElement(candidate), + }); + + if (elements.length >= ${JSON.stringify(maxElements)}) break; + } + + return { + url: window.location.href, + title: document.title || '', + loading: document.readyState !== 'complete', + text: truncateText(document.body?.innerText || document.body?.textContent || '', ${JSON.stringify(maxTextLength)}), + elements, + }; + })()`; +} + +export function buildClickScript(selector: string): string { + return `(() => { + ${DOM_HELPERS_SOURCE} + const requestedSelector = ${JSON.stringify(selector)}; + if (/^(body|html)$/i.test(requestedSelector.trim())) { + return { + ok: false, + error: 'Refusing to click the page body. Read the page again and target a specific element.', + }; + } + + const element = document.querySelector(requestedSelector); + if (!(element instanceof Element)) { + return { ok: false, error: 'Element not found.' }; + } + if (isUselessClickTarget(element)) { + return { + ok: false, + error: 'Refusing to click the page body. Read the page again and target a specific element.', + }; + } + + const target = resolveClickTarget(element); + if (!(target instanceof Element)) { + return { ok: false, error: 'Could not resolve a clickable target.' }; + } + if (isUselessClickTarget(target)) { + return { + ok: false, + error: 'Resolved click target was too generic. Read the page again and choose a specific control.', + }; + } + if (!isVisibleElement(target)) { + return { ok: false, error: 'Resolved click target is not visible.' }; + } + if (isDisabledElement(target)) { + return { ok: false, error: 'Resolved click target is disabled.' }; + } + + const before = { + page: getPageVerificationState(), + target: getVerificationTargetState(target), + }; + + if (target instanceof HTMLElement) { + target.scrollIntoView({ block: 'center', inline: 'center' }); + target.focus({ preventScroll: true }); + } + + const rect = target.getBoundingClientRect(); + const clientX = clampNumber(rect.left + (rect.width / 2), 1, Math.max(1, window.innerWidth - 1)); + const clientY = clampNumber(rect.top + (rect.height / 2), 1, Math.max(1, window.innerHeight - 1)); + const topElement = document.elementFromPoint(clientX, clientY); + const eventTarget = + topElement instanceof Element && (topElement === target || topElement.contains(target) || target.contains(topElement)) + ? topElement + : target; + + if (eventTarget instanceof HTMLElement) { + eventTarget.focus({ preventScroll: true }); + } + + return { + ok: true, + description: describeElement(target), + clickPoint: { + x: Math.round(clientX), + y: Math.round(clientY), + }, + verification: { + before, + targetSelector: buildUniqueSelector(target) || requestedSelector, + }, + }; + })()`; +} + +export function buildVerifyClickScript(targetSelector: string | null, before: unknown): string { + return `(() => { + ${DOM_HELPERS_SOURCE} + const beforeState = ${JSON.stringify(before)}; + const selector = ${JSON.stringify(targetSelector)}; + const afterPage = getPageVerificationState(); + const afterTarget = selector ? getVerificationTargetState(document.querySelector(selector)) : null; + const beforeTarget = beforeState?.target ?? null; + const reasons = []; + + if (beforeState?.page?.url !== afterPage.url) reasons.push('url changed'); + if (beforeState?.page?.title !== afterPage.title) reasons.push('title changed'); + if (beforeState?.page?.textSample !== afterPage.textSample) reasons.push('page text changed'); + if (beforeState?.page?.activeSelector !== afterPage.activeSelector) reasons.push('focus changed'); + + if (beforeTarget && !afterTarget) { + reasons.push('clicked element disappeared'); + } + + if (beforeTarget && afterTarget) { + if (beforeTarget.checked !== afterTarget.checked) reasons.push('checked state changed'); + if (beforeTarget.value !== afterTarget.value) reasons.push('value changed'); + if (beforeTarget.selectedIndex !== afterTarget.selectedIndex) reasons.push('selection changed'); + if (beforeTarget.open !== afterTarget.open) reasons.push('open state changed'); + if (beforeTarget.disabled !== afterTarget.disabled) reasons.push('disabled state changed'); + if (beforeTarget.active !== afterTarget.active) reasons.push('target focus changed'); + if (beforeTarget.ariaChecked !== afterTarget.ariaChecked) reasons.push('aria-checked changed'); + if (beforeTarget.ariaPressed !== afterTarget.ariaPressed) reasons.push('aria-pressed changed'); + if (beforeTarget.ariaExpanded !== afterTarget.ariaExpanded) reasons.push('aria-expanded changed'); + if (beforeTarget.text !== afterTarget.text) reasons.push('target text changed'); + } + + return { + changed: reasons.length > 0, + reasons, + }; + })()`; +} + +export function buildTypeScript(selector: string, text: string): string { + return `(() => { + ${DOM_HELPERS_SOURCE} + const element = document.querySelector(${JSON.stringify(selector)}); + if (!(element instanceof Element)) { + return { ok: false, error: 'Element not found.' }; + } + if (!isVisibleElement(element)) { + return { ok: false, error: 'Element is not visible.' }; + } + if (isDisabledElement(element)) { + return { ok: false, error: 'Element is disabled.' }; + } + + const nextValue = ${JSON.stringify(text)}; + + const setNativeValue = (target, value) => { + const prototype = Object.getPrototypeOf(target); + const descriptor = Object.getOwnPropertyDescriptor(prototype, 'value'); + if (descriptor && typeof descriptor.set === 'function') { + descriptor.set.call(target, value); + } else { + target.value = value; + } + }; + + if (element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement) { + if (element.readOnly) { + return { ok: false, error: 'Element is read-only.' }; + } + element.scrollIntoView({ block: 'center', inline: 'center' }); + element.focus({ preventScroll: true }); + setNativeValue(element, nextValue); + element.dispatchEvent(new InputEvent('input', { bubbles: true, data: nextValue, inputType: 'insertText' })); + element.dispatchEvent(new Event('change', { bubbles: true })); + return { ok: true, description: describeElement(element) }; + } + + if (element instanceof HTMLElement && element.isContentEditable) { + element.scrollIntoView({ block: 'center', inline: 'center' }); + element.focus({ preventScroll: true }); + element.textContent = nextValue; + element.dispatchEvent(new InputEvent('input', { bubbles: true, data: nextValue, inputType: 'insertText' })); + return { ok: true, description: describeElement(element) }; + } + + return { ok: false, error: 'Element does not accept text input.' }; + })()`; +} + +export function buildFocusScript(selector: string): string { + return `(() => { + ${DOM_HELPERS_SOURCE} + const element = document.querySelector(${JSON.stringify(selector)}); + if (!(element instanceof Element)) { + return { ok: false, error: 'Element not found.' }; + } + if (!isVisibleElement(element)) { + return { ok: false, error: 'Element is not visible.' }; + } + if (element instanceof HTMLElement) { + element.scrollIntoView({ block: 'center', inline: 'center' }); + element.focus({ preventScroll: true }); + } + return { ok: true, description: describeElement(element) }; + })()`; +} + +export function buildScrollScript(offset: number): string { + return `(() => { + window.scrollBy({ top: ${JSON.stringify(offset)}, left: 0, behavior: 'auto' }); + return { ok: true }; + })()`; +} + +export function normalizeKeyCode(key: string): string { + const trimmed = key.trim(); + if (!trimmed) return 'Enter'; + + const aliases: Record = { + esc: 'Escape', + escape: 'Escape', + return: 'Enter', + enter: 'Enter', + tab: 'Tab', + space: 'Space', + ' ': 'Space', + left: 'ArrowLeft', + right: 'ArrowRight', + up: 'ArrowUp', + down: 'ArrowDown', + arrowleft: 'ArrowLeft', + arrowright: 'ArrowRight', + arrowup: 'ArrowUp', + arrowdown: 'ArrowDown', + backspace: 'Backspace', + delete: 'Delete', + }; + + const alias = aliases[trimmed.toLowerCase()]; + if (alias) return alias; + if (trimmed.length === 1) return trimmed.toUpperCase(); + return trimmed[0].toUpperCase() + trimmed.slice(1); +} diff --git a/apps/x/apps/main/src/browser/view.ts b/apps/x/apps/main/src/browser/view.ts new file mode 100644 index 00000000..d319c5fb --- /dev/null +++ b/apps/x/apps/main/src/browser/view.ts @@ -0,0 +1,797 @@ +import { randomUUID } from 'node:crypto'; +import { EventEmitter } from 'node:events'; +import { BrowserWindow, WebContentsView, session, shell, type Session } from 'electron'; +import type { + BrowserPageElement, + BrowserPageSnapshot, + BrowserState, + BrowserTabState, +} from '@x/shared/dist/browser-control.js'; +import { normalizeNavigationTarget } from './navigation.js'; +import { + buildClickScript, + buildFocusScript, + buildReadPageScript, + buildScrollScript, + buildTypeScript, + buildVerifyClickScript, + normalizeKeyCode, + type ElementTarget, + type RawBrowserPageSnapshot, +} from './page-scripts.js'; + +export type { BrowserPageSnapshot, BrowserState, BrowserTabState }; + +/** + * Embedded browser pane implementation. + * + * Each browser tab owns its own WebContentsView. Only the active tab's view is + * attached to the main window at a time, but inactive tabs keep their own page + * history and loaded state in memory so switching tabs feels immediate. + * + * All tabs share one persistent session partition so cookies/localStorage/ + * form-fill state survive app restarts, and the browser surface spoofs a + * standard Chrome UA so sites like Google (OAuth) don't reject it. + */ + +export const BROWSER_PARTITION = 'persist:rowboat-browser'; + +// Claims Chrome 130 on macOS — close enough to recent stable for OAuth servers +// that sniff the UA looking for "real browser" shapes. +const SPOOF_UA = + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36'; + +const HOME_URL = 'https://www.google.com'; +const NAVIGATION_TIMEOUT_MS = 10000; +const POST_ACTION_IDLE_MS = 400; +const POST_ACTION_MAX_ELEMENTS = 25; +const POST_ACTION_MAX_TEXT_LENGTH = 4000; +const DEFAULT_READ_MAX_ELEMENTS = 50; +const DEFAULT_READ_MAX_TEXT_LENGTH = 8000; + +export interface BrowserBounds { + x: number; + y: number; + width: number; + height: number; +} + +type BrowserTab = { + id: string; + view: WebContentsView; + domReadyAt: number | null; + loadError: string | null; +}; + +type CachedSnapshot = { + snapshotId: string; + elements: Array<{ index: number; selector: string }>; +}; + +const EMPTY_STATE: BrowserState = { + activeTabId: null, + tabs: [], +}; + +function abortIfNeeded(signal?: AbortSignal): void { + if (!signal?.aborted) return; + throw signal.reason instanceof Error ? signal.reason : new Error('Browser action aborted'); +} + +async function sleep(ms: number, signal?: AbortSignal): Promise { + if (ms <= 0) return; + abortIfNeeded(signal); + await new Promise((resolve, reject) => { + const abortSignal = signal; + const timer = setTimeout(() => { + abortSignal?.removeEventListener('abort', onAbort); + resolve(); + }, ms); + + const onAbort = () => { + clearTimeout(timer); + abortSignal?.removeEventListener('abort', onAbort); + reject(abortSignal?.reason instanceof Error ? abortSignal.reason : new Error('Browser action aborted')); + }; + + abortSignal?.addEventListener('abort', onAbort, { once: true }); + }); +} + + +export class BrowserViewManager extends EventEmitter { + private window: BrowserWindow | null = null; + private browserSession: Session | null = null; + private tabs = new Map(); + private tabOrder: string[] = []; + private activeTabId: string | null = null; + private attachedTabId: string | null = null; + private visible = false; + private bounds: BrowserBounds = { x: 0, y: 0, width: 0, height: 0 }; + private snapshotCache = new Map(); + + attach(window: BrowserWindow): void { + this.window = window; + window.on('closed', () => { + this.window = null; + this.browserSession = null; + this.tabs.clear(); + this.tabOrder = []; + this.activeTabId = null; + this.attachedTabId = null; + this.visible = false; + this.snapshotCache.clear(); + }); + } + + private getSession(): Session { + if (this.browserSession) return this.browserSession; + const browserSession = session.fromPartition(BROWSER_PARTITION); + browserSession.setUserAgent(SPOOF_UA); + this.browserSession = browserSession; + return browserSession; + } + + private emitState(): void { + this.emit('state-updated', this.snapshotState()); + } + + private getTab(tabId: string | null): BrowserTab | null { + if (!tabId) return null; + return this.tabs.get(tabId) ?? null; + } + + private getActiveTab(): BrowserTab | null { + return this.getTab(this.activeTabId); + } + + private invalidateSnapshot(tabId: string): void { + this.snapshotCache.delete(tabId); + } + + private isEmbeddedTabUrl(url: string): boolean { + return /^https?:\/\//i.test(url) || url === 'about:blank'; + } + + private createView(): WebContentsView { + const view = new WebContentsView({ + webPreferences: { + session: this.getSession(), + contextIsolation: true, + sandbox: true, + nodeIntegration: false, + }, + }); + + view.webContents.setUserAgent(SPOOF_UA); + return view; + } + + private wireEvents(tab: BrowserTab): void { + const { id: tabId, view } = tab; + const wc = view.webContents; + + const reapplyBounds = () => { + if ( + this.attachedTabId === tabId && + this.visible && + this.bounds.width > 0 && + this.bounds.height > 0 + ) { + view.setBounds(this.bounds); + } + }; + + const invalidateAndEmit = () => { + this.invalidateSnapshot(tabId); + this.emitState(); + }; + + wc.on('did-start-navigation', (_event, _url, _isInPlace, isMainFrame) => { + if (isMainFrame !== false) { + tab.domReadyAt = null; + tab.loadError = null; + } + this.invalidateSnapshot(tabId); + reapplyBounds(); + }); + wc.on('did-navigate', () => { reapplyBounds(); invalidateAndEmit(); }); + wc.on('did-navigate-in-page', () => { reapplyBounds(); invalidateAndEmit(); }); + wc.on('did-start-loading', () => { + tab.loadError = null; + this.invalidateSnapshot(tabId); + reapplyBounds(); + this.emitState(); + }); + wc.on('did-stop-loading', () => { reapplyBounds(); invalidateAndEmit(); }); + wc.on('did-finish-load', () => { reapplyBounds(); invalidateAndEmit(); }); + wc.on('dom-ready', () => { + tab.domReadyAt = Date.now(); + reapplyBounds(); + invalidateAndEmit(); + }); + wc.on('did-frame-finish-load', reapplyBounds); + wc.on('did-fail-load', (_event, errorCode, errorDescription, validatedURL, isMainFrame) => { + if (isMainFrame && errorCode !== -3) { + const target = validatedURL || wc.getURL() || 'page'; + tab.loadError = errorDescription + ? `Failed to load ${target}: ${errorDescription}.` + : `Failed to load ${target}.`; + } + reapplyBounds(); + invalidateAndEmit(); + }); + wc.on('page-title-updated', this.emitState.bind(this)); + + wc.setWindowOpenHandler(({ url }) => { + if (this.isEmbeddedTabUrl(url)) { + void this.newTab(url); + } else { + void shell.openExternal(url); + } + return { action: 'deny' }; + }); + } + + private snapshotTabState(tab: BrowserTab): BrowserTabState { + const wc = tab.view.webContents; + return { + id: tab.id, + url: wc.getURL(), + title: wc.getTitle(), + canGoBack: wc.navigationHistory.canGoBack(), + canGoForward: wc.navigationHistory.canGoForward(), + loading: wc.isLoading(), + }; + } + + private syncAttachedView(): void { + if (!this.window) return; + + const contentView = this.window.contentView; + const activeTab = this.getActiveTab(); + + if (!this.visible || !activeTab) { + const attachedTab = this.getTab(this.attachedTabId); + if (attachedTab) { + contentView.removeChildView(attachedTab.view); + } + this.attachedTabId = null; + return; + } + + if (this.attachedTabId && this.attachedTabId !== activeTab.id) { + const attachedTab = this.getTab(this.attachedTabId); + if (attachedTab) { + contentView.removeChildView(attachedTab.view); + } + this.attachedTabId = null; + } + + if (this.attachedTabId !== activeTab.id) { + contentView.addChildView(activeTab.view); + this.attachedTabId = activeTab.id; + } + + if (this.bounds.width > 0 && this.bounds.height > 0) { + activeTab.view.setBounds(this.bounds); + } + } + + private createTab(initialUrl: string): BrowserTab { + if (!this.window) { + throw new Error('BrowserViewManager: no window attached'); + } + + const tabId = randomUUID(); + const tab: BrowserTab = { + id: tabId, + view: this.createView(), + domReadyAt: null, + loadError: null, + }; + + this.wireEvents(tab); + this.tabs.set(tabId, tab); + this.tabOrder.push(tabId); + this.activeTabId = tabId; + this.invalidateSnapshot(tabId); + this.syncAttachedView(); + this.emitState(); + + const targetUrl = + initialUrl === 'about:blank' + ? HOME_URL + : normalizeNavigationTarget(initialUrl); + void tab.view.webContents.loadURL(targetUrl).catch((error) => { + tab.loadError = error instanceof Error + ? error.message + : `Failed to load ${targetUrl}.`; + this.emitState(); + }); + + return tab; + } + + private ensureInitialTab(): BrowserTab { + const activeTab = this.getActiveTab(); + if (activeTab) return activeTab; + return this.createTab(HOME_URL); + } + + private destroyTab(tab: BrowserTab): void { + this.invalidateSnapshot(tab.id); + tab.view.webContents.removeAllListeners(); + if (!tab.view.webContents.isDestroyed()) { + tab.view.webContents.close(); + } + } + + private async waitForWebContentsSettle( + tab: BrowserTab, + signal?: AbortSignal, + idleMs = POST_ACTION_IDLE_MS, + timeoutMs = NAVIGATION_TIMEOUT_MS, + ): Promise { + const wc = tab.view.webContents; + const startedAt = Date.now(); + let sawLoading = wc.isLoading(); + + while (Date.now() - startedAt < timeoutMs) { + abortIfNeeded(signal); + if (wc.isDestroyed()) return; + if (tab.loadError) { + throw new Error(tab.loadError); + } + + if (tab.domReadyAt != null) { + const domReadyForMs = Date.now() - tab.domReadyAt; + const requiredIdleMs = sawLoading ? idleMs : Math.min(idleMs, 200); + if (domReadyForMs >= requiredIdleMs) return; + await sleep(Math.min(100, requiredIdleMs - domReadyForMs), signal); + continue; + } + + if (wc.isLoading()) { + sawLoading = true; + await sleep(100, signal); + continue; + } + + await sleep(sawLoading ? idleMs : Math.min(idleMs, 200), signal); + if (tab.loadError) { + throw new Error(tab.loadError); + } + if (!wc.isLoading() || tab.domReadyAt != null) return; + sawLoading = true; + } + } + + private async executeOnActiveTab( + script: string, + signal?: AbortSignal, + options?: { waitForReady?: boolean }, + ): Promise { + abortIfNeeded(signal); + const activeTab = this.getActiveTab() ?? this.ensureInitialTab(); + if (options?.waitForReady !== false) { + await this.waitForWebContentsSettle(activeTab, signal); + } + abortIfNeeded(signal); + return activeTab.view.webContents.executeJavaScript(script, true) as Promise; + } + + private cacheSnapshot(tabId: string, rawSnapshot: RawBrowserPageSnapshot, loading: boolean): BrowserPageSnapshot { + const snapshotId = randomUUID(); + const elements: BrowserPageElement[] = rawSnapshot.elements.map((element, index) => { + const { selector, ...rest } = element; + void selector; + return { + ...rest, + index: index + 1, + }; + }); + + this.snapshotCache.set(tabId, { + snapshotId, + elements: rawSnapshot.elements.map((element, index) => ({ + index: index + 1, + selector: element.selector, + })), + }); + + return { + snapshotId, + url: rawSnapshot.url, + title: rawSnapshot.title, + loading, + text: rawSnapshot.text, + elements, + }; + } + + private resolveElementSelector(tabId: string, target: ElementTarget): { ok: true; selector: string } | { ok: false; error: string } { + if (target.selector?.trim()) { + return { ok: true, selector: target.selector.trim() }; + } + + if (target.index == null) { + return { ok: false, error: 'Provide an element index or selector.' }; + } + + const cachedSnapshot = this.snapshotCache.get(tabId); + if (!cachedSnapshot) { + return { ok: false, error: 'No page snapshot is available yet. Call read-page first.' }; + } + + if (target.snapshotId && cachedSnapshot.snapshotId !== target.snapshotId) { + return { ok: false, error: 'The page changed since the last read-page call. Call read-page again.' }; + } + + const entry = cachedSnapshot.elements.find((element) => element.index === target.index); + if (!entry) { + return { ok: false, error: `No element found for index ${target.index}.` }; + } + + return { ok: true, selector: entry.selector }; + } + + setVisible(visible: boolean): void { + this.visible = visible; + if (visible) { + this.ensureInitialTab(); + } + this.syncAttachedView(); + } + + setBounds(bounds: BrowserBounds): void { + this.bounds = bounds; + const activeTab = this.getActiveTab(); + if (activeTab && this.attachedTabId === activeTab.id && this.visible) { + activeTab.view.setBounds(bounds); + } + } + + async ensureActiveTabReady(signal?: AbortSignal): Promise { + const activeTab = this.getActiveTab() ?? this.ensureInitialTab(); + await this.waitForWebContentsSettle(activeTab, signal); + } + + async newTab(rawUrl?: string): Promise<{ ok: boolean; tabId?: string; error?: string }> { + try { + const tab = this.createTab(rawUrl?.trim() ? rawUrl : HOME_URL); + return { ok: true, tabId: tab.id }; + } catch (err) { + return { ok: false, error: err instanceof Error ? err.message : String(err) }; + } + } + + switchTab(tabId: string): { ok: boolean } { + if (!this.tabs.has(tabId)) return { ok: false }; + if (this.activeTabId === tabId) return { ok: true }; + this.activeTabId = tabId; + this.syncAttachedView(); + this.emitState(); + return { ok: true }; + } + + closeTab(tabId: string): { ok: boolean } { + const tab = this.tabs.get(tabId); + if (!tab) return { ok: false }; + if (this.tabOrder.length <= 1) return { ok: false }; + + const closingIndex = this.tabOrder.indexOf(tabId); + const nextActiveTabId = + this.activeTabId === tabId + ? this.tabOrder[closingIndex + 1] ?? this.tabOrder[closingIndex - 1] ?? null + : this.activeTabId; + + if (this.attachedTabId === tabId && this.window) { + this.window.contentView.removeChildView(tab.view); + this.attachedTabId = null; + } + + this.tabs.delete(tabId); + this.tabOrder = this.tabOrder.filter((id) => id !== tabId); + this.activeTabId = nextActiveTabId; + this.destroyTab(tab); + this.syncAttachedView(); + this.emitState(); + + return { ok: true }; + } + + async navigate(rawUrl: string): Promise<{ ok: boolean; error?: string }> { + try { + const activeTab = this.getActiveTab() ?? this.ensureInitialTab(); + this.invalidateSnapshot(activeTab.id); + await activeTab.view.webContents.loadURL(normalizeNavigationTarget(rawUrl)); + return { ok: true }; + } catch (err) { + return { ok: false, error: err instanceof Error ? err.message : String(err) }; + } + } + + back(): { ok: boolean } { + const activeTab = this.getActiveTab(); + if (!activeTab) return { ok: false }; + const history = activeTab.view.webContents.navigationHistory; + if (!history.canGoBack()) return { ok: false }; + this.invalidateSnapshot(activeTab.id); + history.goBack(); + return { ok: true }; + } + + forward(): { ok: boolean } { + const activeTab = this.getActiveTab(); + if (!activeTab) return { ok: false }; + const history = activeTab.view.webContents.navigationHistory; + if (!history.canGoForward()) return { ok: false }; + this.invalidateSnapshot(activeTab.id); + history.goForward(); + return { ok: true }; + } + + reload(): void { + const activeTab = this.getActiveTab(); + if (!activeTab) return; + this.invalidateSnapshot(activeTab.id); + activeTab.view.webContents.reload(); + } + + async readPage( + options?: { maxElements?: number; maxTextLength?: number; waitForReady?: boolean }, + signal?: AbortSignal, + ): Promise<{ ok: boolean; page?: BrowserPageSnapshot; error?: string }> { + try { + const activeTab = this.getActiveTab() ?? this.ensureInitialTab(); + const rawSnapshot = await this.executeOnActiveTab( + buildReadPageScript( + options?.maxElements ?? DEFAULT_READ_MAX_ELEMENTS, + options?.maxTextLength ?? DEFAULT_READ_MAX_TEXT_LENGTH, + ), + signal, + { waitForReady: options?.waitForReady }, + ); + return { + ok: true, + page: this.cacheSnapshot(activeTab.id, rawSnapshot, activeTab.view.webContents.isLoading()), + }; + } catch (error) { + return { + ok: false, + error: error instanceof Error ? error.message : 'Failed to read the current page.', + }; + } + } + + async readPageSummary( + signal?: AbortSignal, + options?: { waitForReady?: boolean }, + ): Promise { + const result = await this.readPage( + { + maxElements: POST_ACTION_MAX_ELEMENTS, + maxTextLength: POST_ACTION_MAX_TEXT_LENGTH, + waitForReady: options?.waitForReady, + }, + signal, + ); + return result.ok ? result.page ?? null : null; + } + + async click(target: ElementTarget, signal?: AbortSignal): Promise<{ ok: boolean; error?: string; description?: string }> { + const activeTab = this.getActiveTab(); + if (!activeTab) { + return { ok: false, error: 'No active browser tab is open.' }; + } + + const resolved = this.resolveElementSelector(activeTab.id, target); + if (!resolved.ok) return resolved; + + try { + const result = await this.executeOnActiveTab<{ + ok: boolean; + error?: string; + description?: string; + clickPoint?: { + x: number; + y: number; + }; + verification?: { + before: unknown; + targetSelector: string | null; + }; + }>( + buildClickScript(resolved.selector), + signal, + ); + if (!result.ok) return result; + if (!result.clickPoint) { + return { + ok: false, + error: 'Could not determine where to click on the page.', + }; + } + + this.window?.focus(); + activeTab.view.webContents.focus(); + activeTab.view.webContents.sendInputEvent({ + type: 'mouseMove', + x: result.clickPoint.x, + y: result.clickPoint.y, + movementX: 0, + movementY: 0, + }); + activeTab.view.webContents.sendInputEvent({ + type: 'mouseDown', + x: result.clickPoint.x, + y: result.clickPoint.y, + button: 'left', + clickCount: 1, + }); + activeTab.view.webContents.sendInputEvent({ + type: 'mouseUp', + x: result.clickPoint.x, + y: result.clickPoint.y, + button: 'left', + clickCount: 1, + }); + + this.invalidateSnapshot(activeTab.id); + await this.waitForWebContentsSettle(activeTab, signal); + + if (result.verification) { + const verification = await this.executeOnActiveTab<{ changed: boolean; reasons: string[] }>( + buildVerifyClickScript(result.verification.targetSelector, result.verification.before), + signal, + { waitForReady: false }, + ); + + if (!verification.changed) { + return { + ok: false, + error: 'Click did not change the page state. Target may not be the correct control.', + description: result.description, + }; + } + } + + return result; + } catch (error) { + return { + ok: false, + error: error instanceof Error ? error.message : 'Failed to click the element.', + }; + } + } + + async type(target: ElementTarget, text: string, signal?: AbortSignal): Promise<{ ok: boolean; error?: string; description?: string }> { + const activeTab = this.getActiveTab(); + if (!activeTab) { + return { ok: false, error: 'No active browser tab is open.' }; + } + + const resolved = this.resolveElementSelector(activeTab.id, target); + if (!resolved.ok) return resolved; + + try { + const result = await this.executeOnActiveTab<{ ok: boolean; error?: string; description?: string }>( + buildTypeScript(resolved.selector, text), + signal, + ); + if (!result.ok) return result; + this.invalidateSnapshot(activeTab.id); + await this.waitForWebContentsSettle(activeTab, signal); + return result; + } catch (error) { + return { + ok: false, + error: error instanceof Error ? error.message : 'Failed to type into the element.', + }; + } + } + + async press( + key: string, + target?: ElementTarget, + signal?: AbortSignal, + ): Promise<{ ok: boolean; error?: string; description?: string }> { + const activeTab = this.getActiveTab(); + if (!activeTab) { + return { ok: false, error: 'No active browser tab is open.' }; + } + + let description = 'active element'; + + if (target?.index != null || target?.selector?.trim()) { + const resolved = this.resolveElementSelector(activeTab.id, target); + if (!resolved.ok) return resolved; + + try { + const focusResult = await this.executeOnActiveTab<{ ok: boolean; error?: string; description?: string }>( + buildFocusScript(resolved.selector), + signal, + ); + if (!focusResult.ok) return focusResult; + description = focusResult.description ?? description; + } catch (error) { + return { + ok: false, + error: error instanceof Error ? error.message : 'Failed to focus the element before pressing a key.', + }; + } + } + + try { + const wc = activeTab.view.webContents; + const keyCode = normalizeKeyCode(key); + wc.sendInputEvent({ type: 'keyDown', keyCode }); + if (keyCode.length === 1) { + wc.sendInputEvent({ type: 'char', keyCode }); + } + wc.sendInputEvent({ type: 'keyUp', keyCode }); + + this.invalidateSnapshot(activeTab.id); + await this.waitForWebContentsSettle(activeTab, signal); + + return { + ok: true, + description: `${keyCode} on ${description}`, + }; + } catch (error) { + return { + ok: false, + error: error instanceof Error ? error.message : 'Failed to press the requested key.', + }; + } + } + + async scroll(direction: 'up' | 'down' = 'down', amount = 700, signal?: AbortSignal): Promise<{ ok: boolean; error?: string }> { + const activeTab = this.getActiveTab(); + if (!activeTab) { + return { ok: false, error: 'No active browser tab is open.' }; + } + + try { + const offset = Math.max(1, amount) * (direction === 'up' ? -1 : 1); + const result = await this.executeOnActiveTab<{ ok: boolean; error?: string }>( + buildScrollScript(offset), + signal, + ); + if (!result.ok) return result; + this.invalidateSnapshot(activeTab.id); + await sleep(250, signal); + return result; + } catch (error) { + return { + ok: false, + error: error instanceof Error ? error.message : 'Failed to scroll the page.', + }; + } + } + + async wait(ms = 1000, signal?: AbortSignal): Promise { + await sleep(ms, signal); + const activeTab = this.getActiveTab(); + if (!activeTab) return; + await this.waitForWebContentsSettle(activeTab, signal); + } + + getState(): BrowserState { + return this.snapshotState(); + } + + private snapshotState(): BrowserState { + if (this.tabOrder.length === 0) return { ...EMPTY_STATE }; + return { + activeTabId: this.activeTabId, + tabs: this.tabOrder + .map((tabId) => this.tabs.get(tabId)) + .filter((tab): tab is BrowserTab => tab != null) + .map((tab) => this.snapshotTabState(tab)), + }; + } +} + +export const browserViewManager = new BrowserViewManager(); diff --git a/apps/x/apps/main/src/ipc.ts b/apps/x/apps/main/src/ipc.ts index 5a6e37f0..ec1a0aaa 100644 --- a/apps/x/apps/main/src/ipc.ts +++ b/apps/x/apps/main/src/ipc.ts @@ -52,6 +52,7 @@ import { replaceTrackBlockYaml, deleteTrackBlock, } from '@x/core/dist/knowledge/track/fileops.js'; +import { browserIpcHandlers } from './browser/ipc.js'; /** * Convert markdown to a styled HTML document for PDF/DOCX export. @@ -825,5 +826,7 @@ export function setupIpcHandlers() { 'billing:getInfo': async () => { return await getBillingInfo(); }, + // Embedded browser handlers (WebContentsView + navigation) + ...browserIpcHandlers, }); } diff --git a/apps/x/apps/main/src/main.ts b/apps/x/apps/main/src/main.ts index e8c6ee53..97704225 100644 --- a/apps/x/apps/main/src/main.ts +++ b/apps/x/apps/main/src/main.ts @@ -1,4 +1,4 @@ -import { app, BrowserWindow, desktopCapturer, protocol, net, shell, session } from "electron"; +import { app, BrowserWindow, desktopCapturer, protocol, net, shell, session, type Session } from "electron"; import path from "node:path"; import { setupIpcHandlers, @@ -31,6 +31,10 @@ import started from "electron-squirrel-startup"; import { execSync, exec, execFileSync } from "node:child_process"; import { promisify } from "node:util"; import { init as initChromeSync } from "@x/core/dist/knowledge/chrome-extension/server/server.js"; +import { registerBrowserControlService } from "@x/core/dist/di/container.js"; +import { browserViewManager, BROWSER_PARTITION } from "./browser/view.js"; +import { setupBrowserEventForwarding } from "./browser/ipc.js"; +import { ElectronBrowserControlService } from "./browser/control-service.js"; const execAsync = promisify(exec); @@ -112,6 +116,30 @@ protocol.registerSchemesAsPrivileged([ }, ]); +const ALLOWED_SESSION_PERMISSIONS = new Set(["media", "display-capture"]); + +function configureSessionPermissions(targetSession: Session): void { + targetSession.setPermissionCheckHandler((_webContents, permission) => { + return ALLOWED_SESSION_PERMISSIONS.has(permission); + }); + + targetSession.setPermissionRequestHandler((_webContents, permission, callback) => { + callback(ALLOWED_SESSION_PERMISSIONS.has(permission)); + }); + + // Auto-approve display media requests and route system audio as loopback. + // Electron requires a video source in the callback even if we only want audio. + // We pass the first available screen source; the renderer discards the video track. + targetSession.setDisplayMediaRequestHandler(async (_request, callback) => { + const sources = await desktopCapturer.getSources({ types: ['screen'] }); + if (sources.length === 0) { + callback({}); + return; + } + callback({ video: sources[0], audio: 'loopback' }); + }); +} + function createWindow() { const win = new BrowserWindow({ width: 1280, @@ -131,26 +159,8 @@ function createWindow() { }, }); - // Grant microphone and display-capture permissions - session.defaultSession.setPermissionRequestHandler((_webContents, permission, callback) => { - if (permission === 'media' || permission === 'display-capture') { - callback(true); - } else { - callback(false); - } - }); - - // Auto-approve display media requests and route system audio as loopback. - // Electron requires a video source in the callback even if we only want audio. - // We pass the first available screen source; the renderer discards the video track. - session.defaultSession.setDisplayMediaRequestHandler(async (_request, callback) => { - const sources = await desktopCapturer.getSources({ types: ['screen'] }); - if (sources.length === 0) { - callback({}); - return; - } - callback({ video: sources[0], audio: 'loopback' }); - }); + configureSessionPermissions(session.defaultSession); + configureSessionPermissions(session.fromPartition(BROWSER_PARTITION)); // Show window when content is ready to prevent blank screen win.once("ready-to-show", () => { @@ -175,6 +185,10 @@ function createWindow() { } }); + // Attach the embedded browser pane manager to this window. + // The WebContentsView is created lazily on first `browser:setVisible`. + browserViewManager.attach(win); + if (app.isPackaged) { win.loadURL("app://-/index.html"); } else { @@ -215,7 +229,10 @@ app.whenReady().then(async () => { // Initialize all config files before UI can access them await initConfigs(); + registerBrowserControlService(new ElectronBrowserControlService()); + setupIpcHandlers(); + setupBrowserEventForwarding(); createWindow(); diff --git a/apps/x/apps/preload/src/preload.ts b/apps/x/apps/preload/src/preload.ts index 7d7d53e4..bc69d4bb 100644 --- a/apps/x/apps/preload/src/preload.ts +++ b/apps/x/apps/preload/src/preload.ts @@ -1,4 +1,4 @@ -import { contextBridge, ipcRenderer, webUtils } from 'electron'; +import { contextBridge, ipcRenderer, webFrame, webUtils } from 'electron'; import { ipc as ipcShared } from '@x/shared'; type InvokeChannels = ipcShared.InvokeChannels; @@ -55,4 +55,5 @@ contextBridge.exposeInMainWorld('ipc', ipc); contextBridge.exposeInMainWorld('electronUtils', { getPathForFile: (file: File) => webUtils.getPathForFile(file), -}); \ No newline at end of file + getZoomFactor: () => webFrame.getZoomFactor(), +}); diff --git a/apps/x/apps/renderer/src/App.tsx b/apps/x/apps/renderer/src/App.tsx index 31881fe0..1ada0045 100644 --- a/apps/x/apps/renderer/src/App.tsx +++ b/apps/x/apps/renderer/src/App.tsx @@ -5,7 +5,7 @@ import { RunEvent, ListRunsResponse } from '@x/shared/src/runs.js'; import type { LanguageModelUsage, ToolUIPart } from 'ai'; import './App.css' import z from 'zod'; -import { CheckIcon, LoaderIcon, PanelLeftIcon, Maximize2, Minimize2, ChevronLeftIcon, ChevronRightIcon, SquarePen, SearchIcon, HistoryIcon, RadioIcon, SquareIcon } from 'lucide-react'; +import { CheckIcon, LoaderIcon, PanelLeftIcon, Maximize2, Minimize2, ChevronLeftIcon, ChevronRightIcon, SquarePen, SearchIcon, HistoryIcon, RadioIcon, SquareIcon, Globe } from 'lucide-react'; import { cn } from '@/lib/utils'; import { MarkdownEditor, type MarkdownEditorHandle } from './components/markdown-editor'; import { ChatSidebar } from './components/chat-sidebar'; @@ -57,6 +57,7 @@ import { OnboardingModal } from '@/components/onboarding' import { CommandPalette, type CommandPaletteContext, type CommandPaletteMention } from '@/components/search-dialog' import { TrackModal } from '@/components/track-modal' import { BackgroundTaskDetail } from '@/components/background-task-detail' +import { BrowserPane } from '@/components/browser-pane/BrowserPane' import { VersionHistoryPanel } from '@/components/version-history-panel' import { FileCardProvider } from '@/contexts/file-card-context' import { MarkdownPreOverride } from '@/components/ai-elements/markdown-code-override' @@ -449,23 +450,36 @@ function viewStatesEqual(a: ViewState, b: ViewState): boolean { /** Sidebar toggle + utility buttons (fixed position, top-left) */ function FixedSidebarToggle({ + onNavigateBack, + onNavigateForward, + canNavigateBack, + canNavigateForward, onNewChat, onOpenSearch, meetingState, meetingSummarizing, meetingAvailable, onToggleMeeting, + isBrowserOpen, + onToggleBrowser, leftInsetPx, }: { + onNavigateBack: () => void + onNavigateForward: () => void + canNavigateBack: boolean + canNavigateForward: boolean onNewChat: () => void onOpenSearch: () => void meetingState: MeetingTranscriptionState meetingSummarizing: boolean meetingAvailable: boolean onToggleMeeting: () => void + isBrowserOpen: boolean + onToggleBrowser: () => void leftInsetPx: number }) { - const { toggleSidebar } = useSidebar() + const { toggleSidebar, state } = useSidebar() + const isCollapsed = state === "collapsed" return (
) } @@ -606,6 +663,7 @@ function App() { const [expandedPaths, setExpandedPaths] = useState>(new Set()) const [recentWikiFiles, setRecentWikiFiles] = useState([]) const [isGraphOpen, setIsGraphOpen] = useState(false) + const [isBrowserOpen, setIsBrowserOpen] = useState(false) const [expandedFrom, setExpandedFrom] = useState<{ path: string | null; graph: boolean } | null>(null) const [baseConfigByPath, setBaseConfigByPath] = useState>({}) const [graphData, setGraphData] = useState<{ nodes: GraphNode[]; edges: GraphEdge[] }>({ @@ -2714,6 +2772,24 @@ function App() { setIsChatSidebarOpen(prev => !prev) }, []) + // Browser is an overlay on the middle pane: opening it forces the chat + // sidebar to be visible on the right; closing it restores whatever the + // middle pane was showing previously (file/graph/task/chat). + const handleToggleBrowser = useCallback(() => { + setIsBrowserOpen(prev => { + const next = !prev + if (next) { + setIsChatSidebarOpen(true) + setIsRightPaneMaximized(false) + } + return next + }) + }, []) + + const handleCloseBrowser = useCallback(() => { + setIsBrowserOpen(false) + }, []) + const toggleRightPaneMaximize = useCallback(() => { setIsChatSidebarOpen(true) setIsRightPaneMaximized(prev => !prev) @@ -2797,6 +2873,9 @@ function App() { case 'file': setSelectedBackgroundTask(null) setIsGraphOpen(false) + // Navigating to a file dismisses the browser overlay so the file is + // visible in the middle pane. + setIsBrowserOpen(false) setExpandedFrom(null) // Preserve split vs knowledge-max mode when navigating knowledge files. // Only exit chat-only maximize, because that would hide the selected file. @@ -2809,6 +2888,7 @@ function App() { case 'graph': setSelectedBackgroundTask(null) setSelectedPath(null) + setIsBrowserOpen(false) setExpandedFrom(null) setIsGraphOpen(true) ensureGraphFileTab() @@ -2819,6 +2899,7 @@ function App() { case 'task': setSelectedPath(null) setIsGraphOpen(false) + setIsBrowserOpen(false) setExpandedFrom(null) setIsRightPaneMaximized(false) setSelectedBackgroundTask(view.name) @@ -2826,6 +2907,8 @@ function App() { case 'chat': setSelectedPath(null) setIsGraphOpen(false) + // Don't touch isBrowserOpen here — chat navigation should land in + // the right sidebar when the browser overlay is active. setExpandedFrom(null) setIsRightPaneMaximized(false) setSelectedBackgroundTask(null) @@ -3101,7 +3184,7 @@ function App() { }, []) // Keyboard shortcut: Ctrl+L to toggle main chat view - const isFullScreenChat = !selectedPath && !isGraphOpen && !selectedBackgroundTask + const isFullScreenChat = !selectedPath && !isGraphOpen && !selectedBackgroundTask && !isBrowserOpen useEffect(() => { const handleKeyDown = (e: KeyboardEvent) => { if ((e.ctrlKey || e.metaKey) && e.key === 'l') { @@ -3964,7 +4047,7 @@ function App() { const selectedTask = selectedBackgroundTask ? backgroundTasks.find(t => t.name === selectedBackgroundTask) : null - const isRightPaneContext = Boolean(selectedPath || isGraphOpen) + const isRightPaneContext = Boolean(selectedPath || isGraphOpen || isBrowserOpen) const isRightPaneOnlyMode = isRightPaneContext && isChatSidebarOpen && isRightPaneMaximized const shouldCollapseLeftPane = isRightPaneOnlyMode const openMarkdownTabs = React.useMemo(() => { @@ -4006,7 +4089,7 @@ function App() { onNewChat: handleNewChatTab, onSelectRun: (runIdToLoad) => { cancelRecordingIfActive() - if (selectedPath || isGraphOpen) { + if (selectedPath || isGraphOpen || isBrowserOpen) { setIsChatSidebarOpen(true) } @@ -4016,8 +4099,8 @@ function App() { switchChatTab(existingTab.id) return } - // In two-pane mode, keep current knowledge/graph context and just swap chat context. - if (selectedPath || isGraphOpen) { + // In two-pane mode (file/graph/browser), keep the middle pane and just swap chat context in the right sidebar. + if (selectedPath || isGraphOpen || isBrowserOpen) { setChatTabs(prev => prev.map(t => t.id === activeChatTabId ? { ...t, runId: runIdToLoad } : t)) loadRun(runIdToLoad) return @@ -4041,14 +4124,14 @@ function App() { } else { // Only one tab, reset it to new chat setChatTabs([{ id: tabForRun.id, runId: null }]) - if (selectedPath || isGraphOpen) { + if (selectedPath || isGraphOpen || isBrowserOpen) { handleNewChat() } else { void navigateToView({ type: 'chat', runId: null }) } } } else if (runId === runIdToDelete) { - if (selectedPath || isGraphOpen) { + if (selectedPath || isGraphOpen || isBrowserOpen) { setChatTabs(prev => prev.map(t => t.id === activeChatTabId ? { ...t, runId: null } : t)) handleNewChat() } else { @@ -4146,7 +4229,7 @@ function App() { Version history )} - {!selectedPath && !isGraphOpen && !selectedTask && ( + {!selectedPath && !isGraphOpen && !selectedTask && !isBrowserOpen && ( +
+ +
+ + + +
+ setAddressValue(e.target.value)} + onFocus={(e) => { + addressFocusedRef.current = true + e.currentTarget.select() + }} + onBlur={() => { + addressFocusedRef.current = false + setAddressValue(activeTab?.url ?? '') + }} + placeholder="Enter URL or search..." + className={cn( + 'h-7 w-full rounded-md border border-transparent bg-background px-3 text-sm text-foreground', + 'placeholder:text-muted-foreground/60', + 'focus:border-border focus:outline-hidden', + )} + spellCheck={false} + autoCorrect="off" + autoCapitalize="off" + /> +
+ +
+ +
+
+ ) +} diff --git a/apps/x/apps/renderer/src/components/chat-sidebar.tsx b/apps/x/apps/renderer/src/components/chat-sidebar.tsx index f94c94ba..e51d7c8f 100644 --- a/apps/x/apps/renderer/src/components/chat-sidebar.tsx +++ b/apps/x/apps/renderer/src/components/chat-sidebar.tsx @@ -76,12 +76,18 @@ function matchBillingError(message: string) { return BILLING_ERROR_PATTERNS.find(({ pattern }) => pattern.test(message)) ?? null } +interface BillingRowboatAccount { + config?: { + appUrl?: string | null + } | null +} + function BillingErrorCTA({ label }: { label: string }) { const [appUrl, setAppUrl] = useState(null) useEffect(() => { window.ipc.invoke('account:getRowboat', null) - .then((account: any) => setAppUrl(account.config?.appUrl ?? null)) + .then((account: BillingRowboatAccount) => setAppUrl(account.config?.appUrl ?? null)) .catch(() => {}) }, []) @@ -467,6 +473,7 @@ export function ChatSidebar({ return (
string; + getZoomFactor: () => number; }; } } -export { }; \ No newline at end of file +export { }; diff --git a/apps/x/apps/renderer/src/lib/chat-conversation.ts b/apps/x/apps/renderer/src/lib/chat-conversation.ts index 68c8366d..693961c9 100644 --- a/apps/x/apps/renderer/src/lib/chat-conversation.ts +++ b/apps/x/apps/renderer/src/lib/chat-conversation.ts @@ -231,6 +231,194 @@ export const getAppActionCardData = (tool: ToolCall): AppActionCardData | null = } } +const BROWSER_PENDING_LABELS: Record = { + open: 'Opening browser...', + 'get-state': 'Reading browser state...', + 'new-tab': 'Opening new browser tab...', + 'switch-tab': 'Switching browser tab...', + 'close-tab': 'Closing browser tab...', + navigate: 'Navigating browser...', + back: 'Going back...', + forward: 'Going forward...', + reload: 'Reloading page...', + 'read-page': 'Reading page...', + click: 'Clicking page element...', + type: 'Typing into page...', + press: 'Sending key press...', + scroll: 'Scrolling page...', + wait: 'Waiting for page...', +} + +const truncateLabel = (value: string, max = 72): string => { + const normalized = value.replace(/\s+/g, ' ').trim() + if (normalized.length <= max) return normalized + return `${normalized.slice(0, Math.max(0, max - 3)).trim()}...` +} + +const safeBrowserString = (value: unknown): string | null => { + return typeof value === 'string' && value.trim() ? value.trim() : null +} + +const parseBrowserUrl = (value: string | null): URL | null => { + if (!value) return null + try { + return new URL(value) + } catch { + return null + } +} + +const getGoogleSearchQuery = (value: string | null): string | null => { + const parsed = parseBrowserUrl(value) + if (!parsed) return null + const hostname = parsed.hostname.replace(/^www\./, '') + if (hostname !== 'google.com' && !hostname.endsWith('.google.com')) return null + if (parsed.pathname !== '/search') return null + const query = parsed.searchParams.get('q')?.trim() + return query ? truncateLabel(query, 56) : null +} + +const formatBrowserTarget = (value: string | null): string | null => { + const parsed = parseBrowserUrl(value) + if (!parsed) { + return value ? truncateLabel(value, 56) : null + } + + const hostname = parsed.hostname.replace(/^www\./, '') + const path = parsed.pathname === '/' ? '' : parsed.pathname + const suffix = parsed.search ? `${path}${parsed.search}` : path + return truncateLabel(`${hostname}${suffix}`, 56) +} + +const sanitizeBrowserDescription = (value: string | null): string | null => { + if (!value) return null + + let text = value + .replace(/^(clicked|typed into|pressed)\s+/i, '') + .replace(/\.$/, '') + .replace(/\s+/g, ' ') + .trim() + + if (!text) return null + + const looksLikeCssNoise = + /(^|[\s"])(body|html)\b/i.test(text) + || /display:|position:|background-color|align-items|justify-content|z-index|var\(--|left:|top:/i.test(text) + || /\.[A-Za-z0-9_-]+\{/.test(text) + + if (looksLikeCssNoise || text.length > 88) { + const quoted = Array.from(text.matchAll(/"([^"]+)"/g)) + .map((match) => match[1]?.trim()) + .find((candidate) => candidate && !/display:|position:|background-color|var\(--/i.test(candidate)) + + if (!quoted) return null + text = `"${truncateLabel(quoted, 44)}"` + } + + if (/^(body|html)\b/i.test(text)) return null + return truncateLabel(text, 64) +} + +const getBrowserSuccessLabel = ( + action: string, + input: Record | undefined, + result: Record | undefined, +): string | null => { + const page = result?.page as Record | undefined + const pageUrl = safeBrowserString(page?.url) + const resultMessage = safeBrowserString(result?.message) + + switch (action) { + case 'open': + return 'Opened browser' + case 'get-state': + return 'Read browser state' + case 'new-tab': { + const query = getGoogleSearchQuery(pageUrl) + if (query) return `Opened search for "${query}"` + const target = formatBrowserTarget(pageUrl) || safeBrowserString(input?.target) + return target ? `Opened ${target}` : 'Opened new tab' + } + case 'switch-tab': + return 'Switched browser tab' + case 'close-tab': + return 'Closed browser tab' + case 'navigate': { + const query = getGoogleSearchQuery(pageUrl) + if (query) return `Searched Google for "${query}"` + const target = formatBrowserTarget(pageUrl) || formatBrowserTarget(safeBrowserString(input?.target)) + return target ? `Opened ${target}` : 'Navigated browser' + } + case 'back': + return 'Went back' + case 'forward': + return 'Went forward' + case 'reload': + return 'Reloaded page' + case 'read-page': { + const title = safeBrowserString(page?.title) + return title ? `Read ${truncateLabel(title, 52)}` : 'Read page' + } + case 'click': { + const detail = sanitizeBrowserDescription(resultMessage) + if (detail) return `Clicked ${detail}` + if (typeof input?.index === 'number') return `Clicked element ${input.index}` + return 'Clicked page element' + } + case 'type': { + const detail = sanitizeBrowserDescription(resultMessage) + if (detail) return `Typed into ${detail}` + if (typeof input?.index === 'number') return `Typed into element ${input.index}` + return 'Typed into page' + } + case 'press': { + const key = safeBrowserString(input?.key) + return key ? `Pressed ${truncateLabel(key, 20)}` : 'Sent key press' + } + case 'scroll': + return `Scrolled ${input?.direction === 'up' ? 'up' : 'down'}` + case 'wait': { + const ms = typeof input?.ms === 'number' ? input.ms : 1000 + return `Waited ${ms}ms` + } + default: + return resultMessage ? truncateLabel(resultMessage, 72) : 'Controlled browser' + } +} + +export const getBrowserControlLabel = (tool: ToolCall): string | null => { + if (tool.name !== 'browser-control') return null + + const input = normalizeToolInput(tool.input) as Record | undefined + const result = tool.result as Record | undefined + const action = (input?.action as string | undefined) || (result?.action as string | undefined) || 'browser' + + if (tool.status !== 'completed') { + if (action === 'click' && typeof input?.index === 'number') { + return `Clicking element ${input.index}...` + } + if (action === 'type' && typeof input?.index === 'number') { + return `Typing into element ${input.index}...` + } + if (action === 'navigate' && typeof input?.target === 'string') { + return `Navigating to ${input.target}...` + } + return BROWSER_PENDING_LABELS[action] || 'Controlling browser...' + } + + if (result?.success === false) { + const error = safeBrowserString(result.error) + return error ? `Browser error: ${truncateLabel(error, 84)}` : 'Browser action failed' + } + + const label = getBrowserSuccessLabel(action, input, result) + if (label) { + return label + } + + return 'Controlled browser' +} + // Parse attached files from message content and return clean message + file paths. export const parseAttachedFiles = (content: string): { message: string; files: string[] } => { const attachedFilesRegex = /\s*([\s\S]*?)\s*<\/attached-files>/ @@ -315,6 +503,7 @@ const TOOL_DISPLAY_NAMES: Record = { 'web-search': 'Searching the web', 'save-to-memory': 'Saving to memory', 'app-navigation': 'Navigating app', + 'browser-control': 'Controlling browser', 'composio-list-toolkits': 'Listing integrations', 'composio-search-tools': 'Searching tools', 'composio-execute-tool': 'Running tool', @@ -328,6 +517,8 @@ const TOOL_DISPLAY_NAMES: Record = { * Falls back to the raw tool name if no mapping exists. */ export const getToolDisplayName = (tool: ToolCall): string => { + const browserLabel = getBrowserControlLabel(tool) + if (browserLabel) return browserLabel const composioData = getComposioActionCardData(tool) if (composioData) return composioData.label return TOOL_DISPLAY_NAMES[tool.name] || tool.name diff --git a/apps/x/packages/core/src/application/assistant/instructions.ts b/apps/x/packages/core/src/application/assistant/instructions.ts index 31619366..32d22fd9 100644 --- a/apps/x/packages/core/src/application/assistant/instructions.ts +++ b/apps/x/packages/core/src/application/assistant/instructions.ts @@ -71,6 +71,7 @@ Rowboat is an agentic assistant for everyday work - emails, meetings, projects, **App Control:** When users ask you to open notes, show the bases or graph view, filter or search notes, or manage saved views, load the \`app-navigation\` skill first. It provides structured guidance for navigating the app UI and controlling the knowledge base view. **Tracks (Auto-Updating Note Blocks):** When users ask you to **track**, **monitor**, **watch**, or **keep an eye on** something in a note — or say things like "every morning tell me X", "show the current Y in this note", "pin live updates of Z here" — load the \`tracks\` skill first. Also load it when a user presses Cmd+K with a note open and requests auto-refreshing content at the cursor. Track blocks are YAML-fenced scheduled blocks whose output is rewritten on each run — useful for weather, news, prices, status pages, and personal dashboards. +**Browser Control:** When users ask you to open a website, browse in-app, search the web in the embedded browser, or interact with a live webpage inside Rowboat, load the \`browser-control\` skill first. It explains the \`read-page -> indexed action -> refreshed page\` workflow for the browser pane. ## Learning About the User (save-to-memory) @@ -243,6 +244,7 @@ ${runtimeContextPrompt} - \`slack-checkConnection\`, \`slack-listAvailableTools\`, \`slack-executeAction\` - Slack integration (requires Slack to be connected via Composio). Use \`slack-listAvailableTools\` first to discover available tool slugs, then \`slack-executeAction\` to execute them. - \`web-search\` - Search the web. Returns rich results with full text, highlights, and metadata. The \`category\` parameter defaults to \`general\` (full web search) — only use a specific category like \`news\`, \`company\`, \`research paper\` etc. when the query is clearly about that type. For everyday queries (weather, restaurants, prices, how-to), use \`general\`. - \`app-navigation\` - Control the app UI: open notes, switch views, filter/search the knowledge base, manage saved views. **Load the \`app-navigation\` skill before using this tool.** +- \`browser-control\` - Control the embedded browser pane: open sites, inspect the live page, switch tabs, and interact with indexed page elements. **Load the \`browser-control\` skill before using this tool.** - \`save-to-memory\` - Save observations about the user to the agent memory system. Use this proactively during conversations. - \`composio-list-toolkits\`, \`composio-search-tools\`, \`composio-execute-tool\`, \`composio-connect-toolkit\` — Composio integration tools. Load the \`composio-integration\` skill for usage guidance. @@ -281,6 +283,13 @@ This renders as an interactive card in the UI that the user can click to open th - Files on the user's machine (~/Desktop/..., /Users/..., etc.) - Audio files, images, documents, or any file reference +Do NOT use filepath blocks for: +- Website URLs or browser pages (\`https://...\`, \`http://...\`) +- Anything currently open in the embedded browser +- Browser tabs or browser tab ids + +For browser pages, mention the URL in plain text or use the browser-control tool. Do not try to turn browser pages into clickable file cards. + **IMPORTANT:** Only use filepath blocks for files that already exist. The card is clickable and opens the file, so it must point to a real file. If you are proposing a path for a file that hasn't been created yet (e.g., "Shall I save it at ~/Documents/report.pdf?"), use inline code (\`~/Documents/report.pdf\`) instead of a filepath block. Use the filepath block only after the file has been written/created successfully. Never output raw file paths in plain text when they could be wrapped in a filepath block — unless the file does not exist yet.`; diff --git a/apps/x/packages/core/src/application/assistant/skills/browser-control/skill.ts b/apps/x/packages/core/src/application/assistant/skills/browser-control/skill.ts new file mode 100644 index 00000000..f1c06f0c --- /dev/null +++ b/apps/x/packages/core/src/application/assistant/skills/browser-control/skill.ts @@ -0,0 +1,106 @@ +export const skill = String.raw` +# Browser Control Skill + +You have access to the **browser-control** tool, which controls Rowboat's embedded browser pane directly. + +Use this skill when the user asks you to open a website, browse in-app, search the web in the browser pane, click something on a page, fill a form, or otherwise interact with a live webpage inside Rowboat. + +## Core Workflow + +1. Start with ` + "`browser-control({ action: \"open\" })`" + ` if the browser pane may not already be open. +2. Use ` + "`browser-control({ action: \"read-page\" })`" + ` to inspect the current page. +3. The tool returns: + - ` + "`snapshotId`" + ` + - page ` + "`url`" + ` and ` + "`title`" + ` + - visible page text + - interactable elements with numbered ` + "`index`" + ` values +4. Prefer acting on those numbered indices with ` + "`click`" + ` / ` + "`type`" + ` / ` + "`press`" + `. +5. After each action, read the returned page snapshot before deciding the next step. + +## Actions + +### open +Open the browser pane and ensure an active tab exists. + +### get-state +Return the current browser tabs and active tab id. + +### new-tab +Open a new browser tab. + +Parameters: +- ` + "`target`" + ` (optional): URL or plain-language search query + +### switch-tab +Switch to a tab by ` + "`tabId`" + `. + +### close-tab +Close a tab by ` + "`tabId`" + `. + +### navigate +Navigate the active tab. + +Parameters: +- ` + "`target`" + `: URL or plain-language search query + +Plain-language targets are converted into a search automatically. + +### back / forward / reload +Standard browser navigation controls. + +### read-page +Read the current page and return a compact snapshot. + +Parameters: +- ` + "`maxElements`" + ` (optional) +- ` + "`maxTextLength`" + ` (optional) + +### click +Click an element. + +Prefer: +- ` + "`index`" + `: element index from ` + "`read-page`" + ` + +Optional: +- ` + "`snapshotId`" + `: include it when acting on a recent snapshot +- ` + "`selector`" + `: fallback only when no usable index exists + +### type +Type into an input, textarea, or contenteditable element. + +Parameters: +- ` + "`text`" + `: text to enter +- plus the same target fields as ` + "`click`" + ` + +### press +Send a key press such as ` + "`Enter`" + `, ` + "`Tab`" + `, ` + "`Escape`" + `, or arrow keys. + +Parameters: +- ` + "`key`" + ` +- optional target fields if you need to focus a specific element first + +### scroll +Scroll the current page. + +Parameters: +- ` + "`direction`" + `: ` + "`\"up\"`" + ` or ` + "`\"down\"`" + ` (optional; defaults down) +- ` + "`amount`" + `: pixel distance (optional) + +### wait +Wait for the page to settle, useful after async UI changes. + +Parameters: +- ` + "`ms`" + `: milliseconds to wait (optional) + +## Important Rules + +- Prefer ` + "`read-page`" + ` before interacting. +- Prefer element ` + "`index`" + ` over CSS selectors. +- If the tool says the snapshot is stale, call ` + "`read-page`" + ` again. +- After navigation, clicking, typing, pressing, or scrolling, use the returned page snapshot instead of assuming the page state. +- Use Rowboat's browser for live interaction. Use web search tools for research where a live session is unnecessary. +- Do not wrap browser URLs or browser pages in ` + "```filepath" + ` blocks. Filepath cards are only for real files on disk, not web pages or browser tabs. +- If you mention a page the browser opened, use plain text for the URL/title instead of trying to create a clickable file card. +`; + +export default skill; diff --git a/apps/x/packages/core/src/application/assistant/skills/index.ts b/apps/x/packages/core/src/application/assistant/skills/index.ts index 18f29b62..d22db680 100644 --- a/apps/x/packages/core/src/application/assistant/skills/index.ts +++ b/apps/x/packages/core/src/application/assistant/skills/index.ts @@ -11,6 +11,7 @@ import backgroundAgentsSkill from "./background-agents/skill.js"; import createPresentationsSkill from "./create-presentations/skill.js"; import appNavigationSkill from "./app-navigation/skill.js"; +import browserControlSkill from "./browser-control/skill.js"; import composioIntegrationSkill from "./composio-integration/skill.js"; import tracksSkill from "./tracks/skill.js"; @@ -105,6 +106,12 @@ const definitions: SkillDefinition[] = [ summary: "Create and manage track blocks — YAML-scheduled auto-updating content blocks in notes (weather, news, prices, status, dashboards). Insert at cursor (Cmd+K) or append to notes.", content: tracksSkill, }, + { + id: "browser-control", + title: "Browser Control", + summary: "Control the embedded browser pane - open sites, inspect page state, and interact with indexed page elements.", + content: browserControlSkill, + }, ]; const skillEntries = definitions.map((definition) => ({ diff --git a/apps/x/packages/core/src/application/browser-control/service.ts b/apps/x/packages/core/src/application/browser-control/service.ts new file mode 100644 index 00000000..201160c3 --- /dev/null +++ b/apps/x/packages/core/src/application/browser-control/service.ts @@ -0,0 +1,8 @@ +import type { BrowserControlInput, BrowserControlResult } from '@x/shared/dist/browser-control.js'; + +export interface IBrowserControlService { + execute( + input: BrowserControlInput, + ctx?: { signal?: AbortSignal }, + ): Promise; +} diff --git a/apps/x/packages/core/src/application/lib/builtin-tools.ts b/apps/x/packages/core/src/application/lib/builtin-tools.ts index fcad4f32..a2b68427 100644 --- a/apps/x/packages/core/src/application/lib/builtin-tools.ts +++ b/apps/x/packages/core/src/application/lib/builtin-tools.ts @@ -17,6 +17,7 @@ import { WorkDir } from "../../config/config.js"; import { composioAccountsRepo } from "../../composio/repo.js"; import { executeAction as executeComposioAction, isConfigured as isComposioConfigured, searchTools as searchComposioTools } from "../../composio/client.js"; import { CURATED_TOOLKITS, CURATED_TOOLKIT_SLUGS } from "@x/shared/dist/composio.js"; +import { BrowserControlInputSchema, type BrowserControlInput } from "@x/shared/dist/browser-control.js"; import type { ToolContext } from "./exec-tool.js"; import { generateText } from "ai"; import { createProvider } from "../../models/models.js"; @@ -26,6 +27,7 @@ import { getGatewayProvider } from "../../models/gateway.js"; import { getAccessToken } from "../../auth/tokens.js"; import { API_URL } from "../../config/env.js"; import { updateContent, updateTrackBlock } from "../../knowledge/track/fileops.js"; +import type { IBrowserControlService } from "../browser-control/service.js"; // Parser libraries are loaded dynamically inside parseFile.execute() // to avoid pulling pdfjs-dist's DOM polyfills into the main bundle. // Import paths are computed so esbuild cannot statically resolve them. @@ -562,7 +564,7 @@ export const BuiltinTools: z.infer = { count: matches.length, tool: 'ripgrep', }; - } catch (rgError) { + } catch { // Fallback to basic grep if ripgrep not available or failed const grepArgs = [ '-rn', @@ -997,6 +999,39 @@ export const BuiltinTools: z.infer = { }, }, + // ============================================================================ + // Browser Control + // ============================================================================ + + 'browser-control': { + description: 'Control the embedded browser pane. Read the current page, inspect indexed interactable elements, and navigate/click/type/press keys in the active browser tab.', + inputSchema: BrowserControlInputSchema, + isAvailable: async () => { + try { + container.resolve('browserControlService'); + return true; + } catch { + return false; + } + }, + execute: async (input: BrowserControlInput, ctx?: ToolContext) => { + try { + const browserControlService = container.resolve('browserControlService'); + return await browserControlService.execute(input, { signal: ctx?.signal }); + } catch (error) { + return { + success: false, + action: input.action, + error: error instanceof Error ? error.message : 'Browser control is unavailable.', + browser: { + activeTabId: null, + tabs: [], + }, + }; + } + }, + }, + // ============================================================================ // App Navigation // ============================================================================ diff --git a/apps/x/packages/core/src/di/container.ts b/apps/x/packages/core/src/di/container.ts index d7a9b9a0..93ba9ebd 100644 --- a/apps/x/packages/core/src/di/container.ts +++ b/apps/x/packages/core/src/di/container.ts @@ -1,4 +1,4 @@ -import { asClass, createContainer, InjectionMode } from "awilix"; +import { asClass, asValue, createContainer, InjectionMode } from "awilix"; import { FSModelConfigRepo, IModelConfigRepo } from "../models/repo.js"; import { FSMcpConfigRepo, IMcpConfigRepo } from "../mcp/repo.js"; import { FSAgentsRepo, IAgentsRepo } from "../agents/repo.js"; @@ -15,6 +15,7 @@ import { IAbortRegistry, InMemoryAbortRegistry } from "../runs/abort-registry.js import { FSAgentScheduleRepo, IAgentScheduleRepo } from "../agent-schedule/repo.js"; import { FSAgentScheduleStateRepo, IAgentScheduleStateRepo } from "../agent-schedule/state-repo.js"; import { FSSlackConfigRepo, ISlackConfigRepo } from "../slack/repo.js"; +import type { IBrowserControlService } from "../application/browser-control/service.js"; const container = createContainer({ injectionMode: InjectionMode.PROXY, @@ -41,4 +42,10 @@ container.register({ slackConfigRepo: asClass(FSSlackConfigRepo).singleton(), }); -export default container; \ No newline at end of file +export default container; + +export function registerBrowserControlService(service: IBrowserControlService): void { + container.register({ + browserControlService: asValue(service), + }); +} diff --git a/apps/x/packages/shared/src/browser-control.ts b/apps/x/packages/shared/src/browser-control.ts new file mode 100644 index 00000000..e1418a5e --- /dev/null +++ b/apps/x/packages/shared/src/browser-control.ts @@ -0,0 +1,134 @@ +import { z } from 'zod'; + +export const BrowserTabStateSchema = z.object({ + id: z.string(), + url: z.string(), + title: z.string(), + canGoBack: z.boolean(), + canGoForward: z.boolean(), + loading: z.boolean(), +}); + +export const BrowserStateSchema = z.object({ + activeTabId: z.string().nullable(), + tabs: z.array(BrowserTabStateSchema), +}); + +export const BrowserPageElementSchema = z.object({ + index: z.number().int().positive(), + tagName: z.string(), + role: z.string().nullable(), + type: z.string().nullable(), + label: z.string().nullable(), + text: z.string().nullable(), + placeholder: z.string().nullable(), + href: z.string().nullable(), + disabled: z.boolean(), +}); + +export const BrowserPageSnapshotSchema = z.object({ + snapshotId: z.string(), + url: z.string(), + title: z.string(), + loading: z.boolean(), + text: z.string(), + elements: z.array(BrowserPageElementSchema), +}); + +export const BrowserControlActionSchema = z.enum([ + 'open', + 'get-state', + 'new-tab', + 'switch-tab', + 'close-tab', + 'navigate', + 'back', + 'forward', + 'reload', + 'read-page', + 'click', + 'type', + 'press', + 'scroll', + 'wait', +]); + +const BrowserElementTargetFields = { + index: z.number().int().positive().optional(), + selector: z.string().min(1).optional(), + snapshotId: z.string().optional(), +} as const; + +export const BrowserControlInputSchema = z.object({ + action: BrowserControlActionSchema, + target: z.string().min(1).optional(), + tabId: z.string().min(1).optional(), + text: z.string().optional(), + key: z.string().min(1).optional(), + direction: z.enum(['up', 'down']).optional(), + amount: z.number().int().positive().max(5000).optional(), + ms: z.number().int().positive().max(30000).optional(), + maxElements: z.number().int().positive().max(100).optional(), + maxTextLength: z.number().int().positive().max(20000).optional(), + ...BrowserElementTargetFields, +}).strict().superRefine((value, ctx) => { + const needsElementTarget = value.action === 'click' || value.action === 'type'; + const hasElementTarget = value.index !== undefined || value.selector !== undefined; + + if ((value.action === 'switch-tab' || value.action === 'close-tab') && !value.tabId) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['tabId'], + message: 'tabId is required for this action.', + }); + } + + if ((value.action === 'navigate') && !value.target) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['target'], + message: 'target is required for navigate.', + }); + } + + if (value.action === 'type' && value.text === undefined) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['text'], + message: 'text is required for type.', + }); + } + + if (value.action === 'press' && !value.key) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['key'], + message: 'key is required for press.', + }); + } + + if (needsElementTarget && !hasElementTarget) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['index'], + message: 'Provide an element index or selector.', + }); + } +}); + +export const BrowserControlResultSchema = z.object({ + success: z.boolean(), + action: BrowserControlActionSchema, + message: z.string().optional(), + error: z.string().optional(), + browser: BrowserStateSchema, + page: BrowserPageSnapshotSchema.optional(), +}); + +export type BrowserTabState = z.infer; +export type BrowserState = z.infer; +export type BrowserPageElement = z.infer; +export type BrowserPageSnapshot = z.infer; +export type BrowserControlAction = z.infer; +export type BrowserControlInput = z.infer; +export type BrowserControlResult = z.infer; diff --git a/apps/x/packages/shared/src/index.ts b/apps/x/packages/shared/src/index.ts index 8bdee4f9..5bdc49fd 100644 --- a/apps/x/packages/shared/src/index.ts +++ b/apps/x/packages/shared/src/index.ts @@ -12,4 +12,5 @@ export * as blocks from './blocks.js'; export * as trackBlock from './track-block.js'; export * as frontmatter from './frontmatter.js'; export * as bases from './bases.js'; +export * as browserControl from './browser-control.js'; export { PrefixLogger }; diff --git a/apps/x/packages/shared/src/ipc.ts b/apps/x/packages/shared/src/ipc.ts index 1ba5fce0..f0de64af 100644 --- a/apps/x/packages/shared/src/ipc.ts +++ b/apps/x/packages/shared/src/ipc.ts @@ -10,6 +10,7 @@ import { TrackEvent } from './track-block.js'; import { UserMessageContent } from './message.js'; import { RowboatApiConfig } from './rowboat-account.js'; import { ZListToolkitsResponse } from './composio.js'; +import { BrowserStateSchema } from './browser-control.js'; // ============================================================================ // Runtime Validation Schemas (Single Source of Truth) @@ -626,6 +627,87 @@ const ipcSchemas = { error: z.string().optional(), }), }, + // Embedded browser (WebContentsView) channels + 'browser:setBounds': { + req: z.object({ + x: z.number().int(), + y: z.number().int(), + width: z.number().int().nonnegative(), + height: z.number().int().nonnegative(), + }), + res: z.object({ ok: z.literal(true) }), + }, + 'browser:setVisible': { + req: z.object({ visible: z.boolean() }), + res: z.object({ ok: z.literal(true) }), + }, + 'browser:newTab': { + req: z.object({ + url: z.string().min(1).refine( + (u) => { + const lower = u.trim().toLowerCase(); + if (lower.startsWith('javascript:')) return false; + if (lower.startsWith('file://')) return false; + if (lower.startsWith('chrome://')) return false; + if (lower.startsWith('chrome-extension://')) return false; + return true; + }, + { message: 'Unsafe URL scheme' }, + ).optional(), + }), + res: z.object({ + ok: z.boolean(), + tabId: z.string().optional(), + error: z.string().optional(), + }), + }, + 'browser:switchTab': { + req: z.object({ tabId: z.string().min(1) }), + res: z.object({ ok: z.boolean() }), + }, + 'browser:closeTab': { + req: z.object({ tabId: z.string().min(1) }), + res: z.object({ ok: z.boolean() }), + }, + 'browser:navigate': { + req: z.object({ + url: z.string().min(1).refine( + (u) => { + const lower = u.trim().toLowerCase(); + if (lower.startsWith('javascript:')) return false; + if (lower.startsWith('file://')) return false; + if (lower.startsWith('chrome://')) return false; + if (lower.startsWith('chrome-extension://')) return false; + return true; + }, + { message: 'Unsafe URL scheme' }, + ), + }), + res: z.object({ + ok: z.boolean(), + error: z.string().optional(), + }), + }, + 'browser:back': { + req: z.null(), + res: z.object({ ok: z.boolean() }), + }, + 'browser:forward': { + req: z.null(), + res: z.object({ ok: z.boolean() }), + }, + 'browser:reload': { + req: z.null(), + res: z.object({ ok: z.literal(true) }), + }, + 'browser:getState': { + req: z.null(), + res: BrowserStateSchema, + }, + 'browser:didUpdateState': { + req: BrowserStateSchema, + res: z.null(), + }, // Billing channels 'billing:getInfo': { req: z.null(),