mirror of
https://github.com/rowboatlabs/rowboat.git
synced 2026-04-25 00:16:29 +02:00
Browser2 (#495)
Add tabbed embedded browser and assistant browser control
This commit is contained in:
arkml
GitHub
parent
e2c13f0f6f
commit
7dbfcb72f4
23 changed files with 2893 additions and 59 deletions
243
apps/x/apps/main/src/browser/control-service.ts
Normal file
243
apps/x/apps/main/src/browser/control-service.ts
Normal file
|
|
@ -0,0 +1,243 @@
|
|||
import type { IBrowserControlService } from '@x/core/dist/application/browser-control/service.js';
|
||||
import type { BrowserControlAction, BrowserControlInput, BrowserControlResult } from '@x/shared/dist/browser-control.js';
|
||||
import { browserViewManager } from './view.js';
|
||||
import { normalizeNavigationTarget } from './navigation.js';
|
||||
|
||||
function buildSuccessResult(
|
||||
action: BrowserControlAction,
|
||||
message: string,
|
||||
page?: BrowserControlResult['page'],
|
||||
): BrowserControlResult {
|
||||
return {
|
||||
success: true,
|
||||
action,
|
||||
message,
|
||||
browser: browserViewManager.getState(),
|
||||
...(page ? { page } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function buildErrorResult(action: BrowserControlAction, error: string): BrowserControlResult {
|
||||
return {
|
||||
success: false,
|
||||
action,
|
||||
error,
|
||||
browser: browserViewManager.getState(),
|
||||
};
|
||||
}
|
||||
|
||||
export class ElectronBrowserControlService implements IBrowserControlService {
|
||||
async execute(
|
||||
input: BrowserControlInput,
|
||||
ctx?: { signal?: AbortSignal },
|
||||
): Promise<BrowserControlResult> {
|
||||
const signal = ctx?.signal;
|
||||
|
||||
try {
|
||||
switch (input.action) {
|
||||
case 'open': {
|
||||
await browserViewManager.ensureActiveTabReady(signal);
|
||||
const page = await browserViewManager.readPageSummary(signal, { waitForReady: false }) ?? undefined;
|
||||
return buildSuccessResult('open', 'Opened a browser session.', page);
|
||||
}
|
||||
|
||||
case 'get-state':
|
||||
return buildSuccessResult('get-state', 'Read the current browser state.');
|
||||
|
||||
case 'new-tab': {
|
||||
const target = input.target ? normalizeNavigationTarget(input.target) : undefined;
|
||||
const result = await browserViewManager.newTab(target);
|
||||
if (!result.ok) {
|
||||
return buildErrorResult('new-tab', result.error ?? 'Failed to open a new tab.');
|
||||
}
|
||||
await browserViewManager.ensureActiveTabReady(signal);
|
||||
const page = await browserViewManager.readPageSummary(signal, { waitForReady: false }) ?? undefined;
|
||||
return buildSuccessResult(
|
||||
'new-tab',
|
||||
target ? `Opened a new tab for ${target}.` : 'Opened a new tab.',
|
||||
page,
|
||||
);
|
||||
}
|
||||
|
||||
case 'switch-tab': {
|
||||
const tabId = input.tabId;
|
||||
if (!tabId) {
|
||||
return buildErrorResult('switch-tab', 'tabId is required for switch-tab.');
|
||||
}
|
||||
const result = browserViewManager.switchTab(tabId);
|
||||
if (!result.ok) {
|
||||
return buildErrorResult('switch-tab', `No browser tab exists with id ${tabId}.`);
|
||||
}
|
||||
await browserViewManager.ensureActiveTabReady(signal);
|
||||
const page = await browserViewManager.readPageSummary(signal, { waitForReady: false }) ?? undefined;
|
||||
return buildSuccessResult('switch-tab', `Switched to tab ${tabId}.`, page);
|
||||
}
|
||||
|
||||
case 'close-tab': {
|
||||
const tabId = input.tabId;
|
||||
if (!tabId) {
|
||||
return buildErrorResult('close-tab', 'tabId is required for close-tab.');
|
||||
}
|
||||
const result = browserViewManager.closeTab(tabId);
|
||||
if (!result.ok) {
|
||||
return buildErrorResult('close-tab', `Could not close tab ${tabId}.`);
|
||||
}
|
||||
await browserViewManager.ensureActiveTabReady(signal);
|
||||
const page = await browserViewManager.readPageSummary(signal, { waitForReady: false }) ?? undefined;
|
||||
return buildSuccessResult('close-tab', `Closed tab ${tabId}.`, page);
|
||||
}
|
||||
|
||||
case 'navigate': {
|
||||
const rawTarget = input.target;
|
||||
if (!rawTarget) {
|
||||
return buildErrorResult('navigate', 'target is required for navigate.');
|
||||
}
|
||||
const target = normalizeNavigationTarget(rawTarget);
|
||||
const result = await browserViewManager.navigate(target);
|
||||
if (!result.ok) {
|
||||
return buildErrorResult('navigate', result.error ?? `Failed to navigate to ${target}.`);
|
||||
}
|
||||
await browserViewManager.ensureActiveTabReady(signal);
|
||||
const page = await browserViewManager.readPageSummary(signal, { waitForReady: false }) ?? undefined;
|
||||
return buildSuccessResult('navigate', `Navigated to ${target}.`, page);
|
||||
}
|
||||
|
||||
case 'back': {
|
||||
const result = browserViewManager.back();
|
||||
if (!result.ok) {
|
||||
return buildErrorResult('back', 'The active tab cannot go back.');
|
||||
}
|
||||
await browserViewManager.ensureActiveTabReady(signal);
|
||||
const page = await browserViewManager.readPageSummary(signal, { waitForReady: false }) ?? undefined;
|
||||
return buildSuccessResult('back', 'Went back in the active tab.', page);
|
||||
}
|
||||
|
||||
case 'forward': {
|
||||
const result = browserViewManager.forward();
|
||||
if (!result.ok) {
|
||||
return buildErrorResult('forward', 'The active tab cannot go forward.');
|
||||
}
|
||||
await browserViewManager.ensureActiveTabReady(signal);
|
||||
const page = await browserViewManager.readPageSummary(signal, { waitForReady: false }) ?? undefined;
|
||||
return buildSuccessResult('forward', 'Went forward in the active tab.', page);
|
||||
}
|
||||
|
||||
case 'reload': {
|
||||
browserViewManager.reload();
|
||||
await browserViewManager.ensureActiveTabReady(signal);
|
||||
const page = await browserViewManager.readPageSummary(signal, { waitForReady: false }) ?? undefined;
|
||||
return buildSuccessResult('reload', 'Reloaded the active tab.', page);
|
||||
}
|
||||
|
||||
case 'read-page': {
|
||||
const result = await browserViewManager.readPage(
|
||||
{
|
||||
maxElements: input.maxElements,
|
||||
maxTextLength: input.maxTextLength,
|
||||
},
|
||||
signal,
|
||||
);
|
||||
if (!result.ok || !result.page) {
|
||||
return buildErrorResult('read-page', result.error ?? 'Failed to read the current page.');
|
||||
}
|
||||
return buildSuccessResult('read-page', 'Read the current page.', result.page);
|
||||
}
|
||||
|
||||
case 'click': {
|
||||
const result = await browserViewManager.click(
|
||||
{
|
||||
index: input.index,
|
||||
selector: input.selector,
|
||||
snapshotId: input.snapshotId,
|
||||
},
|
||||
signal,
|
||||
);
|
||||
if (!result.ok) {
|
||||
return buildErrorResult('click', result.error ?? 'Failed to click the requested element.');
|
||||
}
|
||||
const page = await browserViewManager.readPageSummary(signal, { waitForReady: false }) ?? undefined;
|
||||
return buildSuccessResult(
|
||||
'click',
|
||||
result.description ? `Clicked ${result.description}.` : 'Clicked the requested element.',
|
||||
page,
|
||||
);
|
||||
}
|
||||
|
||||
case 'type': {
|
||||
const text = input.text;
|
||||
if (text === undefined) {
|
||||
return buildErrorResult('type', 'text is required for type.');
|
||||
}
|
||||
const result = await browserViewManager.type(
|
||||
{
|
||||
index: input.index,
|
||||
selector: input.selector,
|
||||
snapshotId: input.snapshotId,
|
||||
},
|
||||
text,
|
||||
signal,
|
||||
);
|
||||
if (!result.ok) {
|
||||
return buildErrorResult('type', result.error ?? 'Failed to type into the requested element.');
|
||||
}
|
||||
const page = await browserViewManager.readPageSummary(signal, { waitForReady: false }) ?? undefined;
|
||||
return buildSuccessResult(
|
||||
'type',
|
||||
result.description ? `Typed into ${result.description}.` : 'Typed into the requested element.',
|
||||
page,
|
||||
);
|
||||
}
|
||||
|
||||
case 'press': {
|
||||
const key = input.key;
|
||||
if (!key) {
|
||||
return buildErrorResult('press', 'key is required for press.');
|
||||
}
|
||||
const result = await browserViewManager.press(
|
||||
key,
|
||||
{
|
||||
index: input.index,
|
||||
selector: input.selector,
|
||||
snapshotId: input.snapshotId,
|
||||
},
|
||||
signal,
|
||||
);
|
||||
if (!result.ok) {
|
||||
return buildErrorResult('press', result.error ?? `Failed to press ${key}.`);
|
||||
}
|
||||
const page = await browserViewManager.readPageSummary(signal, { waitForReady: false }) ?? undefined;
|
||||
return buildSuccessResult(
|
||||
'press',
|
||||
result.description ? `Pressed ${result.description}.` : `Pressed ${key}.`,
|
||||
page,
|
||||
);
|
||||
}
|
||||
|
||||
case 'scroll': {
|
||||
const result = await browserViewManager.scroll(
|
||||
input.direction ?? 'down',
|
||||
input.amount ?? 700,
|
||||
signal,
|
||||
);
|
||||
if (!result.ok) {
|
||||
return buildErrorResult('scroll', result.error ?? 'Failed to scroll the page.');
|
||||
}
|
||||
const page = await browserViewManager.readPageSummary(signal, { waitForReady: false }) ?? undefined;
|
||||
return buildSuccessResult('scroll', `Scrolled ${input.direction ?? 'down'}.`, page);
|
||||
}
|
||||
|
||||
case 'wait': {
|
||||
const duration = input.ms ?? 1000;
|
||||
await browserViewManager.wait(duration, signal);
|
||||
const page = await browserViewManager.readPageSummary(signal, { waitForReady: false }) ?? undefined;
|
||||
return buildSuccessResult('wait', `Waited ${duration}ms for the page to settle.`, page);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
return buildErrorResult(
|
||||
input.action,
|
||||
error instanceof Error ? error.message : 'Browser control failed unexpectedly.',
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
81
apps/x/apps/main/src/browser/ipc.ts
Normal file
81
apps/x/apps/main/src/browser/ipc.ts
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
import { BrowserWindow } from 'electron';
|
||||
import { ipc } from '@x/shared';
|
||||
import { browserViewManager, type BrowserState } from './view.js';
|
||||
|
||||
type IPCChannels = ipc.IPCChannels;
|
||||
|
||||
type InvokeHandler<K extends keyof IPCChannels> = (
|
||||
event: Electron.IpcMainInvokeEvent,
|
||||
args: IPCChannels[K]['req'],
|
||||
) => IPCChannels[K]['res'] | Promise<IPCChannels[K]['res']>;
|
||||
|
||||
type BrowserHandlers = {
|
||||
'browser:setBounds': InvokeHandler<'browser:setBounds'>;
|
||||
'browser:setVisible': InvokeHandler<'browser:setVisible'>;
|
||||
'browser:newTab': InvokeHandler<'browser:newTab'>;
|
||||
'browser:switchTab': InvokeHandler<'browser:switchTab'>;
|
||||
'browser:closeTab': InvokeHandler<'browser:closeTab'>;
|
||||
'browser:navigate': InvokeHandler<'browser:navigate'>;
|
||||
'browser:back': InvokeHandler<'browser:back'>;
|
||||
'browser:forward': InvokeHandler<'browser:forward'>;
|
||||
'browser:reload': InvokeHandler<'browser:reload'>;
|
||||
'browser:getState': InvokeHandler<'browser:getState'>;
|
||||
};
|
||||
|
||||
/**
|
||||
* Browser-specific IPC handlers, exported as a plain object so they can be
|
||||
* spread into the main `registerIpcHandlers({...})` call in ipc.ts. This
|
||||
* mirrors the convention of keeping feature handlers flat and namespaced by
|
||||
* channel prefix (`browser:*`).
|
||||
*/
|
||||
export const browserIpcHandlers: BrowserHandlers = {
|
||||
'browser:setBounds': async (_event, args) => {
|
||||
browserViewManager.setBounds(args);
|
||||
return { ok: true };
|
||||
},
|
||||
'browser:setVisible': async (_event, args) => {
|
||||
browserViewManager.setVisible(args.visible);
|
||||
return { ok: true };
|
||||
},
|
||||
'browser:newTab': async (_event, args) => {
|
||||
return browserViewManager.newTab(args.url);
|
||||
},
|
||||
'browser:switchTab': async (_event, args) => {
|
||||
return browserViewManager.switchTab(args.tabId);
|
||||
},
|
||||
'browser:closeTab': async (_event, args) => {
|
||||
return browserViewManager.closeTab(args.tabId);
|
||||
},
|
||||
'browser:navigate': async (_event, args) => {
|
||||
return browserViewManager.navigate(args.url);
|
||||
},
|
||||
'browser:back': async () => {
|
||||
return browserViewManager.back();
|
||||
},
|
||||
'browser:forward': async () => {
|
||||
return browserViewManager.forward();
|
||||
},
|
||||
'browser:reload': async () => {
|
||||
browserViewManager.reload();
|
||||
return { ok: true };
|
||||
},
|
||||
'browser:getState': async () => {
|
||||
return browserViewManager.getState();
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Wire the BrowserViewManager's state-updated event to all renderer windows
|
||||
* as a `browser:didUpdateState` push. Must be called once after the main
|
||||
* window is created so the manager has a window to attach to.
|
||||
*/
|
||||
export function setupBrowserEventForwarding(): void {
|
||||
browserViewManager.on('state-updated', (state: BrowserState) => {
|
||||
const windows = BrowserWindow.getAllWindows();
|
||||
for (const win of windows) {
|
||||
if (!win.isDestroyed() && win.webContents) {
|
||||
win.webContents.send('browser:didUpdateState', state);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
41
apps/x/apps/main/src/browser/navigation.ts
Normal file
41
apps/x/apps/main/src/browser/navigation.ts
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
const SEARCH_ENGINE_BASE_URL = 'https://www.google.com/search?q=';
|
||||
|
||||
const HAS_SCHEME_RE = /^[a-z][a-z0-9+.-]*:/i;
|
||||
const IPV4_HOST_RE = /^\d{1,3}(?:\.\d{1,3}){3}(?::\d+)?(?:\/.*)?$/;
|
||||
const LOCALHOST_RE = /^localhost(?::\d+)?(?:\/.*)?$/i;
|
||||
const DOMAIN_LIKE_RE = /^[\w.-]+\.[a-z]{2,}(?::\d+)?(?:\/.*)?$/i;
|
||||
|
||||
export function normalizeNavigationTarget(target: string): string {
|
||||
const trimmed = target.trim();
|
||||
if (!trimmed) {
|
||||
throw new Error('Navigation target cannot be empty.');
|
||||
}
|
||||
|
||||
const lower = trimmed.toLowerCase();
|
||||
if (
|
||||
lower.startsWith('javascript:')
|
||||
|| lower.startsWith('file://')
|
||||
|| lower.startsWith('chrome://')
|
||||
|| lower.startsWith('chrome-extension://')
|
||||
) {
|
||||
throw new Error('That URL scheme is not allowed in the embedded browser.');
|
||||
}
|
||||
|
||||
if (HAS_SCHEME_RE.test(trimmed)) {
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
const looksLikeHost =
|
||||
LOCALHOST_RE.test(trimmed)
|
||||
|| DOMAIN_LIKE_RE.test(trimmed)
|
||||
|| IPV4_HOST_RE.test(trimmed);
|
||||
|
||||
if (looksLikeHost && !/\s/.test(trimmed)) {
|
||||
const scheme = LOCALHOST_RE.test(trimmed) || IPV4_HOST_RE.test(trimmed)
|
||||
? 'http://'
|
||||
: 'https://';
|
||||
return `${scheme}${trimmed}`;
|
||||
}
|
||||
|
||||
return `${SEARCH_ENGINE_BASE_URL}${encodeURIComponent(trimmed)}`;
|
||||
}
|
||||
546
apps/x/apps/main/src/browser/page-scripts.ts
Normal file
546
apps/x/apps/main/src/browser/page-scripts.ts
Normal file
|
|
@ -0,0 +1,546 @@
|
|||
import type { BrowserPageElement } from '@x/shared/dist/browser-control.js';
|
||||
|
||||
const INTERACTABLE_SELECTORS = [
|
||||
'a[href]',
|
||||
'button',
|
||||
'input',
|
||||
'textarea',
|
||||
'select',
|
||||
'summary',
|
||||
'[role="button"]',
|
||||
'[role="link"]',
|
||||
'[role="tab"]',
|
||||
'[role="menuitem"]',
|
||||
'[role="option"]',
|
||||
'[contenteditable="true"]',
|
||||
'[tabindex]:not([tabindex="-1"])',
|
||||
].join(', ');
|
||||
|
||||
const CLICKABLE_TARGET_SELECTORS = [
|
||||
'a[href]',
|
||||
'button',
|
||||
'summary',
|
||||
'label',
|
||||
'input',
|
||||
'textarea',
|
||||
'select',
|
||||
'[role="button"]',
|
||||
'[role="link"]',
|
||||
'[role="tab"]',
|
||||
'[role="menuitem"]',
|
||||
'[role="option"]',
|
||||
'[role="checkbox"]',
|
||||
'[role="radio"]',
|
||||
'[role="switch"]',
|
||||
'[role="menuitemcheckbox"]',
|
||||
'[role="menuitemradio"]',
|
||||
'[aria-pressed]',
|
||||
'[aria-expanded]',
|
||||
'[aria-checked]',
|
||||
'[contenteditable="true"]',
|
||||
'[tabindex]:not([tabindex="-1"])',
|
||||
].join(', ');
|
||||
|
||||
const DOM_HELPERS_SOURCE = String.raw`
|
||||
const truncateText = (value, max) => {
|
||||
const normalized = String(value ?? '').replace(/\s+/g, ' ').trim();
|
||||
if (!normalized) return '';
|
||||
if (normalized.length <= max) return normalized;
|
||||
const safeMax = Math.max(0, max - 3);
|
||||
return normalized.slice(0, safeMax).trim() + '...';
|
||||
};
|
||||
|
||||
const cssEscapeValue = (value) => {
|
||||
if (typeof CSS !== 'undefined' && typeof CSS.escape === 'function') {
|
||||
return CSS.escape(value);
|
||||
}
|
||||
return String(value).replace(/[^a-zA-Z0-9_-]/g, (char) => '\\' + char);
|
||||
};
|
||||
|
||||
const isVisibleElement = (element) => {
|
||||
if (!(element instanceof Element)) return false;
|
||||
const style = window.getComputedStyle(element);
|
||||
if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') {
|
||||
return false;
|
||||
}
|
||||
if (element.getAttribute('aria-hidden') === 'true') return false;
|
||||
const rect = element.getBoundingClientRect();
|
||||
return rect.width > 0 && rect.height > 0;
|
||||
};
|
||||
|
||||
const isDisabledElement = (element) => {
|
||||
if (!(element instanceof Element)) return true;
|
||||
if (element.getAttribute('aria-disabled') === 'true') return true;
|
||||
return 'disabled' in element && Boolean(element.disabled);
|
||||
};
|
||||
|
||||
const isUselessClickTarget = (element) => (
|
||||
element === document.body
|
||||
|| element === document.documentElement
|
||||
);
|
||||
|
||||
const getElementRole = (element) => {
|
||||
const explicitRole = element.getAttribute('role');
|
||||
if (explicitRole) return explicitRole;
|
||||
if (element instanceof HTMLAnchorElement) return 'link';
|
||||
if (element instanceof HTMLButtonElement) return 'button';
|
||||
if (element instanceof HTMLInputElement) return element.type === 'checkbox' ? 'checkbox' : 'input';
|
||||
if (element instanceof HTMLTextAreaElement) return 'textbox';
|
||||
if (element instanceof HTMLSelectElement) return 'combobox';
|
||||
if (element instanceof HTMLElement && element.isContentEditable) return 'textbox';
|
||||
return null;
|
||||
};
|
||||
|
||||
const getElementType = (element) => {
|
||||
if (element instanceof HTMLInputElement) return element.type || 'text';
|
||||
if (element instanceof HTMLTextAreaElement) return 'textarea';
|
||||
if (element instanceof HTMLSelectElement) return 'select';
|
||||
if (element instanceof HTMLButtonElement) return 'button';
|
||||
if (element instanceof HTMLElement && element.isContentEditable) return 'contenteditable';
|
||||
return null;
|
||||
};
|
||||
|
||||
const getElementLabel = (element) => {
|
||||
const ariaLabel = truncateText(element.getAttribute('aria-label') ?? '', 120);
|
||||
if (ariaLabel) return ariaLabel;
|
||||
|
||||
if ('labels' in element && element.labels && element.labels.length > 0) {
|
||||
const labelText = truncateText(
|
||||
Array.from(element.labels).map((label) => label.innerText || label.textContent || '').join(' '),
|
||||
120,
|
||||
);
|
||||
if (labelText) return labelText;
|
||||
}
|
||||
|
||||
if (element.id) {
|
||||
const label = document.querySelector('label[for="' + cssEscapeValue(element.id) + '"]');
|
||||
const labelText = truncateText(label?.textContent ?? '', 120);
|
||||
if (labelText) return labelText;
|
||||
}
|
||||
|
||||
const placeholder = truncateText(element.getAttribute('placeholder') ?? '', 120);
|
||||
if (placeholder) return placeholder;
|
||||
|
||||
const text = truncateText(
|
||||
element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement
|
||||
? element.value
|
||||
: element.textContent ?? '',
|
||||
120,
|
||||
);
|
||||
return text || null;
|
||||
};
|
||||
|
||||
const describeElement = (element) => {
|
||||
const role = getElementRole(element) || element.tagName.toLowerCase();
|
||||
const label = getElementLabel(element);
|
||||
return label ? role + ' "' + label + '"' : role;
|
||||
};
|
||||
|
||||
const clampNumber = (value, min, max) => Math.min(Math.max(value, min), max);
|
||||
|
||||
const getAssociatedControl = (element) => {
|
||||
if (!(element instanceof Element)) return null;
|
||||
if (element instanceof HTMLLabelElement) return element.control;
|
||||
const parentLabel = element.closest('label');
|
||||
return parentLabel instanceof HTMLLabelElement ? parentLabel.control : null;
|
||||
};
|
||||
|
||||
const resolveClickTarget = (element) => {
|
||||
if (!(element instanceof Element)) return null;
|
||||
|
||||
const clickableAncestor = element.closest(${JSON.stringify(CLICKABLE_TARGET_SELECTORS)});
|
||||
const labelAncestor = element.closest('label');
|
||||
const associatedControl = getAssociatedControl(element);
|
||||
const candidates = [clickableAncestor, labelAncestor, associatedControl, element];
|
||||
|
||||
for (const candidate of candidates) {
|
||||
if (!(candidate instanceof Element)) continue;
|
||||
if (isUselessClickTarget(candidate)) continue;
|
||||
if (!isVisibleElement(candidate)) continue;
|
||||
if (isDisabledElement(candidate)) continue;
|
||||
return candidate;
|
||||
}
|
||||
|
||||
for (const candidate of candidates) {
|
||||
if (candidate instanceof Element) return candidate;
|
||||
}
|
||||
|
||||
return null;
|
||||
};
|
||||
|
||||
const getVerificationTargetState = (element) => {
|
||||
if (!(element instanceof Element)) return null;
|
||||
|
||||
const text = truncateText(element.innerText || element.textContent || '', 200);
|
||||
const activeElement = document.activeElement;
|
||||
const isActive =
|
||||
activeElement instanceof Element
|
||||
? activeElement === element || element.contains(activeElement)
|
||||
: false;
|
||||
|
||||
return {
|
||||
selector: buildUniqueSelector(element),
|
||||
descriptor: describeElement(element),
|
||||
text: text || null,
|
||||
checked:
|
||||
element instanceof HTMLInputElement && (element.type === 'checkbox' || element.type === 'radio')
|
||||
? element.checked
|
||||
: null,
|
||||
value:
|
||||
element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement
|
||||
? truncateText(element.value ?? '', 200)
|
||||
: element instanceof HTMLSelectElement
|
||||
? truncateText(element.value ?? '', 200)
|
||||
: element instanceof HTMLElement && element.isContentEditable
|
||||
? truncateText(element.innerText || element.textContent || '', 200)
|
||||
: null,
|
||||
selectedIndex: element instanceof HTMLSelectElement ? element.selectedIndex : null,
|
||||
open:
|
||||
'open' in element && typeof element.open === 'boolean'
|
||||
? element.open
|
||||
: null,
|
||||
disabled: isDisabledElement(element),
|
||||
active: isActive,
|
||||
ariaChecked: element.getAttribute('aria-checked'),
|
||||
ariaPressed: element.getAttribute('aria-pressed'),
|
||||
ariaExpanded: element.getAttribute('aria-expanded'),
|
||||
};
|
||||
};
|
||||
|
||||
const getPageVerificationState = () => {
|
||||
const activeElement = document.activeElement instanceof Element ? document.activeElement : null;
|
||||
return {
|
||||
url: window.location.href,
|
||||
title: document.title || '',
|
||||
textSample: truncateText(document.body?.innerText || document.body?.textContent || '', 2000),
|
||||
activeSelector: activeElement ? buildUniqueSelector(activeElement) : null,
|
||||
};
|
||||
};
|
||||
|
||||
const buildUniqueSelector = (element) => {
|
||||
if (!(element instanceof Element)) return null;
|
||||
|
||||
if (element.id) {
|
||||
const idSelector = '#' + cssEscapeValue(element.id);
|
||||
try {
|
||||
if (document.querySelectorAll(idSelector).length === 1) return idSelector;
|
||||
} catch {}
|
||||
}
|
||||
|
||||
const segments = [];
|
||||
let current = element;
|
||||
while (current && current instanceof Element && current !== document.documentElement) {
|
||||
const tag = current.tagName.toLowerCase();
|
||||
if (!tag) break;
|
||||
|
||||
let segment = tag;
|
||||
const name = current.getAttribute('name');
|
||||
if (name) {
|
||||
const nameSelector = tag + '[name="' + cssEscapeValue(name) + '"]';
|
||||
try {
|
||||
if (document.querySelectorAll(nameSelector).length === 1) {
|
||||
segments.unshift(nameSelector);
|
||||
return segments.join(' > ');
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
|
||||
const parent = current.parentElement;
|
||||
if (parent) {
|
||||
const sameTagSiblings = Array.from(parent.children).filter((child) => child.tagName === current.tagName);
|
||||
const position = sameTagSiblings.indexOf(current) + 1;
|
||||
segment += ':nth-of-type(' + position + ')';
|
||||
}
|
||||
|
||||
segments.unshift(segment);
|
||||
const selector = segments.join(' > ');
|
||||
try {
|
||||
if (document.querySelectorAll(selector).length === 1) return selector;
|
||||
} catch {}
|
||||
|
||||
current = current.parentElement;
|
||||
}
|
||||
|
||||
return segments.length > 0 ? segments.join(' > ') : null;
|
||||
};
|
||||
`;
|
||||
|
||||
type RawBrowserPageElement = BrowserPageElement & {
|
||||
selector: string;
|
||||
};
|
||||
|
||||
export type RawBrowserPageSnapshot = {
|
||||
url: string;
|
||||
title: string;
|
||||
loading: boolean;
|
||||
text: string;
|
||||
elements: RawBrowserPageElement[];
|
||||
};
|
||||
|
||||
export type ElementTarget = {
|
||||
index?: number;
|
||||
selector?: string;
|
||||
snapshotId?: string;
|
||||
};
|
||||
|
||||
export function buildReadPageScript(maxElements: number, maxTextLength: number): string {
|
||||
return `(() => {
|
||||
${DOM_HELPERS_SOURCE}
|
||||
const candidates = Array.from(document.querySelectorAll(${JSON.stringify(INTERACTABLE_SELECTORS)}));
|
||||
const elements = [];
|
||||
const seenSelectors = new Set();
|
||||
|
||||
for (const candidate of candidates) {
|
||||
if (!(candidate instanceof Element)) continue;
|
||||
if (!isVisibleElement(candidate)) continue;
|
||||
|
||||
const selector = buildUniqueSelector(candidate);
|
||||
if (!selector || seenSelectors.has(selector)) continue;
|
||||
seenSelectors.add(selector);
|
||||
|
||||
elements.push({
|
||||
index: elements.length + 1,
|
||||
selector,
|
||||
tagName: candidate.tagName.toLowerCase(),
|
||||
role: getElementRole(candidate),
|
||||
type: getElementType(candidate),
|
||||
label: getElementLabel(candidate),
|
||||
text: truncateText(candidate.innerText || candidate.textContent || '', 120) || null,
|
||||
placeholder: truncateText(candidate.getAttribute('placeholder') ?? '', 120) || null,
|
||||
href: candidate instanceof HTMLAnchorElement ? candidate.href : candidate.getAttribute('href'),
|
||||
disabled: isDisabledElement(candidate),
|
||||
});
|
||||
|
||||
if (elements.length >= ${JSON.stringify(maxElements)}) break;
|
||||
}
|
||||
|
||||
return {
|
||||
url: window.location.href,
|
||||
title: document.title || '',
|
||||
loading: document.readyState !== 'complete',
|
||||
text: truncateText(document.body?.innerText || document.body?.textContent || '', ${JSON.stringify(maxTextLength)}),
|
||||
elements,
|
||||
};
|
||||
})()`;
|
||||
}
|
||||
|
||||
export function buildClickScript(selector: string): string {
|
||||
return `(() => {
|
||||
${DOM_HELPERS_SOURCE}
|
||||
const requestedSelector = ${JSON.stringify(selector)};
|
||||
if (/^(body|html)$/i.test(requestedSelector.trim())) {
|
||||
return {
|
||||
ok: false,
|
||||
error: 'Refusing to click the page body. Read the page again and target a specific element.',
|
||||
};
|
||||
}
|
||||
|
||||
const element = document.querySelector(requestedSelector);
|
||||
if (!(element instanceof Element)) {
|
||||
return { ok: false, error: 'Element not found.' };
|
||||
}
|
||||
if (isUselessClickTarget(element)) {
|
||||
return {
|
||||
ok: false,
|
||||
error: 'Refusing to click the page body. Read the page again and target a specific element.',
|
||||
};
|
||||
}
|
||||
|
||||
const target = resolveClickTarget(element);
|
||||
if (!(target instanceof Element)) {
|
||||
return { ok: false, error: 'Could not resolve a clickable target.' };
|
||||
}
|
||||
if (isUselessClickTarget(target)) {
|
||||
return {
|
||||
ok: false,
|
||||
error: 'Resolved click target was too generic. Read the page again and choose a specific control.',
|
||||
};
|
||||
}
|
||||
if (!isVisibleElement(target)) {
|
||||
return { ok: false, error: 'Resolved click target is not visible.' };
|
||||
}
|
||||
if (isDisabledElement(target)) {
|
||||
return { ok: false, error: 'Resolved click target is disabled.' };
|
||||
}
|
||||
|
||||
const before = {
|
||||
page: getPageVerificationState(),
|
||||
target: getVerificationTargetState(target),
|
||||
};
|
||||
|
||||
if (target instanceof HTMLElement) {
|
||||
target.scrollIntoView({ block: 'center', inline: 'center' });
|
||||
target.focus({ preventScroll: true });
|
||||
}
|
||||
|
||||
const rect = target.getBoundingClientRect();
|
||||
const clientX = clampNumber(rect.left + (rect.width / 2), 1, Math.max(1, window.innerWidth - 1));
|
||||
const clientY = clampNumber(rect.top + (rect.height / 2), 1, Math.max(1, window.innerHeight - 1));
|
||||
const topElement = document.elementFromPoint(clientX, clientY);
|
||||
const eventTarget =
|
||||
topElement instanceof Element && (topElement === target || topElement.contains(target) || target.contains(topElement))
|
||||
? topElement
|
||||
: target;
|
||||
|
||||
if (eventTarget instanceof HTMLElement) {
|
||||
eventTarget.focus({ preventScroll: true });
|
||||
}
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
description: describeElement(target),
|
||||
clickPoint: {
|
||||
x: Math.round(clientX),
|
||||
y: Math.round(clientY),
|
||||
},
|
||||
verification: {
|
||||
before,
|
||||
targetSelector: buildUniqueSelector(target) || requestedSelector,
|
||||
},
|
||||
};
|
||||
})()`;
|
||||
}
|
||||
|
||||
export function buildVerifyClickScript(targetSelector: string | null, before: unknown): string {
|
||||
return `(() => {
|
||||
${DOM_HELPERS_SOURCE}
|
||||
const beforeState = ${JSON.stringify(before)};
|
||||
const selector = ${JSON.stringify(targetSelector)};
|
||||
const afterPage = getPageVerificationState();
|
||||
const afterTarget = selector ? getVerificationTargetState(document.querySelector(selector)) : null;
|
||||
const beforeTarget = beforeState?.target ?? null;
|
||||
const reasons = [];
|
||||
|
||||
if (beforeState?.page?.url !== afterPage.url) reasons.push('url changed');
|
||||
if (beforeState?.page?.title !== afterPage.title) reasons.push('title changed');
|
||||
if (beforeState?.page?.textSample !== afterPage.textSample) reasons.push('page text changed');
|
||||
if (beforeState?.page?.activeSelector !== afterPage.activeSelector) reasons.push('focus changed');
|
||||
|
||||
if (beforeTarget && !afterTarget) {
|
||||
reasons.push('clicked element disappeared');
|
||||
}
|
||||
|
||||
if (beforeTarget && afterTarget) {
|
||||
if (beforeTarget.checked !== afterTarget.checked) reasons.push('checked state changed');
|
||||
if (beforeTarget.value !== afterTarget.value) reasons.push('value changed');
|
||||
if (beforeTarget.selectedIndex !== afterTarget.selectedIndex) reasons.push('selection changed');
|
||||
if (beforeTarget.open !== afterTarget.open) reasons.push('open state changed');
|
||||
if (beforeTarget.disabled !== afterTarget.disabled) reasons.push('disabled state changed');
|
||||
if (beforeTarget.active !== afterTarget.active) reasons.push('target focus changed');
|
||||
if (beforeTarget.ariaChecked !== afterTarget.ariaChecked) reasons.push('aria-checked changed');
|
||||
if (beforeTarget.ariaPressed !== afterTarget.ariaPressed) reasons.push('aria-pressed changed');
|
||||
if (beforeTarget.ariaExpanded !== afterTarget.ariaExpanded) reasons.push('aria-expanded changed');
|
||||
if (beforeTarget.text !== afterTarget.text) reasons.push('target text changed');
|
||||
}
|
||||
|
||||
return {
|
||||
changed: reasons.length > 0,
|
||||
reasons,
|
||||
};
|
||||
})()`;
|
||||
}
|
||||
|
||||
export function buildTypeScript(selector: string, text: string): string {
|
||||
return `(() => {
|
||||
${DOM_HELPERS_SOURCE}
|
||||
const element = document.querySelector(${JSON.stringify(selector)});
|
||||
if (!(element instanceof Element)) {
|
||||
return { ok: false, error: 'Element not found.' };
|
||||
}
|
||||
if (!isVisibleElement(element)) {
|
||||
return { ok: false, error: 'Element is not visible.' };
|
||||
}
|
||||
if (isDisabledElement(element)) {
|
||||
return { ok: false, error: 'Element is disabled.' };
|
||||
}
|
||||
|
||||
const nextValue = ${JSON.stringify(text)};
|
||||
|
||||
const setNativeValue = (target, value) => {
|
||||
const prototype = Object.getPrototypeOf(target);
|
||||
const descriptor = Object.getOwnPropertyDescriptor(prototype, 'value');
|
||||
if (descriptor && typeof descriptor.set === 'function') {
|
||||
descriptor.set.call(target, value);
|
||||
} else {
|
||||
target.value = value;
|
||||
}
|
||||
};
|
||||
|
||||
if (element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement) {
|
||||
if (element.readOnly) {
|
||||
return { ok: false, error: 'Element is read-only.' };
|
||||
}
|
||||
element.scrollIntoView({ block: 'center', inline: 'center' });
|
||||
element.focus({ preventScroll: true });
|
||||
setNativeValue(element, nextValue);
|
||||
element.dispatchEvent(new InputEvent('input', { bubbles: true, data: nextValue, inputType: 'insertText' }));
|
||||
element.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
return { ok: true, description: describeElement(element) };
|
||||
}
|
||||
|
||||
if (element instanceof HTMLElement && element.isContentEditable) {
|
||||
element.scrollIntoView({ block: 'center', inline: 'center' });
|
||||
element.focus({ preventScroll: true });
|
||||
element.textContent = nextValue;
|
||||
element.dispatchEvent(new InputEvent('input', { bubbles: true, data: nextValue, inputType: 'insertText' }));
|
||||
return { ok: true, description: describeElement(element) };
|
||||
}
|
||||
|
||||
return { ok: false, error: 'Element does not accept text input.' };
|
||||
})()`;
|
||||
}
|
||||
|
||||
export function buildFocusScript(selector: string): string {
|
||||
return `(() => {
|
||||
${DOM_HELPERS_SOURCE}
|
||||
const element = document.querySelector(${JSON.stringify(selector)});
|
||||
if (!(element instanceof Element)) {
|
||||
return { ok: false, error: 'Element not found.' };
|
||||
}
|
||||
if (!isVisibleElement(element)) {
|
||||
return { ok: false, error: 'Element is not visible.' };
|
||||
}
|
||||
if (element instanceof HTMLElement) {
|
||||
element.scrollIntoView({ block: 'center', inline: 'center' });
|
||||
element.focus({ preventScroll: true });
|
||||
}
|
||||
return { ok: true, description: describeElement(element) };
|
||||
})()`;
|
||||
}
|
||||
|
||||
export function buildScrollScript(offset: number): string {
|
||||
return `(() => {
|
||||
window.scrollBy({ top: ${JSON.stringify(offset)}, left: 0, behavior: 'auto' });
|
||||
return { ok: true };
|
||||
})()`;
|
||||
}
|
||||
|
||||
export function normalizeKeyCode(key: string): string {
|
||||
const trimmed = key.trim();
|
||||
if (!trimmed) return 'Enter';
|
||||
|
||||
const aliases: Record<string, string> = {
|
||||
esc: 'Escape',
|
||||
escape: 'Escape',
|
||||
return: 'Enter',
|
||||
enter: 'Enter',
|
||||
tab: 'Tab',
|
||||
space: 'Space',
|
||||
' ': 'Space',
|
||||
left: 'ArrowLeft',
|
||||
right: 'ArrowRight',
|
||||
up: 'ArrowUp',
|
||||
down: 'ArrowDown',
|
||||
arrowleft: 'ArrowLeft',
|
||||
arrowright: 'ArrowRight',
|
||||
arrowup: 'ArrowUp',
|
||||
arrowdown: 'ArrowDown',
|
||||
backspace: 'Backspace',
|
||||
delete: 'Delete',
|
||||
};
|
||||
|
||||
const alias = aliases[trimmed.toLowerCase()];
|
||||
if (alias) return alias;
|
||||
if (trimmed.length === 1) return trimmed.toUpperCase();
|
||||
return trimmed[0].toUpperCase() + trimmed.slice(1);
|
||||
}
|
||||
797
apps/x/apps/main/src/browser/view.ts
Normal file
797
apps/x/apps/main/src/browser/view.ts
Normal file
|
|
@ -0,0 +1,797 @@
|
|||
import { randomUUID } from 'node:crypto';
|
||||
import { EventEmitter } from 'node:events';
|
||||
import { BrowserWindow, WebContentsView, session, shell, type Session } from 'electron';
|
||||
import type {
|
||||
BrowserPageElement,
|
||||
BrowserPageSnapshot,
|
||||
BrowserState,
|
||||
BrowserTabState,
|
||||
} from '@x/shared/dist/browser-control.js';
|
||||
import { normalizeNavigationTarget } from './navigation.js';
|
||||
import {
|
||||
buildClickScript,
|
||||
buildFocusScript,
|
||||
buildReadPageScript,
|
||||
buildScrollScript,
|
||||
buildTypeScript,
|
||||
buildVerifyClickScript,
|
||||
normalizeKeyCode,
|
||||
type ElementTarget,
|
||||
type RawBrowserPageSnapshot,
|
||||
} from './page-scripts.js';
|
||||
|
||||
export type { BrowserPageSnapshot, BrowserState, BrowserTabState };
|
||||
|
||||
/**
|
||||
* Embedded browser pane implementation.
|
||||
*
|
||||
* Each browser tab owns its own WebContentsView. Only the active tab's view is
|
||||
* attached to the main window at a time, but inactive tabs keep their own page
|
||||
* history and loaded state in memory so switching tabs feels immediate.
|
||||
*
|
||||
* All tabs share one persistent session partition so cookies/localStorage/
|
||||
* form-fill state survive app restarts, and the browser surface spoofs a
|
||||
* standard Chrome UA so sites like Google (OAuth) don't reject it.
|
||||
*/
|
||||
|
||||
export const BROWSER_PARTITION = 'persist:rowboat-browser';
|
||||
|
||||
// Claims Chrome 130 on macOS — close enough to recent stable for OAuth servers
|
||||
// that sniff the UA looking for "real browser" shapes.
|
||||
const SPOOF_UA =
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36';
|
||||
|
||||
const HOME_URL = 'https://www.google.com';
|
||||
const NAVIGATION_TIMEOUT_MS = 10000;
|
||||
const POST_ACTION_IDLE_MS = 400;
|
||||
const POST_ACTION_MAX_ELEMENTS = 25;
|
||||
const POST_ACTION_MAX_TEXT_LENGTH = 4000;
|
||||
const DEFAULT_READ_MAX_ELEMENTS = 50;
|
||||
const DEFAULT_READ_MAX_TEXT_LENGTH = 8000;
|
||||
|
||||
export interface BrowserBounds {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
type BrowserTab = {
|
||||
id: string;
|
||||
view: WebContentsView;
|
||||
domReadyAt: number | null;
|
||||
loadError: string | null;
|
||||
};
|
||||
|
||||
type CachedSnapshot = {
|
||||
snapshotId: string;
|
||||
elements: Array<{ index: number; selector: string }>;
|
||||
};
|
||||
|
||||
const EMPTY_STATE: BrowserState = {
|
||||
activeTabId: null,
|
||||
tabs: [],
|
||||
};
|
||||
|
||||
function abortIfNeeded(signal?: AbortSignal): void {
|
||||
if (!signal?.aborted) return;
|
||||
throw signal.reason instanceof Error ? signal.reason : new Error('Browser action aborted');
|
||||
}
|
||||
|
||||
async function sleep(ms: number, signal?: AbortSignal): Promise<void> {
|
||||
if (ms <= 0) return;
|
||||
abortIfNeeded(signal);
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const abortSignal = signal;
|
||||
const timer = setTimeout(() => {
|
||||
abortSignal?.removeEventListener('abort', onAbort);
|
||||
resolve();
|
||||
}, ms);
|
||||
|
||||
const onAbort = () => {
|
||||
clearTimeout(timer);
|
||||
abortSignal?.removeEventListener('abort', onAbort);
|
||||
reject(abortSignal?.reason instanceof Error ? abortSignal.reason : new Error('Browser action aborted'));
|
||||
};
|
||||
|
||||
abortSignal?.addEventListener('abort', onAbort, { once: true });
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
export class BrowserViewManager extends EventEmitter {
|
||||
private window: BrowserWindow | null = null;
|
||||
private browserSession: Session | null = null;
|
||||
private tabs = new Map<string, BrowserTab>();
|
||||
private tabOrder: string[] = [];
|
||||
private activeTabId: string | null = null;
|
||||
private attachedTabId: string | null = null;
|
||||
private visible = false;
|
||||
private bounds: BrowserBounds = { x: 0, y: 0, width: 0, height: 0 };
|
||||
private snapshotCache = new Map<string, CachedSnapshot>();
|
||||
|
||||
attach(window: BrowserWindow): void {
|
||||
this.window = window;
|
||||
window.on('closed', () => {
|
||||
this.window = null;
|
||||
this.browserSession = null;
|
||||
this.tabs.clear();
|
||||
this.tabOrder = [];
|
||||
this.activeTabId = null;
|
||||
this.attachedTabId = null;
|
||||
this.visible = false;
|
||||
this.snapshotCache.clear();
|
||||
});
|
||||
}
|
||||
|
||||
private getSession(): Session {
|
||||
if (this.browserSession) return this.browserSession;
|
||||
const browserSession = session.fromPartition(BROWSER_PARTITION);
|
||||
browserSession.setUserAgent(SPOOF_UA);
|
||||
this.browserSession = browserSession;
|
||||
return browserSession;
|
||||
}
|
||||
|
||||
private emitState(): void {
|
||||
this.emit('state-updated', this.snapshotState());
|
||||
}
|
||||
|
||||
private getTab(tabId: string | null): BrowserTab | null {
|
||||
if (!tabId) return null;
|
||||
return this.tabs.get(tabId) ?? null;
|
||||
}
|
||||
|
||||
private getActiveTab(): BrowserTab | null {
|
||||
return this.getTab(this.activeTabId);
|
||||
}
|
||||
|
||||
private invalidateSnapshot(tabId: string): void {
|
||||
this.snapshotCache.delete(tabId);
|
||||
}
|
||||
|
||||
private isEmbeddedTabUrl(url: string): boolean {
|
||||
return /^https?:\/\//i.test(url) || url === 'about:blank';
|
||||
}
|
||||
|
||||
private createView(): WebContentsView {
|
||||
const view = new WebContentsView({
|
||||
webPreferences: {
|
||||
session: this.getSession(),
|
||||
contextIsolation: true,
|
||||
sandbox: true,
|
||||
nodeIntegration: false,
|
||||
},
|
||||
});
|
||||
|
||||
view.webContents.setUserAgent(SPOOF_UA);
|
||||
return view;
|
||||
}
|
||||
|
||||
private wireEvents(tab: BrowserTab): void {
|
||||
const { id: tabId, view } = tab;
|
||||
const wc = view.webContents;
|
||||
|
||||
const reapplyBounds = () => {
|
||||
if (
|
||||
this.attachedTabId === tabId &&
|
||||
this.visible &&
|
||||
this.bounds.width > 0 &&
|
||||
this.bounds.height > 0
|
||||
) {
|
||||
view.setBounds(this.bounds);
|
||||
}
|
||||
};
|
||||
|
||||
const invalidateAndEmit = () => {
|
||||
this.invalidateSnapshot(tabId);
|
||||
this.emitState();
|
||||
};
|
||||
|
||||
wc.on('did-start-navigation', (_event, _url, _isInPlace, isMainFrame) => {
|
||||
if (isMainFrame !== false) {
|
||||
tab.domReadyAt = null;
|
||||
tab.loadError = null;
|
||||
}
|
||||
this.invalidateSnapshot(tabId);
|
||||
reapplyBounds();
|
||||
});
|
||||
wc.on('did-navigate', () => { reapplyBounds(); invalidateAndEmit(); });
|
||||
wc.on('did-navigate-in-page', () => { reapplyBounds(); invalidateAndEmit(); });
|
||||
wc.on('did-start-loading', () => {
|
||||
tab.loadError = null;
|
||||
this.invalidateSnapshot(tabId);
|
||||
reapplyBounds();
|
||||
this.emitState();
|
||||
});
|
||||
wc.on('did-stop-loading', () => { reapplyBounds(); invalidateAndEmit(); });
|
||||
wc.on('did-finish-load', () => { reapplyBounds(); invalidateAndEmit(); });
|
||||
wc.on('dom-ready', () => {
|
||||
tab.domReadyAt = Date.now();
|
||||
reapplyBounds();
|
||||
invalidateAndEmit();
|
||||
});
|
||||
wc.on('did-frame-finish-load', reapplyBounds);
|
||||
wc.on('did-fail-load', (_event, errorCode, errorDescription, validatedURL, isMainFrame) => {
|
||||
if (isMainFrame && errorCode !== -3) {
|
||||
const target = validatedURL || wc.getURL() || 'page';
|
||||
tab.loadError = errorDescription
|
||||
? `Failed to load ${target}: ${errorDescription}.`
|
||||
: `Failed to load ${target}.`;
|
||||
}
|
||||
reapplyBounds();
|
||||
invalidateAndEmit();
|
||||
});
|
||||
wc.on('page-title-updated', this.emitState.bind(this));
|
||||
|
||||
wc.setWindowOpenHandler(({ url }) => {
|
||||
if (this.isEmbeddedTabUrl(url)) {
|
||||
void this.newTab(url);
|
||||
} else {
|
||||
void shell.openExternal(url);
|
||||
}
|
||||
return { action: 'deny' };
|
||||
});
|
||||
}
|
||||
|
||||
private snapshotTabState(tab: BrowserTab): BrowserTabState {
|
||||
const wc = tab.view.webContents;
|
||||
return {
|
||||
id: tab.id,
|
||||
url: wc.getURL(),
|
||||
title: wc.getTitle(),
|
||||
canGoBack: wc.navigationHistory.canGoBack(),
|
||||
canGoForward: wc.navigationHistory.canGoForward(),
|
||||
loading: wc.isLoading(),
|
||||
};
|
||||
}
|
||||
|
||||
private syncAttachedView(): void {
|
||||
if (!this.window) return;
|
||||
|
||||
const contentView = this.window.contentView;
|
||||
const activeTab = this.getActiveTab();
|
||||
|
||||
if (!this.visible || !activeTab) {
|
||||
const attachedTab = this.getTab(this.attachedTabId);
|
||||
if (attachedTab) {
|
||||
contentView.removeChildView(attachedTab.view);
|
||||
}
|
||||
this.attachedTabId = null;
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.attachedTabId && this.attachedTabId !== activeTab.id) {
|
||||
const attachedTab = this.getTab(this.attachedTabId);
|
||||
if (attachedTab) {
|
||||
contentView.removeChildView(attachedTab.view);
|
||||
}
|
||||
this.attachedTabId = null;
|
||||
}
|
||||
|
||||
if (this.attachedTabId !== activeTab.id) {
|
||||
contentView.addChildView(activeTab.view);
|
||||
this.attachedTabId = activeTab.id;
|
||||
}
|
||||
|
||||
if (this.bounds.width > 0 && this.bounds.height > 0) {
|
||||
activeTab.view.setBounds(this.bounds);
|
||||
}
|
||||
}
|
||||
|
||||
private createTab(initialUrl: string): BrowserTab {
|
||||
if (!this.window) {
|
||||
throw new Error('BrowserViewManager: no window attached');
|
||||
}
|
||||
|
||||
const tabId = randomUUID();
|
||||
const tab: BrowserTab = {
|
||||
id: tabId,
|
||||
view: this.createView(),
|
||||
domReadyAt: null,
|
||||
loadError: null,
|
||||
};
|
||||
|
||||
this.wireEvents(tab);
|
||||
this.tabs.set(tabId, tab);
|
||||
this.tabOrder.push(tabId);
|
||||
this.activeTabId = tabId;
|
||||
this.invalidateSnapshot(tabId);
|
||||
this.syncAttachedView();
|
||||
this.emitState();
|
||||
|
||||
const targetUrl =
|
||||
initialUrl === 'about:blank'
|
||||
? HOME_URL
|
||||
: normalizeNavigationTarget(initialUrl);
|
||||
void tab.view.webContents.loadURL(targetUrl).catch((error) => {
|
||||
tab.loadError = error instanceof Error
|
||||
? error.message
|
||||
: `Failed to load ${targetUrl}.`;
|
||||
this.emitState();
|
||||
});
|
||||
|
||||
return tab;
|
||||
}
|
||||
|
||||
private ensureInitialTab(): BrowserTab {
|
||||
const activeTab = this.getActiveTab();
|
||||
if (activeTab) return activeTab;
|
||||
return this.createTab(HOME_URL);
|
||||
}
|
||||
|
||||
private destroyTab(tab: BrowserTab): void {
|
||||
this.invalidateSnapshot(tab.id);
|
||||
tab.view.webContents.removeAllListeners();
|
||||
if (!tab.view.webContents.isDestroyed()) {
|
||||
tab.view.webContents.close();
|
||||
}
|
||||
}
|
||||
|
||||
private async waitForWebContentsSettle(
|
||||
tab: BrowserTab,
|
||||
signal?: AbortSignal,
|
||||
idleMs = POST_ACTION_IDLE_MS,
|
||||
timeoutMs = NAVIGATION_TIMEOUT_MS,
|
||||
): Promise<void> {
|
||||
const wc = tab.view.webContents;
|
||||
const startedAt = Date.now();
|
||||
let sawLoading = wc.isLoading();
|
||||
|
||||
while (Date.now() - startedAt < timeoutMs) {
|
||||
abortIfNeeded(signal);
|
||||
if (wc.isDestroyed()) return;
|
||||
if (tab.loadError) {
|
||||
throw new Error(tab.loadError);
|
||||
}
|
||||
|
||||
if (tab.domReadyAt != null) {
|
||||
const domReadyForMs = Date.now() - tab.domReadyAt;
|
||||
const requiredIdleMs = sawLoading ? idleMs : Math.min(idleMs, 200);
|
||||
if (domReadyForMs >= requiredIdleMs) return;
|
||||
await sleep(Math.min(100, requiredIdleMs - domReadyForMs), signal);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (wc.isLoading()) {
|
||||
sawLoading = true;
|
||||
await sleep(100, signal);
|
||||
continue;
|
||||
}
|
||||
|
||||
await sleep(sawLoading ? idleMs : Math.min(idleMs, 200), signal);
|
||||
if (tab.loadError) {
|
||||
throw new Error(tab.loadError);
|
||||
}
|
||||
if (!wc.isLoading() || tab.domReadyAt != null) return;
|
||||
sawLoading = true;
|
||||
}
|
||||
}
|
||||
|
||||
private async executeOnActiveTab<T>(
|
||||
script: string,
|
||||
signal?: AbortSignal,
|
||||
options?: { waitForReady?: boolean },
|
||||
): Promise<T> {
|
||||
abortIfNeeded(signal);
|
||||
const activeTab = this.getActiveTab() ?? this.ensureInitialTab();
|
||||
if (options?.waitForReady !== false) {
|
||||
await this.waitForWebContentsSettle(activeTab, signal);
|
||||
}
|
||||
abortIfNeeded(signal);
|
||||
return activeTab.view.webContents.executeJavaScript(script, true) as Promise<T>;
|
||||
}
|
||||
|
||||
private cacheSnapshot(tabId: string, rawSnapshot: RawBrowserPageSnapshot, loading: boolean): BrowserPageSnapshot {
|
||||
const snapshotId = randomUUID();
|
||||
const elements: BrowserPageElement[] = rawSnapshot.elements.map((element, index) => {
|
||||
const { selector, ...rest } = element;
|
||||
void selector;
|
||||
return {
|
||||
...rest,
|
||||
index: index + 1,
|
||||
};
|
||||
});
|
||||
|
||||
this.snapshotCache.set(tabId, {
|
||||
snapshotId,
|
||||
elements: rawSnapshot.elements.map((element, index) => ({
|
||||
index: index + 1,
|
||||
selector: element.selector,
|
||||
})),
|
||||
});
|
||||
|
||||
return {
|
||||
snapshotId,
|
||||
url: rawSnapshot.url,
|
||||
title: rawSnapshot.title,
|
||||
loading,
|
||||
text: rawSnapshot.text,
|
||||
elements,
|
||||
};
|
||||
}
|
||||
|
||||
private resolveElementSelector(tabId: string, target: ElementTarget): { ok: true; selector: string } | { ok: false; error: string } {
|
||||
if (target.selector?.trim()) {
|
||||
return { ok: true, selector: target.selector.trim() };
|
||||
}
|
||||
|
||||
if (target.index == null) {
|
||||
return { ok: false, error: 'Provide an element index or selector.' };
|
||||
}
|
||||
|
||||
const cachedSnapshot = this.snapshotCache.get(tabId);
|
||||
if (!cachedSnapshot) {
|
||||
return { ok: false, error: 'No page snapshot is available yet. Call read-page first.' };
|
||||
}
|
||||
|
||||
if (target.snapshotId && cachedSnapshot.snapshotId !== target.snapshotId) {
|
||||
return { ok: false, error: 'The page changed since the last read-page call. Call read-page again.' };
|
||||
}
|
||||
|
||||
const entry = cachedSnapshot.elements.find((element) => element.index === target.index);
|
||||
if (!entry) {
|
||||
return { ok: false, error: `No element found for index ${target.index}.` };
|
||||
}
|
||||
|
||||
return { ok: true, selector: entry.selector };
|
||||
}
|
||||
|
||||
setVisible(visible: boolean): void {
|
||||
this.visible = visible;
|
||||
if (visible) {
|
||||
this.ensureInitialTab();
|
||||
}
|
||||
this.syncAttachedView();
|
||||
}
|
||||
|
||||
setBounds(bounds: BrowserBounds): void {
|
||||
this.bounds = bounds;
|
||||
const activeTab = this.getActiveTab();
|
||||
if (activeTab && this.attachedTabId === activeTab.id && this.visible) {
|
||||
activeTab.view.setBounds(bounds);
|
||||
}
|
||||
}
|
||||
|
||||
async ensureActiveTabReady(signal?: AbortSignal): Promise<void> {
|
||||
const activeTab = this.getActiveTab() ?? this.ensureInitialTab();
|
||||
await this.waitForWebContentsSettle(activeTab, signal);
|
||||
}
|
||||
|
||||
async newTab(rawUrl?: string): Promise<{ ok: boolean; tabId?: string; error?: string }> {
|
||||
try {
|
||||
const tab = this.createTab(rawUrl?.trim() ? rawUrl : HOME_URL);
|
||||
return { ok: true, tabId: tab.id };
|
||||
} catch (err) {
|
||||
return { ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
}
|
||||
}
|
||||
|
||||
switchTab(tabId: string): { ok: boolean } {
|
||||
if (!this.tabs.has(tabId)) return { ok: false };
|
||||
if (this.activeTabId === tabId) return { ok: true };
|
||||
this.activeTabId = tabId;
|
||||
this.syncAttachedView();
|
||||
this.emitState();
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
closeTab(tabId: string): { ok: boolean } {
|
||||
const tab = this.tabs.get(tabId);
|
||||
if (!tab) return { ok: false };
|
||||
if (this.tabOrder.length <= 1) return { ok: false };
|
||||
|
||||
const closingIndex = this.tabOrder.indexOf(tabId);
|
||||
const nextActiveTabId =
|
||||
this.activeTabId === tabId
|
||||
? this.tabOrder[closingIndex + 1] ?? this.tabOrder[closingIndex - 1] ?? null
|
||||
: this.activeTabId;
|
||||
|
||||
if (this.attachedTabId === tabId && this.window) {
|
||||
this.window.contentView.removeChildView(tab.view);
|
||||
this.attachedTabId = null;
|
||||
}
|
||||
|
||||
this.tabs.delete(tabId);
|
||||
this.tabOrder = this.tabOrder.filter((id) => id !== tabId);
|
||||
this.activeTabId = nextActiveTabId;
|
||||
this.destroyTab(tab);
|
||||
this.syncAttachedView();
|
||||
this.emitState();
|
||||
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
async navigate(rawUrl: string): Promise<{ ok: boolean; error?: string }> {
|
||||
try {
|
||||
const activeTab = this.getActiveTab() ?? this.ensureInitialTab();
|
||||
this.invalidateSnapshot(activeTab.id);
|
||||
await activeTab.view.webContents.loadURL(normalizeNavigationTarget(rawUrl));
|
||||
return { ok: true };
|
||||
} catch (err) {
|
||||
return { ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
}
|
||||
}
|
||||
|
||||
back(): { ok: boolean } {
|
||||
const activeTab = this.getActiveTab();
|
||||
if (!activeTab) return { ok: false };
|
||||
const history = activeTab.view.webContents.navigationHistory;
|
||||
if (!history.canGoBack()) return { ok: false };
|
||||
this.invalidateSnapshot(activeTab.id);
|
||||
history.goBack();
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
forward(): { ok: boolean } {
|
||||
const activeTab = this.getActiveTab();
|
||||
if (!activeTab) return { ok: false };
|
||||
const history = activeTab.view.webContents.navigationHistory;
|
||||
if (!history.canGoForward()) return { ok: false };
|
||||
this.invalidateSnapshot(activeTab.id);
|
||||
history.goForward();
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
reload(): void {
|
||||
const activeTab = this.getActiveTab();
|
||||
if (!activeTab) return;
|
||||
this.invalidateSnapshot(activeTab.id);
|
||||
activeTab.view.webContents.reload();
|
||||
}
|
||||
|
||||
async readPage(
|
||||
options?: { maxElements?: number; maxTextLength?: number; waitForReady?: boolean },
|
||||
signal?: AbortSignal,
|
||||
): Promise<{ ok: boolean; page?: BrowserPageSnapshot; error?: string }> {
|
||||
try {
|
||||
const activeTab = this.getActiveTab() ?? this.ensureInitialTab();
|
||||
const rawSnapshot = await this.executeOnActiveTab<RawBrowserPageSnapshot>(
|
||||
buildReadPageScript(
|
||||
options?.maxElements ?? DEFAULT_READ_MAX_ELEMENTS,
|
||||
options?.maxTextLength ?? DEFAULT_READ_MAX_TEXT_LENGTH,
|
||||
),
|
||||
signal,
|
||||
{ waitForReady: options?.waitForReady },
|
||||
);
|
||||
return {
|
||||
ok: true,
|
||||
page: this.cacheSnapshot(activeTab.id, rawSnapshot, activeTab.view.webContents.isLoading()),
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
ok: false,
|
||||
error: error instanceof Error ? error.message : 'Failed to read the current page.',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async readPageSummary(
|
||||
signal?: AbortSignal,
|
||||
options?: { waitForReady?: boolean },
|
||||
): Promise<BrowserPageSnapshot | null> {
|
||||
const result = await this.readPage(
|
||||
{
|
||||
maxElements: POST_ACTION_MAX_ELEMENTS,
|
||||
maxTextLength: POST_ACTION_MAX_TEXT_LENGTH,
|
||||
waitForReady: options?.waitForReady,
|
||||
},
|
||||
signal,
|
||||
);
|
||||
return result.ok ? result.page ?? null : null;
|
||||
}
|
||||
|
||||
async click(target: ElementTarget, signal?: AbortSignal): Promise<{ ok: boolean; error?: string; description?: string }> {
|
||||
const activeTab = this.getActiveTab();
|
||||
if (!activeTab) {
|
||||
return { ok: false, error: 'No active browser tab is open.' };
|
||||
}
|
||||
|
||||
const resolved = this.resolveElementSelector(activeTab.id, target);
|
||||
if (!resolved.ok) return resolved;
|
||||
|
||||
try {
|
||||
const result = await this.executeOnActiveTab<{
|
||||
ok: boolean;
|
||||
error?: string;
|
||||
description?: string;
|
||||
clickPoint?: {
|
||||
x: number;
|
||||
y: number;
|
||||
};
|
||||
verification?: {
|
||||
before: unknown;
|
||||
targetSelector: string | null;
|
||||
};
|
||||
}>(
|
||||
buildClickScript(resolved.selector),
|
||||
signal,
|
||||
);
|
||||
if (!result.ok) return result;
|
||||
if (!result.clickPoint) {
|
||||
return {
|
||||
ok: false,
|
||||
error: 'Could not determine where to click on the page.',
|
||||
};
|
||||
}
|
||||
|
||||
this.window?.focus();
|
||||
activeTab.view.webContents.focus();
|
||||
activeTab.view.webContents.sendInputEvent({
|
||||
type: 'mouseMove',
|
||||
x: result.clickPoint.x,
|
||||
y: result.clickPoint.y,
|
||||
movementX: 0,
|
||||
movementY: 0,
|
||||
});
|
||||
activeTab.view.webContents.sendInputEvent({
|
||||
type: 'mouseDown',
|
||||
x: result.clickPoint.x,
|
||||
y: result.clickPoint.y,
|
||||
button: 'left',
|
||||
clickCount: 1,
|
||||
});
|
||||
activeTab.view.webContents.sendInputEvent({
|
||||
type: 'mouseUp',
|
||||
x: result.clickPoint.x,
|
||||
y: result.clickPoint.y,
|
||||
button: 'left',
|
||||
clickCount: 1,
|
||||
});
|
||||
|
||||
this.invalidateSnapshot(activeTab.id);
|
||||
await this.waitForWebContentsSettle(activeTab, signal);
|
||||
|
||||
if (result.verification) {
|
||||
const verification = await this.executeOnActiveTab<{ changed: boolean; reasons: string[] }>(
|
||||
buildVerifyClickScript(result.verification.targetSelector, result.verification.before),
|
||||
signal,
|
||||
{ waitForReady: false },
|
||||
);
|
||||
|
||||
if (!verification.changed) {
|
||||
return {
|
||||
ok: false,
|
||||
error: 'Click did not change the page state. Target may not be the correct control.',
|
||||
description: result.description,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
} catch (error) {
|
||||
return {
|
||||
ok: false,
|
||||
error: error instanceof Error ? error.message : 'Failed to click the element.',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async type(target: ElementTarget, text: string, signal?: AbortSignal): Promise<{ ok: boolean; error?: string; description?: string }> {
|
||||
const activeTab = this.getActiveTab();
|
||||
if (!activeTab) {
|
||||
return { ok: false, error: 'No active browser tab is open.' };
|
||||
}
|
||||
|
||||
const resolved = this.resolveElementSelector(activeTab.id, target);
|
||||
if (!resolved.ok) return resolved;
|
||||
|
||||
try {
|
||||
const result = await this.executeOnActiveTab<{ ok: boolean; error?: string; description?: string }>(
|
||||
buildTypeScript(resolved.selector, text),
|
||||
signal,
|
||||
);
|
||||
if (!result.ok) return result;
|
||||
this.invalidateSnapshot(activeTab.id);
|
||||
await this.waitForWebContentsSettle(activeTab, signal);
|
||||
return result;
|
||||
} catch (error) {
|
||||
return {
|
||||
ok: false,
|
||||
error: error instanceof Error ? error.message : 'Failed to type into the element.',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async press(
|
||||
key: string,
|
||||
target?: ElementTarget,
|
||||
signal?: AbortSignal,
|
||||
): Promise<{ ok: boolean; error?: string; description?: string }> {
|
||||
const activeTab = this.getActiveTab();
|
||||
if (!activeTab) {
|
||||
return { ok: false, error: 'No active browser tab is open.' };
|
||||
}
|
||||
|
||||
let description = 'active element';
|
||||
|
||||
if (target?.index != null || target?.selector?.trim()) {
|
||||
const resolved = this.resolveElementSelector(activeTab.id, target);
|
||||
if (!resolved.ok) return resolved;
|
||||
|
||||
try {
|
||||
const focusResult = await this.executeOnActiveTab<{ ok: boolean; error?: string; description?: string }>(
|
||||
buildFocusScript(resolved.selector),
|
||||
signal,
|
||||
);
|
||||
if (!focusResult.ok) return focusResult;
|
||||
description = focusResult.description ?? description;
|
||||
} catch (error) {
|
||||
return {
|
||||
ok: false,
|
||||
error: error instanceof Error ? error.message : 'Failed to focus the element before pressing a key.',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const wc = activeTab.view.webContents;
|
||||
const keyCode = normalizeKeyCode(key);
|
||||
wc.sendInputEvent({ type: 'keyDown', keyCode });
|
||||
if (keyCode.length === 1) {
|
||||
wc.sendInputEvent({ type: 'char', keyCode });
|
||||
}
|
||||
wc.sendInputEvent({ type: 'keyUp', keyCode });
|
||||
|
||||
this.invalidateSnapshot(activeTab.id);
|
||||
await this.waitForWebContentsSettle(activeTab, signal);
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
description: `${keyCode} on ${description}`,
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
ok: false,
|
||||
error: error instanceof Error ? error.message : 'Failed to press the requested key.',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async scroll(direction: 'up' | 'down' = 'down', amount = 700, signal?: AbortSignal): Promise<{ ok: boolean; error?: string }> {
|
||||
const activeTab = this.getActiveTab();
|
||||
if (!activeTab) {
|
||||
return { ok: false, error: 'No active browser tab is open.' };
|
||||
}
|
||||
|
||||
try {
|
||||
const offset = Math.max(1, amount) * (direction === 'up' ? -1 : 1);
|
||||
const result = await this.executeOnActiveTab<{ ok: boolean; error?: string }>(
|
||||
buildScrollScript(offset),
|
||||
signal,
|
||||
);
|
||||
if (!result.ok) return result;
|
||||
this.invalidateSnapshot(activeTab.id);
|
||||
await sleep(250, signal);
|
||||
return result;
|
||||
} catch (error) {
|
||||
return {
|
||||
ok: false,
|
||||
error: error instanceof Error ? error.message : 'Failed to scroll the page.',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async wait(ms = 1000, signal?: AbortSignal): Promise<void> {
|
||||
await sleep(ms, signal);
|
||||
const activeTab = this.getActiveTab();
|
||||
if (!activeTab) return;
|
||||
await this.waitForWebContentsSettle(activeTab, signal);
|
||||
}
|
||||
|
||||
getState(): BrowserState {
|
||||
return this.snapshotState();
|
||||
}
|
||||
|
||||
private snapshotState(): BrowserState {
|
||||
if (this.tabOrder.length === 0) return { ...EMPTY_STATE };
|
||||
return {
|
||||
activeTabId: this.activeTabId,
|
||||
tabs: this.tabOrder
|
||||
.map((tabId) => this.tabs.get(tabId))
|
||||
.filter((tab): tab is BrowserTab => tab != null)
|
||||
.map((tab) => this.snapshotTabState(tab)),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export const browserViewManager = new BrowserViewManager();
|
||||
|
|
@ -52,6 +52,7 @@ import {
|
|||
replaceTrackBlockYaml,
|
||||
deleteTrackBlock,
|
||||
} from '@x/core/dist/knowledge/track/fileops.js';
|
||||
import { browserIpcHandlers } from './browser/ipc.js';
|
||||
|
||||
/**
|
||||
* Convert markdown to a styled HTML document for PDF/DOCX export.
|
||||
|
|
@ -825,5 +826,7 @@ export function setupIpcHandlers() {
|
|||
'billing:getInfo': async () => {
|
||||
return await getBillingInfo();
|
||||
},
|
||||
// Embedded browser handlers (WebContentsView + navigation)
|
||||
...browserIpcHandlers,
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { app, BrowserWindow, desktopCapturer, protocol, net, shell, session } from "electron";
|
||||
import { app, BrowserWindow, desktopCapturer, protocol, net, shell, session, type Session } from "electron";
|
||||
import path from "node:path";
|
||||
import {
|
||||
setupIpcHandlers,
|
||||
|
|
@ -31,6 +31,10 @@ import started from "electron-squirrel-startup";
|
|||
import { execSync, exec, execFileSync } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { init as initChromeSync } from "@x/core/dist/knowledge/chrome-extension/server/server.js";
|
||||
import { registerBrowserControlService } from "@x/core/dist/di/container.js";
|
||||
import { browserViewManager, BROWSER_PARTITION } from "./browser/view.js";
|
||||
import { setupBrowserEventForwarding } from "./browser/ipc.js";
|
||||
import { ElectronBrowserControlService } from "./browser/control-service.js";
|
||||
|
||||
const execAsync = promisify(exec);
|
||||
|
||||
|
|
@ -112,6 +116,30 @@ protocol.registerSchemesAsPrivileged([
|
|||
},
|
||||
]);
|
||||
|
||||
const ALLOWED_SESSION_PERMISSIONS = new Set(["media", "display-capture"]);
|
||||
|
||||
function configureSessionPermissions(targetSession: Session): void {
|
||||
targetSession.setPermissionCheckHandler((_webContents, permission) => {
|
||||
return ALLOWED_SESSION_PERMISSIONS.has(permission);
|
||||
});
|
||||
|
||||
targetSession.setPermissionRequestHandler((_webContents, permission, callback) => {
|
||||
callback(ALLOWED_SESSION_PERMISSIONS.has(permission));
|
||||
});
|
||||
|
||||
// Auto-approve display media requests and route system audio as loopback.
|
||||
// Electron requires a video source in the callback even if we only want audio.
|
||||
// We pass the first available screen source; the renderer discards the video track.
|
||||
targetSession.setDisplayMediaRequestHandler(async (_request, callback) => {
|
||||
const sources = await desktopCapturer.getSources({ types: ['screen'] });
|
||||
if (sources.length === 0) {
|
||||
callback({});
|
||||
return;
|
||||
}
|
||||
callback({ video: sources[0], audio: 'loopback' });
|
||||
});
|
||||
}
|
||||
|
||||
function createWindow() {
|
||||
const win = new BrowserWindow({
|
||||
width: 1280,
|
||||
|
|
@ -131,26 +159,8 @@ function createWindow() {
|
|||
},
|
||||
});
|
||||
|
||||
// Grant microphone and display-capture permissions
|
||||
session.defaultSession.setPermissionRequestHandler((_webContents, permission, callback) => {
|
||||
if (permission === 'media' || permission === 'display-capture') {
|
||||
callback(true);
|
||||
} else {
|
||||
callback(false);
|
||||
}
|
||||
});
|
||||
|
||||
// Auto-approve display media requests and route system audio as loopback.
|
||||
// Electron requires a video source in the callback even if we only want audio.
|
||||
// We pass the first available screen source; the renderer discards the video track.
|
||||
session.defaultSession.setDisplayMediaRequestHandler(async (_request, callback) => {
|
||||
const sources = await desktopCapturer.getSources({ types: ['screen'] });
|
||||
if (sources.length === 0) {
|
||||
callback({});
|
||||
return;
|
||||
}
|
||||
callback({ video: sources[0], audio: 'loopback' });
|
||||
});
|
||||
configureSessionPermissions(session.defaultSession);
|
||||
configureSessionPermissions(session.fromPartition(BROWSER_PARTITION));
|
||||
|
||||
// Show window when content is ready to prevent blank screen
|
||||
win.once("ready-to-show", () => {
|
||||
|
|
@ -175,6 +185,10 @@ function createWindow() {
|
|||
}
|
||||
});
|
||||
|
||||
// Attach the embedded browser pane manager to this window.
|
||||
// The WebContentsView is created lazily on first `browser:setVisible`.
|
||||
browserViewManager.attach(win);
|
||||
|
||||
if (app.isPackaged) {
|
||||
win.loadURL("app://-/index.html");
|
||||
} else {
|
||||
|
|
@ -215,7 +229,10 @@ app.whenReady().then(async () => {
|
|||
// Initialize all config files before UI can access them
|
||||
await initConfigs();
|
||||
|
||||
registerBrowserControlService(new ElectronBrowserControlService());
|
||||
|
||||
setupIpcHandlers();
|
||||
setupBrowserEventForwarding();
|
||||
|
||||
createWindow();
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { contextBridge, ipcRenderer, webUtils } from 'electron';
|
||||
import { contextBridge, ipcRenderer, webFrame, webUtils } from 'electron';
|
||||
import { ipc as ipcShared } from '@x/shared';
|
||||
|
||||
type InvokeChannels = ipcShared.InvokeChannels;
|
||||
|
|
@ -55,4 +55,5 @@ contextBridge.exposeInMainWorld('ipc', ipc);
|
|||
|
||||
contextBridge.exposeInMainWorld('electronUtils', {
|
||||
getPathForFile: (file: File) => webUtils.getPathForFile(file),
|
||||
});
|
||||
getZoomFactor: () => webFrame.getZoomFactor(),
|
||||
});
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import { RunEvent, ListRunsResponse } from '@x/shared/src/runs.js';
|
|||
import type { LanguageModelUsage, ToolUIPart } from 'ai';
|
||||
import './App.css'
|
||||
import z from 'zod';
|
||||
import { CheckIcon, LoaderIcon, PanelLeftIcon, Maximize2, Minimize2, ChevronLeftIcon, ChevronRightIcon, SquarePen, SearchIcon, HistoryIcon, RadioIcon, SquareIcon } from 'lucide-react';
|
||||
import { CheckIcon, LoaderIcon, PanelLeftIcon, Maximize2, Minimize2, ChevronLeftIcon, ChevronRightIcon, SquarePen, SearchIcon, HistoryIcon, RadioIcon, SquareIcon, Globe } from 'lucide-react';
|
||||
import { cn } from '@/lib/utils';
|
||||
import { MarkdownEditor, type MarkdownEditorHandle } from './components/markdown-editor';
|
||||
import { ChatSidebar } from './components/chat-sidebar';
|
||||
|
|
@ -57,6 +57,7 @@ import { OnboardingModal } from '@/components/onboarding'
|
|||
import { CommandPalette, type CommandPaletteContext, type CommandPaletteMention } from '@/components/search-dialog'
|
||||
import { TrackModal } from '@/components/track-modal'
|
||||
import { BackgroundTaskDetail } from '@/components/background-task-detail'
|
||||
import { BrowserPane } from '@/components/browser-pane/BrowserPane'
|
||||
import { VersionHistoryPanel } from '@/components/version-history-panel'
|
||||
import { FileCardProvider } from '@/contexts/file-card-context'
|
||||
import { MarkdownPreOverride } from '@/components/ai-elements/markdown-code-override'
|
||||
|
|
@ -449,23 +450,36 @@ function viewStatesEqual(a: ViewState, b: ViewState): boolean {
|
|||
|
||||
/** Sidebar toggle + utility buttons (fixed position, top-left) */
|
||||
function FixedSidebarToggle({
|
||||
onNavigateBack,
|
||||
onNavigateForward,
|
||||
canNavigateBack,
|
||||
canNavigateForward,
|
||||
onNewChat,
|
||||
onOpenSearch,
|
||||
meetingState,
|
||||
meetingSummarizing,
|
||||
meetingAvailable,
|
||||
onToggleMeeting,
|
||||
isBrowserOpen,
|
||||
onToggleBrowser,
|
||||
leftInsetPx,
|
||||
}: {
|
||||
onNavigateBack: () => void
|
||||
onNavigateForward: () => void
|
||||
canNavigateBack: boolean
|
||||
canNavigateForward: boolean
|
||||
onNewChat: () => void
|
||||
onOpenSearch: () => void
|
||||
meetingState: MeetingTranscriptionState
|
||||
meetingSummarizing: boolean
|
||||
meetingAvailable: boolean
|
||||
onToggleMeeting: () => void
|
||||
isBrowserOpen: boolean
|
||||
onToggleBrowser: () => void
|
||||
leftInsetPx: number
|
||||
}) {
|
||||
const { toggleSidebar } = useSidebar()
|
||||
const { toggleSidebar, state } = useSidebar()
|
||||
const isCollapsed = state === "collapsed"
|
||||
return (
|
||||
<div className="fixed left-0 top-0 z-50 flex h-10 items-center" style={{ WebkitAppRegion: 'no-drag' } as React.CSSProperties}>
|
||||
<div aria-hidden="true" className="h-10 shrink-0" style={{ width: leftInsetPx }} />
|
||||
|
|
@ -528,6 +542,49 @@ function FixedSidebarToggle({
|
|||
</TooltipContent>
|
||||
</Tooltip>
|
||||
)}
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onToggleBrowser}
|
||||
className={cn(
|
||||
"flex h-8 w-8 items-center justify-center rounded-md transition-colors",
|
||||
isBrowserOpen
|
||||
? "bg-accent text-foreground"
|
||||
: "text-muted-foreground hover:bg-accent hover:text-foreground"
|
||||
)}
|
||||
style={{ marginLeft: TITLEBAR_BUTTON_GAP_PX }}
|
||||
aria-label={isBrowserOpen ? "Close browser" : "Open browser"}
|
||||
>
|
||||
<Globe className="size-5" />
|
||||
</button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent side="bottom">{isBrowserOpen ? 'Close browser' : 'Open browser'}</TooltipContent>
|
||||
</Tooltip>
|
||||
{/* Back / Forward navigation */}
|
||||
{isCollapsed && (
|
||||
<>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onNavigateBack}
|
||||
disabled={!canNavigateBack}
|
||||
className="flex h-8 w-8 items-center justify-center rounded-md text-muted-foreground hover:bg-accent hover:text-foreground transition-colors disabled:opacity-30 disabled:pointer-events-none"
|
||||
style={{ marginLeft: TITLEBAR_BUTTON_GAP_PX }}
|
||||
aria-label="Go back"
|
||||
>
|
||||
<ChevronLeftIcon className="size-5" />
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onNavigateForward}
|
||||
disabled={!canNavigateForward}
|
||||
className="flex h-8 w-8 items-center justify-center rounded-md text-muted-foreground hover:bg-accent hover:text-foreground transition-colors disabled:opacity-30 disabled:pointer-events-none"
|
||||
aria-label="Go forward"
|
||||
>
|
||||
<ChevronRightIcon className="size-5" />
|
||||
</button>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
|
@ -606,6 +663,7 @@ function App() {
|
|||
const [expandedPaths, setExpandedPaths] = useState<Set<string>>(new Set())
|
||||
const [recentWikiFiles, setRecentWikiFiles] = useState<string[]>([])
|
||||
const [isGraphOpen, setIsGraphOpen] = useState(false)
|
||||
const [isBrowserOpen, setIsBrowserOpen] = useState(false)
|
||||
const [expandedFrom, setExpandedFrom] = useState<{ path: string | null; graph: boolean } | null>(null)
|
||||
const [baseConfigByPath, setBaseConfigByPath] = useState<Record<string, BaseConfig>>({})
|
||||
const [graphData, setGraphData] = useState<{ nodes: GraphNode[]; edges: GraphEdge[] }>({
|
||||
|
|
@ -2714,6 +2772,24 @@ function App() {
|
|||
setIsChatSidebarOpen(prev => !prev)
|
||||
}, [])
|
||||
|
||||
// Browser is an overlay on the middle pane: opening it forces the chat
|
||||
// sidebar to be visible on the right; closing it restores whatever the
|
||||
// middle pane was showing previously (file/graph/task/chat).
|
||||
const handleToggleBrowser = useCallback(() => {
|
||||
setIsBrowserOpen(prev => {
|
||||
const next = !prev
|
||||
if (next) {
|
||||
setIsChatSidebarOpen(true)
|
||||
setIsRightPaneMaximized(false)
|
||||
}
|
||||
return next
|
||||
})
|
||||
}, [])
|
||||
|
||||
const handleCloseBrowser = useCallback(() => {
|
||||
setIsBrowserOpen(false)
|
||||
}, [])
|
||||
|
||||
const toggleRightPaneMaximize = useCallback(() => {
|
||||
setIsChatSidebarOpen(true)
|
||||
setIsRightPaneMaximized(prev => !prev)
|
||||
|
|
@ -2797,6 +2873,9 @@ function App() {
|
|||
case 'file':
|
||||
setSelectedBackgroundTask(null)
|
||||
setIsGraphOpen(false)
|
||||
// Navigating to a file dismisses the browser overlay so the file is
|
||||
// visible in the middle pane.
|
||||
setIsBrowserOpen(false)
|
||||
setExpandedFrom(null)
|
||||
// Preserve split vs knowledge-max mode when navigating knowledge files.
|
||||
// Only exit chat-only maximize, because that would hide the selected file.
|
||||
|
|
@ -2809,6 +2888,7 @@ function App() {
|
|||
case 'graph':
|
||||
setSelectedBackgroundTask(null)
|
||||
setSelectedPath(null)
|
||||
setIsBrowserOpen(false)
|
||||
setExpandedFrom(null)
|
||||
setIsGraphOpen(true)
|
||||
ensureGraphFileTab()
|
||||
|
|
@ -2819,6 +2899,7 @@ function App() {
|
|||
case 'task':
|
||||
setSelectedPath(null)
|
||||
setIsGraphOpen(false)
|
||||
setIsBrowserOpen(false)
|
||||
setExpandedFrom(null)
|
||||
setIsRightPaneMaximized(false)
|
||||
setSelectedBackgroundTask(view.name)
|
||||
|
|
@ -2826,6 +2907,8 @@ function App() {
|
|||
case 'chat':
|
||||
setSelectedPath(null)
|
||||
setIsGraphOpen(false)
|
||||
// Don't touch isBrowserOpen here — chat navigation should land in
|
||||
// the right sidebar when the browser overlay is active.
|
||||
setExpandedFrom(null)
|
||||
setIsRightPaneMaximized(false)
|
||||
setSelectedBackgroundTask(null)
|
||||
|
|
@ -3101,7 +3184,7 @@ function App() {
|
|||
}, [])
|
||||
|
||||
// Keyboard shortcut: Ctrl+L to toggle main chat view
|
||||
const isFullScreenChat = !selectedPath && !isGraphOpen && !selectedBackgroundTask
|
||||
const isFullScreenChat = !selectedPath && !isGraphOpen && !selectedBackgroundTask && !isBrowserOpen
|
||||
useEffect(() => {
|
||||
const handleKeyDown = (e: KeyboardEvent) => {
|
||||
if ((e.ctrlKey || e.metaKey) && e.key === 'l') {
|
||||
|
|
@ -3964,7 +4047,7 @@ function App() {
|
|||
const selectedTask = selectedBackgroundTask
|
||||
? backgroundTasks.find(t => t.name === selectedBackgroundTask)
|
||||
: null
|
||||
const isRightPaneContext = Boolean(selectedPath || isGraphOpen)
|
||||
const isRightPaneContext = Boolean(selectedPath || isGraphOpen || isBrowserOpen)
|
||||
const isRightPaneOnlyMode = isRightPaneContext && isChatSidebarOpen && isRightPaneMaximized
|
||||
const shouldCollapseLeftPane = isRightPaneOnlyMode
|
||||
const openMarkdownTabs = React.useMemo(() => {
|
||||
|
|
@ -4006,7 +4089,7 @@ function App() {
|
|||
onNewChat: handleNewChatTab,
|
||||
onSelectRun: (runIdToLoad) => {
|
||||
cancelRecordingIfActive()
|
||||
if (selectedPath || isGraphOpen) {
|
||||
if (selectedPath || isGraphOpen || isBrowserOpen) {
|
||||
setIsChatSidebarOpen(true)
|
||||
}
|
||||
|
||||
|
|
@ -4016,8 +4099,8 @@ function App() {
|
|||
switchChatTab(existingTab.id)
|
||||
return
|
||||
}
|
||||
// In two-pane mode, keep current knowledge/graph context and just swap chat context.
|
||||
if (selectedPath || isGraphOpen) {
|
||||
// In two-pane mode (file/graph/browser), keep the middle pane and just swap chat context in the right sidebar.
|
||||
if (selectedPath || isGraphOpen || isBrowserOpen) {
|
||||
setChatTabs(prev => prev.map(t => t.id === activeChatTabId ? { ...t, runId: runIdToLoad } : t))
|
||||
loadRun(runIdToLoad)
|
||||
return
|
||||
|
|
@ -4041,14 +4124,14 @@ function App() {
|
|||
} else {
|
||||
// Only one tab, reset it to new chat
|
||||
setChatTabs([{ id: tabForRun.id, runId: null }])
|
||||
if (selectedPath || isGraphOpen) {
|
||||
if (selectedPath || isGraphOpen || isBrowserOpen) {
|
||||
handleNewChat()
|
||||
} else {
|
||||
void navigateToView({ type: 'chat', runId: null })
|
||||
}
|
||||
}
|
||||
} else if (runId === runIdToDelete) {
|
||||
if (selectedPath || isGraphOpen) {
|
||||
if (selectedPath || isGraphOpen || isBrowserOpen) {
|
||||
setChatTabs(prev => prev.map(t => t.id === activeChatTabId ? { ...t, runId: null } : t))
|
||||
handleNewChat()
|
||||
} else {
|
||||
|
|
@ -4146,7 +4229,7 @@ function App() {
|
|||
<TooltipContent side="bottom">Version history</TooltipContent>
|
||||
</Tooltip>
|
||||
)}
|
||||
{!selectedPath && !isGraphOpen && !selectedTask && (
|
||||
{!selectedPath && !isGraphOpen && !selectedTask && !isBrowserOpen && (
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<button
|
||||
|
|
@ -4161,7 +4244,7 @@ function App() {
|
|||
<TooltipContent side="bottom">New chat tab</TooltipContent>
|
||||
</Tooltip>
|
||||
)}
|
||||
{!selectedPath && !isGraphOpen && expandedFrom && (
|
||||
{!selectedPath && !isGraphOpen && !isBrowserOpen && expandedFrom && (
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<button
|
||||
|
|
@ -4195,7 +4278,9 @@ function App() {
|
|||
)}
|
||||
</ContentHeader>
|
||||
|
||||
{selectedPath && isBaseFilePath(selectedPath) ? (
|
||||
{isBrowserOpen ? (
|
||||
<BrowserPane onClose={handleCloseBrowser} />
|
||||
) : selectedPath && isBaseFilePath(selectedPath) ? (
|
||||
<div className="flex-1 min-h-0 flex flex-col overflow-hidden">
|
||||
<BasesView
|
||||
tree={tree}
|
||||
|
|
@ -4562,12 +4647,18 @@ function App() {
|
|||
)}
|
||||
{/* Rendered last so its no-drag region paints over the sidebar drag region */}
|
||||
<FixedSidebarToggle
|
||||
onNavigateBack={() => { void navigateBack() }}
|
||||
onNavigateForward={() => { void navigateForward() }}
|
||||
canNavigateBack={canNavigateBack}
|
||||
canNavigateForward={canNavigateForward}
|
||||
onNewChat={handleNewChatTab}
|
||||
onOpenSearch={() => setIsSearchOpen(true)}
|
||||
meetingState={meetingTranscription.state}
|
||||
meetingSummarizing={meetingSummarizing}
|
||||
meetingAvailable={voiceAvailable}
|
||||
onToggleMeeting={() => { void handleToggleMeeting() }}
|
||||
isBrowserOpen={isBrowserOpen}
|
||||
onToggleBrowser={handleToggleBrowser}
|
||||
leftInsetPx={isMac ? MACOS_TRAFFIC_LIGHTS_RESERVED_PX : 0}
|
||||
/>
|
||||
</SidebarProvider>
|
||||
|
|
|
|||
|
|
@ -98,24 +98,33 @@ export const ToolHeader = ({
|
|||
type,
|
||||
state,
|
||||
...props
|
||||
}: ToolHeaderProps) => (
|
||||
<CollapsibleTrigger
|
||||
className={cn(
|
||||
"flex w-full items-center justify-between gap-4 p-3",
|
||||
className
|
||||
)}
|
||||
{...props}
|
||||
>
|
||||
<div className="flex items-center gap-2">
|
||||
<WrenchIcon className="size-4 text-muted-foreground" />
|
||||
<span className="font-medium text-sm">
|
||||
{title ?? type.split("-").slice(1).join("-")}
|
||||
</span>
|
||||
{getStatusBadge(state)}
|
||||
</div>
|
||||
<ChevronDownIcon className="size-4 text-muted-foreground transition-transform group-data-[state=open]:rotate-180" />
|
||||
</CollapsibleTrigger>
|
||||
);
|
||||
}: ToolHeaderProps) => {
|
||||
const displayTitle = title ?? type.split("-").slice(1).join("-")
|
||||
|
||||
return (
|
||||
<CollapsibleTrigger
|
||||
className={cn(
|
||||
"flex w-full items-center justify-between gap-4 p-3",
|
||||
className
|
||||
)}
|
||||
{...props}
|
||||
>
|
||||
<div className="flex min-w-0 flex-1 items-center gap-2">
|
||||
<WrenchIcon className="size-4 shrink-0 text-muted-foreground" />
|
||||
<span
|
||||
className="min-w-0 flex-1 truncate text-left font-medium text-sm"
|
||||
title={displayTitle}
|
||||
>
|
||||
{displayTitle}
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex shrink-0 items-center gap-3">
|
||||
{getStatusBadge(state)}
|
||||
<ChevronDownIcon className="size-4 text-muted-foreground transition-transform group-data-[state=open]:rotate-180" />
|
||||
</div>
|
||||
</CollapsibleTrigger>
|
||||
)
|
||||
};
|
||||
|
||||
export type ToolContentProps = ComponentProps<typeof CollapsibleContent>;
|
||||
|
||||
|
|
@ -215,4 +224,3 @@ export const ToolTabbedContent = ({
|
|||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
|
|
|
|||
418
apps/x/apps/renderer/src/components/browser-pane/BrowserPane.tsx
Normal file
418
apps/x/apps/renderer/src/components/browser-pane/BrowserPane.tsx
Normal file
|
|
@ -0,0 +1,418 @@
|
|||
import { useCallback, useEffect, useRef, useState } from 'react'
|
||||
import { ArrowLeft, ArrowRight, Loader2, Plus, RotateCw, X } from 'lucide-react'
|
||||
|
||||
import { TabBar } from '@/components/tab-bar'
|
||||
import { cn } from '@/lib/utils'
|
||||
|
||||
/**
|
||||
* Embedded browser pane.
|
||||
*
|
||||
* Renders a transparent placeholder div whose bounds are reported to the
|
||||
* main process via `browser:setBounds`. The actual browsing surface is an
|
||||
* Electron WebContentsView layered on top of the renderer by the main
|
||||
* process — this component only owns the chrome (tabs, address bar, nav
|
||||
* buttons) and the sizing/visibility lifecycle.
|
||||
*/
|
||||
|
||||
interface BrowserTabState {
|
||||
id: string
|
||||
url: string
|
||||
title: string
|
||||
canGoBack: boolean
|
||||
canGoForward: boolean
|
||||
loading: boolean
|
||||
}
|
||||
|
||||
interface BrowserState {
|
||||
activeTabId: string | null
|
||||
tabs: BrowserTabState[]
|
||||
}
|
||||
|
||||
const EMPTY_STATE: BrowserState = {
|
||||
activeTabId: null,
|
||||
tabs: [],
|
||||
}
|
||||
|
||||
const CHROME_HEIGHT = 40
|
||||
const BLOCKING_OVERLAY_SLOTS = new Set([
|
||||
'alert-dialog-content',
|
||||
'context-menu-content',
|
||||
'context-menu-sub-content',
|
||||
'dialog-content',
|
||||
'dropdown-menu-content',
|
||||
'dropdown-menu-sub-content',
|
||||
'hover-card-content',
|
||||
'popover-content',
|
||||
'select-content',
|
||||
'sheet-content',
|
||||
])
|
||||
|
||||
interface BrowserPaneProps {
|
||||
onClose: () => void
|
||||
}
|
||||
|
||||
const getActiveTab = (state: BrowserState) =>
|
||||
state.tabs.find((tab) => tab.id === state.activeTabId) ?? null
|
||||
|
||||
const isVisibleOverlayElement = (el: HTMLElement) => {
|
||||
const style = window.getComputedStyle(el)
|
||||
if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') {
|
||||
return false
|
||||
}
|
||||
const rect = el.getBoundingClientRect()
|
||||
return rect.width > 0 && rect.height > 0
|
||||
}
|
||||
|
||||
const hasBlockingOverlay = (doc: Document) => {
|
||||
const openContent = doc.querySelectorAll<HTMLElement>('[data-slot][data-state="open"]')
|
||||
return Array.from(openContent).some((el) => {
|
||||
const slot = el.dataset.slot
|
||||
if (!slot || !BLOCKING_OVERLAY_SLOTS.has(slot)) return false
|
||||
return isVisibleOverlayElement(el)
|
||||
})
|
||||
}
|
||||
|
||||
const getBrowserTabTitle = (tab: BrowserTabState) => {
|
||||
const title = tab.title.trim()
|
||||
if (title) return title
|
||||
const url = tab.url.trim()
|
||||
if (!url) return 'New tab'
|
||||
try {
|
||||
const parsed = new URL(url)
|
||||
return parsed.hostname || parsed.href
|
||||
} catch {
|
||||
return url.replace(/^https?:\/\//i, '') || 'New tab'
|
||||
}
|
||||
}
|
||||
|
||||
export function BrowserPane({ onClose }: BrowserPaneProps) {
|
||||
const [state, setState] = useState<BrowserState>(EMPTY_STATE)
|
||||
const [addressValue, setAddressValue] = useState('')
|
||||
|
||||
const activeTabIdRef = useRef<string | null>(null)
|
||||
const addressFocusedRef = useRef(false)
|
||||
const viewportRef = useRef<HTMLDivElement>(null)
|
||||
const lastBoundsRef = useRef<{ x: number; y: number; width: number; height: number } | null>(null)
|
||||
const viewVisibleRef = useRef(false)
|
||||
|
||||
const activeTab = getActiveTab(state)
|
||||
|
||||
const applyState = useCallback((next: BrowserState) => {
|
||||
const previousActiveTabId = activeTabIdRef.current
|
||||
activeTabIdRef.current = next.activeTabId
|
||||
setState(next)
|
||||
|
||||
const nextActiveTab = getActiveTab(next)
|
||||
if (!addressFocusedRef.current || next.activeTabId !== previousActiveTabId) {
|
||||
setAddressValue(nextActiveTab?.url ?? '')
|
||||
}
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
const cleanup = window.ipc.on('browser:didUpdateState', (incoming) => {
|
||||
applyState(incoming as BrowserState)
|
||||
})
|
||||
|
||||
void window.ipc.invoke('browser:getState', null).then((initial) => {
|
||||
applyState(initial as BrowserState)
|
||||
})
|
||||
|
||||
return cleanup
|
||||
}, [applyState])
|
||||
|
||||
const setViewVisible = useCallback((visible: boolean) => {
|
||||
if (viewVisibleRef.current === visible) return
|
||||
viewVisibleRef.current = visible
|
||||
void window.ipc.invoke('browser:setVisible', { visible })
|
||||
}, [])
|
||||
|
||||
const measureBounds = useCallback(() => {
|
||||
const el = viewportRef.current
|
||||
if (!el) return null
|
||||
|
||||
const zoomFactor = Math.max(window.electronUtils.getZoomFactor(), 0.01)
|
||||
const rect = el.getBoundingClientRect()
|
||||
const chatSidebar = el.ownerDocument.querySelector<HTMLElement>('[data-chat-sidebar-root]')
|
||||
const chatSidebarRect = chatSidebar?.getBoundingClientRect()
|
||||
const clampedRightCss = chatSidebarRect && chatSidebarRect.width > 0
|
||||
? Math.min(rect.right, chatSidebarRect.left)
|
||||
: rect.right
|
||||
|
||||
// `getBoundingClientRect()` is reported in zoomed CSS pixels. Electron's
|
||||
// native view bounds are in unzoomed window coordinates, so convert back
|
||||
// using the renderer zoom factor before calling into the main process.
|
||||
const left = Math.ceil(rect.left * zoomFactor)
|
||||
const top = Math.ceil(rect.top * zoomFactor)
|
||||
const right = Math.floor(clampedRightCss * zoomFactor)
|
||||
const bottom = Math.floor(rect.bottom * zoomFactor)
|
||||
const width = right - left
|
||||
const height = bottom - top
|
||||
|
||||
if (width <= 0 || height <= 0) return null
|
||||
|
||||
return {
|
||||
x: left,
|
||||
y: top,
|
||||
width,
|
||||
height,
|
||||
}
|
||||
}, [])
|
||||
|
||||
const pushBounds = useCallback((bounds: { x: number; y: number; width: number; height: number }) => {
|
||||
const last = lastBoundsRef.current
|
||||
if (
|
||||
last &&
|
||||
last.x === bounds.x &&
|
||||
last.y === bounds.y &&
|
||||
last.width === bounds.width &&
|
||||
last.height === bounds.height
|
||||
) {
|
||||
return bounds
|
||||
}
|
||||
lastBoundsRef.current = bounds
|
||||
void window.ipc.invoke('browser:setBounds', bounds)
|
||||
return bounds
|
||||
}, [])
|
||||
|
||||
const syncView = useCallback(() => {
|
||||
const doc = viewportRef.current?.ownerDocument
|
||||
if (doc && hasBlockingOverlay(doc)) {
|
||||
lastBoundsRef.current = null
|
||||
setViewVisible(false)
|
||||
return null
|
||||
}
|
||||
|
||||
const bounds = measureBounds()
|
||||
if (!bounds) {
|
||||
lastBoundsRef.current = null
|
||||
setViewVisible(false)
|
||||
return null
|
||||
}
|
||||
pushBounds(bounds)
|
||||
setViewVisible(true)
|
||||
return bounds
|
||||
}, [measureBounds, pushBounds, setViewVisible])
|
||||
|
||||
useEffect(() => {
|
||||
syncView()
|
||||
}, [activeTab?.id, activeTab?.loading, activeTab?.url, syncView])
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false
|
||||
const rafId = requestAnimationFrame(() => {
|
||||
if (cancelled) return
|
||||
syncView()
|
||||
})
|
||||
return () => {
|
||||
cancelled = true
|
||||
cancelAnimationFrame(rafId)
|
||||
lastBoundsRef.current = null
|
||||
setViewVisible(false)
|
||||
}
|
||||
}, [setViewVisible, syncView])
|
||||
|
||||
useEffect(() => {
|
||||
const el = viewportRef.current
|
||||
if (!el) return
|
||||
|
||||
const sidebarInset = el.closest<HTMLElement>('[data-slot="sidebar-inset"]')
|
||||
const chatSidebar = el.ownerDocument.querySelector<HTMLElement>('[data-chat-sidebar-root]')
|
||||
const documentElement = el.ownerDocument.documentElement
|
||||
|
||||
let pendingRaf: number | null = null
|
||||
const schedule = () => {
|
||||
if (pendingRaf !== null) return
|
||||
pendingRaf = requestAnimationFrame(() => {
|
||||
pendingRaf = null
|
||||
syncView()
|
||||
})
|
||||
}
|
||||
|
||||
const ro = new ResizeObserver(schedule)
|
||||
ro.observe(el)
|
||||
if (sidebarInset) ro.observe(sidebarInset)
|
||||
if (chatSidebar) ro.observe(chatSidebar)
|
||||
ro.observe(documentElement)
|
||||
|
||||
return () => {
|
||||
if (pendingRaf !== null) cancelAnimationFrame(pendingRaf)
|
||||
ro.disconnect()
|
||||
}
|
||||
}, [syncView])
|
||||
|
||||
useEffect(() => {
|
||||
const doc = viewportRef.current?.ownerDocument
|
||||
if (!doc?.body) return
|
||||
|
||||
let pendingRaf: number | null = null
|
||||
const schedule = () => {
|
||||
if (pendingRaf !== null) return
|
||||
pendingRaf = requestAnimationFrame(() => {
|
||||
pendingRaf = null
|
||||
syncView()
|
||||
})
|
||||
}
|
||||
|
||||
const observer = new MutationObserver(schedule)
|
||||
observer.observe(doc.body, {
|
||||
subtree: true,
|
||||
childList: true,
|
||||
attributes: true,
|
||||
attributeFilter: ['data-state', 'style', 'hidden', 'aria-hidden', 'open'],
|
||||
})
|
||||
|
||||
return () => {
|
||||
if (pendingRaf !== null) cancelAnimationFrame(pendingRaf)
|
||||
observer.disconnect()
|
||||
}
|
||||
}, [syncView])
|
||||
|
||||
const handleNewTab = useCallback(() => {
|
||||
void window.ipc.invoke('browser:newTab', {}).then((res) => {
|
||||
const result = res as { ok: boolean; error?: string }
|
||||
if (!result.ok && result.error) {
|
||||
console.error('browser:newTab failed', result.error)
|
||||
}
|
||||
})
|
||||
}, [])
|
||||
|
||||
const handleSwitchTab = useCallback((tabId: string) => {
|
||||
void window.ipc.invoke('browser:switchTab', { tabId })
|
||||
}, [])
|
||||
|
||||
const handleCloseTab = useCallback((tabId: string) => {
|
||||
void window.ipc.invoke('browser:closeTab', { tabId })
|
||||
}, [])
|
||||
|
||||
const handleSubmitAddress = useCallback((e: React.FormEvent) => {
|
||||
e.preventDefault()
|
||||
const trimmed = addressValue.trim()
|
||||
if (!trimmed) return
|
||||
void window.ipc.invoke('browser:navigate', { url: trimmed }).then((res) => {
|
||||
const result = res as { ok: boolean; error?: string }
|
||||
if (!result.ok && result.error) {
|
||||
console.error('browser:navigate failed', result.error)
|
||||
}
|
||||
})
|
||||
}, [addressValue])
|
||||
|
||||
const handleBack = useCallback(() => {
|
||||
void window.ipc.invoke('browser:back', null)
|
||||
}, [])
|
||||
|
||||
const handleForward = useCallback(() => {
|
||||
void window.ipc.invoke('browser:forward', null)
|
||||
}, [])
|
||||
|
||||
const handleReload = useCallback(() => {
|
||||
void window.ipc.invoke('browser:reload', null)
|
||||
}, [])
|
||||
|
||||
return (
|
||||
<div className="flex min-h-0 min-w-0 flex-1 flex-col overflow-hidden bg-background">
|
||||
<div className="flex h-9 shrink-0 items-stretch border-b border-border bg-sidebar">
|
||||
<TabBar
|
||||
tabs={state.tabs}
|
||||
activeTabId={state.activeTabId ?? ''}
|
||||
getTabTitle={getBrowserTabTitle}
|
||||
getTabId={(tab) => tab.id}
|
||||
onSwitchTab={handleSwitchTab}
|
||||
onCloseTab={handleCloseTab}
|
||||
layout="scroll"
|
||||
/>
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleNewTab}
|
||||
className="flex h-9 w-9 shrink-0 items-center justify-center border-l border-border text-muted-foreground transition-colors hover:bg-accent hover:text-foreground"
|
||||
aria-label="New browser tab"
|
||||
>
|
||||
<Plus className="size-4" />
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div
|
||||
className="flex h-10 shrink-0 items-center gap-1 border-b border-border bg-sidebar px-2"
|
||||
style={{ minHeight: CHROME_HEIGHT }}
|
||||
>
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleBack}
|
||||
disabled={!activeTab?.canGoBack}
|
||||
className={cn(
|
||||
'flex h-7 w-7 items-center justify-center rounded-md text-muted-foreground transition-colors',
|
||||
activeTab?.canGoBack ? 'hover:bg-accent hover:text-foreground' : 'opacity-40',
|
||||
)}
|
||||
aria-label="Back"
|
||||
>
|
||||
<ArrowLeft className="size-4" />
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleForward}
|
||||
disabled={!activeTab?.canGoForward}
|
||||
className={cn(
|
||||
'flex h-7 w-7 items-center justify-center rounded-md text-muted-foreground transition-colors',
|
||||
activeTab?.canGoForward ? 'hover:bg-accent hover:text-foreground' : 'opacity-40',
|
||||
)}
|
||||
aria-label="Forward"
|
||||
>
|
||||
<ArrowRight className="size-4" />
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleReload}
|
||||
disabled={!activeTab}
|
||||
className={cn(
|
||||
'flex h-7 w-7 items-center justify-center rounded-md text-muted-foreground transition-colors',
|
||||
activeTab ? 'hover:bg-accent hover:text-foreground' : 'opacity-40',
|
||||
)}
|
||||
aria-label="Reload"
|
||||
>
|
||||
{activeTab?.loading ? (
|
||||
<Loader2 className="size-4 animate-spin" />
|
||||
) : (
|
||||
<RotateCw className="size-4" />
|
||||
)}
|
||||
</button>
|
||||
<form onSubmit={handleSubmitAddress} className="flex-1 min-w-0">
|
||||
<input
|
||||
type="text"
|
||||
value={addressValue}
|
||||
onChange={(e) => setAddressValue(e.target.value)}
|
||||
onFocus={(e) => {
|
||||
addressFocusedRef.current = true
|
||||
e.currentTarget.select()
|
||||
}}
|
||||
onBlur={() => {
|
||||
addressFocusedRef.current = false
|
||||
setAddressValue(activeTab?.url ?? '')
|
||||
}}
|
||||
placeholder="Enter URL or search..."
|
||||
className={cn(
|
||||
'h-7 w-full rounded-md border border-transparent bg-background px-3 text-sm text-foreground',
|
||||
'placeholder:text-muted-foreground/60',
|
||||
'focus:border-border focus:outline-hidden',
|
||||
)}
|
||||
spellCheck={false}
|
||||
autoCorrect="off"
|
||||
autoCapitalize="off"
|
||||
/>
|
||||
</form>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onClose}
|
||||
className="ml-1 flex h-7 w-7 shrink-0 items-center justify-center rounded-md text-muted-foreground transition-colors hover:bg-accent hover:text-foreground"
|
||||
aria-label="Close browser"
|
||||
>
|
||||
<X className="size-4" />
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div
|
||||
ref={viewportRef}
|
||||
className="relative min-h-0 min-w-0 flex-1"
|
||||
data-browser-viewport
|
||||
/>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
|
@ -76,12 +76,18 @@ function matchBillingError(message: string) {
|
|||
return BILLING_ERROR_PATTERNS.find(({ pattern }) => pattern.test(message)) ?? null
|
||||
}
|
||||
|
||||
interface BillingRowboatAccount {
|
||||
config?: {
|
||||
appUrl?: string | null
|
||||
} | null
|
||||
}
|
||||
|
||||
function BillingErrorCTA({ label }: { label: string }) {
|
||||
const [appUrl, setAppUrl] = useState<string | null>(null)
|
||||
|
||||
useEffect(() => {
|
||||
window.ipc.invoke('account:getRowboat', null)
|
||||
.then((account: any) => setAppUrl(account.config?.appUrl ?? null))
|
||||
.then((account: BillingRowboatAccount) => setAppUrl(account.config?.appUrl ?? null))
|
||||
.catch(() => {})
|
||||
}, [])
|
||||
|
||||
|
|
@ -467,6 +473,7 @@ export function ChatSidebar({
|
|||
return (
|
||||
<div
|
||||
ref={paneRef}
|
||||
data-chat-sidebar-root
|
||||
onMouseDownCapture={onActivate}
|
||||
onFocusCapture={onActivate}
|
||||
className={cn(
|
||||
|
|
|
|||
3
apps/x/apps/renderer/src/global.d.ts
vendored
3
apps/x/apps/renderer/src/global.d.ts
vendored
|
|
@ -35,8 +35,9 @@ declare global {
|
|||
};
|
||||
electronUtils: {
|
||||
getPathForFile: (file: File) => string;
|
||||
getZoomFactor: () => number;
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export { };
|
||||
export { };
|
||||
|
|
|
|||
|
|
@ -231,6 +231,194 @@ export const getAppActionCardData = (tool: ToolCall): AppActionCardData | null =
|
|||
}
|
||||
}
|
||||
|
||||
const BROWSER_PENDING_LABELS: Record<string, string> = {
|
||||
open: 'Opening browser...',
|
||||
'get-state': 'Reading browser state...',
|
||||
'new-tab': 'Opening new browser tab...',
|
||||
'switch-tab': 'Switching browser tab...',
|
||||
'close-tab': 'Closing browser tab...',
|
||||
navigate: 'Navigating browser...',
|
||||
back: 'Going back...',
|
||||
forward: 'Going forward...',
|
||||
reload: 'Reloading page...',
|
||||
'read-page': 'Reading page...',
|
||||
click: 'Clicking page element...',
|
||||
type: 'Typing into page...',
|
||||
press: 'Sending key press...',
|
||||
scroll: 'Scrolling page...',
|
||||
wait: 'Waiting for page...',
|
||||
}
|
||||
|
||||
const truncateLabel = (value: string, max = 72): string => {
|
||||
const normalized = value.replace(/\s+/g, ' ').trim()
|
||||
if (normalized.length <= max) return normalized
|
||||
return `${normalized.slice(0, Math.max(0, max - 3)).trim()}...`
|
||||
}
|
||||
|
||||
const safeBrowserString = (value: unknown): string | null => {
|
||||
return typeof value === 'string' && value.trim() ? value.trim() : null
|
||||
}
|
||||
|
||||
const parseBrowserUrl = (value: string | null): URL | null => {
|
||||
if (!value) return null
|
||||
try {
|
||||
return new URL(value)
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
const getGoogleSearchQuery = (value: string | null): string | null => {
|
||||
const parsed = parseBrowserUrl(value)
|
||||
if (!parsed) return null
|
||||
const hostname = parsed.hostname.replace(/^www\./, '')
|
||||
if (hostname !== 'google.com' && !hostname.endsWith('.google.com')) return null
|
||||
if (parsed.pathname !== '/search') return null
|
||||
const query = parsed.searchParams.get('q')?.trim()
|
||||
return query ? truncateLabel(query, 56) : null
|
||||
}
|
||||
|
||||
const formatBrowserTarget = (value: string | null): string | null => {
|
||||
const parsed = parseBrowserUrl(value)
|
||||
if (!parsed) {
|
||||
return value ? truncateLabel(value, 56) : null
|
||||
}
|
||||
|
||||
const hostname = parsed.hostname.replace(/^www\./, '')
|
||||
const path = parsed.pathname === '/' ? '' : parsed.pathname
|
||||
const suffix = parsed.search ? `${path}${parsed.search}` : path
|
||||
return truncateLabel(`${hostname}${suffix}`, 56)
|
||||
}
|
||||
|
||||
const sanitizeBrowserDescription = (value: string | null): string | null => {
|
||||
if (!value) return null
|
||||
|
||||
let text = value
|
||||
.replace(/^(clicked|typed into|pressed)\s+/i, '')
|
||||
.replace(/\.$/, '')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
|
||||
if (!text) return null
|
||||
|
||||
const looksLikeCssNoise =
|
||||
/(^|[\s"])(body|html)\b/i.test(text)
|
||||
|| /display:|position:|background-color|align-items|justify-content|z-index|var\(--|left:|top:/i.test(text)
|
||||
|| /\.[A-Za-z0-9_-]+\{/.test(text)
|
||||
|
||||
if (looksLikeCssNoise || text.length > 88) {
|
||||
const quoted = Array.from(text.matchAll(/"([^"]+)"/g))
|
||||
.map((match) => match[1]?.trim())
|
||||
.find((candidate) => candidate && !/display:|position:|background-color|var\(--/i.test(candidate))
|
||||
|
||||
if (!quoted) return null
|
||||
text = `"${truncateLabel(quoted, 44)}"`
|
||||
}
|
||||
|
||||
if (/^(body|html)\b/i.test(text)) return null
|
||||
return truncateLabel(text, 64)
|
||||
}
|
||||
|
||||
const getBrowserSuccessLabel = (
|
||||
action: string,
|
||||
input: Record<string, unknown> | undefined,
|
||||
result: Record<string, unknown> | undefined,
|
||||
): string | null => {
|
||||
const page = result?.page as Record<string, unknown> | undefined
|
||||
const pageUrl = safeBrowserString(page?.url)
|
||||
const resultMessage = safeBrowserString(result?.message)
|
||||
|
||||
switch (action) {
|
||||
case 'open':
|
||||
return 'Opened browser'
|
||||
case 'get-state':
|
||||
return 'Read browser state'
|
||||
case 'new-tab': {
|
||||
const query = getGoogleSearchQuery(pageUrl)
|
||||
if (query) return `Opened search for "${query}"`
|
||||
const target = formatBrowserTarget(pageUrl) || safeBrowserString(input?.target)
|
||||
return target ? `Opened ${target}` : 'Opened new tab'
|
||||
}
|
||||
case 'switch-tab':
|
||||
return 'Switched browser tab'
|
||||
case 'close-tab':
|
||||
return 'Closed browser tab'
|
||||
case 'navigate': {
|
||||
const query = getGoogleSearchQuery(pageUrl)
|
||||
if (query) return `Searched Google for "${query}"`
|
||||
const target = formatBrowserTarget(pageUrl) || formatBrowserTarget(safeBrowserString(input?.target))
|
||||
return target ? `Opened ${target}` : 'Navigated browser'
|
||||
}
|
||||
case 'back':
|
||||
return 'Went back'
|
||||
case 'forward':
|
||||
return 'Went forward'
|
||||
case 'reload':
|
||||
return 'Reloaded page'
|
||||
case 'read-page': {
|
||||
const title = safeBrowserString(page?.title)
|
||||
return title ? `Read ${truncateLabel(title, 52)}` : 'Read page'
|
||||
}
|
||||
case 'click': {
|
||||
const detail = sanitizeBrowserDescription(resultMessage)
|
||||
if (detail) return `Clicked ${detail}`
|
||||
if (typeof input?.index === 'number') return `Clicked element ${input.index}`
|
||||
return 'Clicked page element'
|
||||
}
|
||||
case 'type': {
|
||||
const detail = sanitizeBrowserDescription(resultMessage)
|
||||
if (detail) return `Typed into ${detail}`
|
||||
if (typeof input?.index === 'number') return `Typed into element ${input.index}`
|
||||
return 'Typed into page'
|
||||
}
|
||||
case 'press': {
|
||||
const key = safeBrowserString(input?.key)
|
||||
return key ? `Pressed ${truncateLabel(key, 20)}` : 'Sent key press'
|
||||
}
|
||||
case 'scroll':
|
||||
return `Scrolled ${input?.direction === 'up' ? 'up' : 'down'}`
|
||||
case 'wait': {
|
||||
const ms = typeof input?.ms === 'number' ? input.ms : 1000
|
||||
return `Waited ${ms}ms`
|
||||
}
|
||||
default:
|
||||
return resultMessage ? truncateLabel(resultMessage, 72) : 'Controlled browser'
|
||||
}
|
||||
}
|
||||
|
||||
export const getBrowserControlLabel = (tool: ToolCall): string | null => {
|
||||
if (tool.name !== 'browser-control') return null
|
||||
|
||||
const input = normalizeToolInput(tool.input) as Record<string, unknown> | undefined
|
||||
const result = tool.result as Record<string, unknown> | undefined
|
||||
const action = (input?.action as string | undefined) || (result?.action as string | undefined) || 'browser'
|
||||
|
||||
if (tool.status !== 'completed') {
|
||||
if (action === 'click' && typeof input?.index === 'number') {
|
||||
return `Clicking element ${input.index}...`
|
||||
}
|
||||
if (action === 'type' && typeof input?.index === 'number') {
|
||||
return `Typing into element ${input.index}...`
|
||||
}
|
||||
if (action === 'navigate' && typeof input?.target === 'string') {
|
||||
return `Navigating to ${input.target}...`
|
||||
}
|
||||
return BROWSER_PENDING_LABELS[action] || 'Controlling browser...'
|
||||
}
|
||||
|
||||
if (result?.success === false) {
|
||||
const error = safeBrowserString(result.error)
|
||||
return error ? `Browser error: ${truncateLabel(error, 84)}` : 'Browser action failed'
|
||||
}
|
||||
|
||||
const label = getBrowserSuccessLabel(action, input, result)
|
||||
if (label) {
|
||||
return label
|
||||
}
|
||||
|
||||
return 'Controlled browser'
|
||||
}
|
||||
|
||||
// Parse attached files from message content and return clean message + file paths.
|
||||
export const parseAttachedFiles = (content: string): { message: string; files: string[] } => {
|
||||
const attachedFilesRegex = /<attached-files>\s*([\s\S]*?)\s*<\/attached-files>/
|
||||
|
|
@ -315,6 +503,7 @@ const TOOL_DISPLAY_NAMES: Record<string, string> = {
|
|||
'web-search': 'Searching the web',
|
||||
'save-to-memory': 'Saving to memory',
|
||||
'app-navigation': 'Navigating app',
|
||||
'browser-control': 'Controlling browser',
|
||||
'composio-list-toolkits': 'Listing integrations',
|
||||
'composio-search-tools': 'Searching tools',
|
||||
'composio-execute-tool': 'Running tool',
|
||||
|
|
@ -328,6 +517,8 @@ const TOOL_DISPLAY_NAMES: Record<string, string> = {
|
|||
* Falls back to the raw tool name if no mapping exists.
|
||||
*/
|
||||
export const getToolDisplayName = (tool: ToolCall): string => {
|
||||
const browserLabel = getBrowserControlLabel(tool)
|
||||
if (browserLabel) return browserLabel
|
||||
const composioData = getComposioActionCardData(tool)
|
||||
if (composioData) return composioData.label
|
||||
return TOOL_DISPLAY_NAMES[tool.name] || tool.name
|
||||
|
|
|
|||
|
|
@ -71,6 +71,7 @@ Rowboat is an agentic assistant for everyday work - emails, meetings, projects,
|
|||
**App Control:** When users ask you to open notes, show the bases or graph view, filter or search notes, or manage saved views, load the \`app-navigation\` skill first. It provides structured guidance for navigating the app UI and controlling the knowledge base view.
|
||||
|
||||
**Tracks (Auto-Updating Note Blocks):** When users ask you to **track**, **monitor**, **watch**, or **keep an eye on** something in a note — or say things like "every morning tell me X", "show the current Y in this note", "pin live updates of Z here" — load the \`tracks\` skill first. Also load it when a user presses Cmd+K with a note open and requests auto-refreshing content at the cursor. Track blocks are YAML-fenced scheduled blocks whose output is rewritten on each run — useful for weather, news, prices, status pages, and personal dashboards.
|
||||
**Browser Control:** When users ask you to open a website, browse in-app, search the web in the embedded browser, or interact with a live webpage inside Rowboat, load the \`browser-control\` skill first. It explains the \`read-page -> indexed action -> refreshed page\` workflow for the browser pane.
|
||||
|
||||
|
||||
## Learning About the User (save-to-memory)
|
||||
|
|
@ -243,6 +244,7 @@ ${runtimeContextPrompt}
|
|||
- \`slack-checkConnection\`, \`slack-listAvailableTools\`, \`slack-executeAction\` - Slack integration (requires Slack to be connected via Composio). Use \`slack-listAvailableTools\` first to discover available tool slugs, then \`slack-executeAction\` to execute them.
|
||||
- \`web-search\` - Search the web. Returns rich results with full text, highlights, and metadata. The \`category\` parameter defaults to \`general\` (full web search) — only use a specific category like \`news\`, \`company\`, \`research paper\` etc. when the query is clearly about that type. For everyday queries (weather, restaurants, prices, how-to), use \`general\`.
|
||||
- \`app-navigation\` - Control the app UI: open notes, switch views, filter/search the knowledge base, manage saved views. **Load the \`app-navigation\` skill before using this tool.**
|
||||
- \`browser-control\` - Control the embedded browser pane: open sites, inspect the live page, switch tabs, and interact with indexed page elements. **Load the \`browser-control\` skill before using this tool.**
|
||||
- \`save-to-memory\` - Save observations about the user to the agent memory system. Use this proactively during conversations.
|
||||
- \`composio-list-toolkits\`, \`composio-search-tools\`, \`composio-execute-tool\`, \`composio-connect-toolkit\` — Composio integration tools. Load the \`composio-integration\` skill for usage guidance.
|
||||
|
||||
|
|
@ -281,6 +283,13 @@ This renders as an interactive card in the UI that the user can click to open th
|
|||
- Files on the user's machine (~/Desktop/..., /Users/..., etc.)
|
||||
- Audio files, images, documents, or any file reference
|
||||
|
||||
Do NOT use filepath blocks for:
|
||||
- Website URLs or browser pages (\`https://...\`, \`http://...\`)
|
||||
- Anything currently open in the embedded browser
|
||||
- Browser tabs or browser tab ids
|
||||
|
||||
For browser pages, mention the URL in plain text or use the browser-control tool. Do not try to turn browser pages into clickable file cards.
|
||||
|
||||
**IMPORTANT:** Only use filepath blocks for files that already exist. The card is clickable and opens the file, so it must point to a real file. If you are proposing a path for a file that hasn't been created yet (e.g., "Shall I save it at ~/Documents/report.pdf?"), use inline code (\`~/Documents/report.pdf\`) instead of a filepath block. Use the filepath block only after the file has been written/created successfully.
|
||||
|
||||
Never output raw file paths in plain text when they could be wrapped in a filepath block — unless the file does not exist yet.`;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,106 @@
|
|||
export const skill = String.raw`
|
||||
# Browser Control Skill
|
||||
|
||||
You have access to the **browser-control** tool, which controls Rowboat's embedded browser pane directly.
|
||||
|
||||
Use this skill when the user asks you to open a website, browse in-app, search the web in the browser pane, click something on a page, fill a form, or otherwise interact with a live webpage inside Rowboat.
|
||||
|
||||
## Core Workflow
|
||||
|
||||
1. Start with ` + "`browser-control({ action: \"open\" })`" + ` if the browser pane may not already be open.
|
||||
2. Use ` + "`browser-control({ action: \"read-page\" })`" + ` to inspect the current page.
|
||||
3. The tool returns:
|
||||
- ` + "`snapshotId`" + `
|
||||
- page ` + "`url`" + ` and ` + "`title`" + `
|
||||
- visible page text
|
||||
- interactable elements with numbered ` + "`index`" + ` values
|
||||
4. Prefer acting on those numbered indices with ` + "`click`" + ` / ` + "`type`" + ` / ` + "`press`" + `.
|
||||
5. After each action, read the returned page snapshot before deciding the next step.
|
||||
|
||||
## Actions
|
||||
|
||||
### open
|
||||
Open the browser pane and ensure an active tab exists.
|
||||
|
||||
### get-state
|
||||
Return the current browser tabs and active tab id.
|
||||
|
||||
### new-tab
|
||||
Open a new browser tab.
|
||||
|
||||
Parameters:
|
||||
- ` + "`target`" + ` (optional): URL or plain-language search query
|
||||
|
||||
### switch-tab
|
||||
Switch to a tab by ` + "`tabId`" + `.
|
||||
|
||||
### close-tab
|
||||
Close a tab by ` + "`tabId`" + `.
|
||||
|
||||
### navigate
|
||||
Navigate the active tab.
|
||||
|
||||
Parameters:
|
||||
- ` + "`target`" + `: URL or plain-language search query
|
||||
|
||||
Plain-language targets are converted into a search automatically.
|
||||
|
||||
### back / forward / reload
|
||||
Standard browser navigation controls.
|
||||
|
||||
### read-page
|
||||
Read the current page and return a compact snapshot.
|
||||
|
||||
Parameters:
|
||||
- ` + "`maxElements`" + ` (optional)
|
||||
- ` + "`maxTextLength`" + ` (optional)
|
||||
|
||||
### click
|
||||
Click an element.
|
||||
|
||||
Prefer:
|
||||
- ` + "`index`" + `: element index from ` + "`read-page`" + `
|
||||
|
||||
Optional:
|
||||
- ` + "`snapshotId`" + `: include it when acting on a recent snapshot
|
||||
- ` + "`selector`" + `: fallback only when no usable index exists
|
||||
|
||||
### type
|
||||
Type into an input, textarea, or contenteditable element.
|
||||
|
||||
Parameters:
|
||||
- ` + "`text`" + `: text to enter
|
||||
- plus the same target fields as ` + "`click`" + `
|
||||
|
||||
### press
|
||||
Send a key press such as ` + "`Enter`" + `, ` + "`Tab`" + `, ` + "`Escape`" + `, or arrow keys.
|
||||
|
||||
Parameters:
|
||||
- ` + "`key`" + `
|
||||
- optional target fields if you need to focus a specific element first
|
||||
|
||||
### scroll
|
||||
Scroll the current page.
|
||||
|
||||
Parameters:
|
||||
- ` + "`direction`" + `: ` + "`\"up\"`" + ` or ` + "`\"down\"`" + ` (optional; defaults down)
|
||||
- ` + "`amount`" + `: pixel distance (optional)
|
||||
|
||||
### wait
|
||||
Wait for the page to settle, useful after async UI changes.
|
||||
|
||||
Parameters:
|
||||
- ` + "`ms`" + `: milliseconds to wait (optional)
|
||||
|
||||
## Important Rules
|
||||
|
||||
- Prefer ` + "`read-page`" + ` before interacting.
|
||||
- Prefer element ` + "`index`" + ` over CSS selectors.
|
||||
- If the tool says the snapshot is stale, call ` + "`read-page`" + ` again.
|
||||
- After navigation, clicking, typing, pressing, or scrolling, use the returned page snapshot instead of assuming the page state.
|
||||
- Use Rowboat's browser for live interaction. Use web search tools for research where a live session is unnecessary.
|
||||
- Do not wrap browser URLs or browser pages in ` + "```filepath" + ` blocks. Filepath cards are only for real files on disk, not web pages or browser tabs.
|
||||
- If you mention a page the browser opened, use plain text for the URL/title instead of trying to create a clickable file card.
|
||||
`;
|
||||
|
||||
export default skill;
|
||||
|
|
@ -11,6 +11,7 @@ import backgroundAgentsSkill from "./background-agents/skill.js";
|
|||
import createPresentationsSkill from "./create-presentations/skill.js";
|
||||
|
||||
import appNavigationSkill from "./app-navigation/skill.js";
|
||||
import browserControlSkill from "./browser-control/skill.js";
|
||||
import composioIntegrationSkill from "./composio-integration/skill.js";
|
||||
import tracksSkill from "./tracks/skill.js";
|
||||
|
||||
|
|
@ -105,6 +106,12 @@ const definitions: SkillDefinition[] = [
|
|||
summary: "Create and manage track blocks — YAML-scheduled auto-updating content blocks in notes (weather, news, prices, status, dashboards). Insert at cursor (Cmd+K) or append to notes.",
|
||||
content: tracksSkill,
|
||||
},
|
||||
{
|
||||
id: "browser-control",
|
||||
title: "Browser Control",
|
||||
summary: "Control the embedded browser pane - open sites, inspect page state, and interact with indexed page elements.",
|
||||
content: browserControlSkill,
|
||||
},
|
||||
];
|
||||
|
||||
const skillEntries = definitions.map((definition) => ({
|
||||
|
|
|
|||
|
|
@ -0,0 +1,8 @@
|
|||
import type { BrowserControlInput, BrowserControlResult } from '@x/shared/dist/browser-control.js';
|
||||
|
||||
export interface IBrowserControlService {
|
||||
execute(
|
||||
input: BrowserControlInput,
|
||||
ctx?: { signal?: AbortSignal },
|
||||
): Promise<BrowserControlResult>;
|
||||
}
|
||||
|
|
@ -17,6 +17,7 @@ import { WorkDir } from "../../config/config.js";
|
|||
import { composioAccountsRepo } from "../../composio/repo.js";
|
||||
import { executeAction as executeComposioAction, isConfigured as isComposioConfigured, searchTools as searchComposioTools } from "../../composio/client.js";
|
||||
import { CURATED_TOOLKITS, CURATED_TOOLKIT_SLUGS } from "@x/shared/dist/composio.js";
|
||||
import { BrowserControlInputSchema, type BrowserControlInput } from "@x/shared/dist/browser-control.js";
|
||||
import type { ToolContext } from "./exec-tool.js";
|
||||
import { generateText } from "ai";
|
||||
import { createProvider } from "../../models/models.js";
|
||||
|
|
@ -26,6 +27,7 @@ import { getGatewayProvider } from "../../models/gateway.js";
|
|||
import { getAccessToken } from "../../auth/tokens.js";
|
||||
import { API_URL } from "../../config/env.js";
|
||||
import { updateContent, updateTrackBlock } from "../../knowledge/track/fileops.js";
|
||||
import type { IBrowserControlService } from "../browser-control/service.js";
|
||||
// Parser libraries are loaded dynamically inside parseFile.execute()
|
||||
// to avoid pulling pdfjs-dist's DOM polyfills into the main bundle.
|
||||
// Import paths are computed so esbuild cannot statically resolve them.
|
||||
|
|
@ -562,7 +564,7 @@ export const BuiltinTools: z.infer<typeof BuiltinToolsSchema> = {
|
|||
count: matches.length,
|
||||
tool: 'ripgrep',
|
||||
};
|
||||
} catch (rgError) {
|
||||
} catch {
|
||||
// Fallback to basic grep if ripgrep not available or failed
|
||||
const grepArgs = [
|
||||
'-rn',
|
||||
|
|
@ -997,6 +999,39 @@ export const BuiltinTools: z.infer<typeof BuiltinToolsSchema> = {
|
|||
},
|
||||
},
|
||||
|
||||
// ============================================================================
|
||||
// Browser Control
|
||||
// ============================================================================
|
||||
|
||||
'browser-control': {
|
||||
description: 'Control the embedded browser pane. Read the current page, inspect indexed interactable elements, and navigate/click/type/press keys in the active browser tab.',
|
||||
inputSchema: BrowserControlInputSchema,
|
||||
isAvailable: async () => {
|
||||
try {
|
||||
container.resolve<IBrowserControlService>('browserControlService');
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
},
|
||||
execute: async (input: BrowserControlInput, ctx?: ToolContext) => {
|
||||
try {
|
||||
const browserControlService = container.resolve<IBrowserControlService>('browserControlService');
|
||||
return await browserControlService.execute(input, { signal: ctx?.signal });
|
||||
} catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
action: input.action,
|
||||
error: error instanceof Error ? error.message : 'Browser control is unavailable.',
|
||||
browser: {
|
||||
activeTabId: null,
|
||||
tabs: [],
|
||||
},
|
||||
};
|
||||
}
|
||||
},
|
||||
},
|
||||
|
||||
// ============================================================================
|
||||
// App Navigation
|
||||
// ============================================================================
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { asClass, createContainer, InjectionMode } from "awilix";
|
||||
import { asClass, asValue, createContainer, InjectionMode } from "awilix";
|
||||
import { FSModelConfigRepo, IModelConfigRepo } from "../models/repo.js";
|
||||
import { FSMcpConfigRepo, IMcpConfigRepo } from "../mcp/repo.js";
|
||||
import { FSAgentsRepo, IAgentsRepo } from "../agents/repo.js";
|
||||
|
|
@ -15,6 +15,7 @@ import { IAbortRegistry, InMemoryAbortRegistry } from "../runs/abort-registry.js
|
|||
import { FSAgentScheduleRepo, IAgentScheduleRepo } from "../agent-schedule/repo.js";
|
||||
import { FSAgentScheduleStateRepo, IAgentScheduleStateRepo } from "../agent-schedule/state-repo.js";
|
||||
import { FSSlackConfigRepo, ISlackConfigRepo } from "../slack/repo.js";
|
||||
import type { IBrowserControlService } from "../application/browser-control/service.js";
|
||||
|
||||
const container = createContainer({
|
||||
injectionMode: InjectionMode.PROXY,
|
||||
|
|
@ -41,4 +42,10 @@ container.register({
|
|||
slackConfigRepo: asClass<ISlackConfigRepo>(FSSlackConfigRepo).singleton(),
|
||||
});
|
||||
|
||||
export default container;
|
||||
export default container;
|
||||
|
||||
export function registerBrowserControlService(service: IBrowserControlService): void {
|
||||
container.register({
|
||||
browserControlService: asValue(service),
|
||||
});
|
||||
}
|
||||
|
|
|
|||
134
apps/x/packages/shared/src/browser-control.ts
Normal file
134
apps/x/packages/shared/src/browser-control.ts
Normal file
|
|
@ -0,0 +1,134 @@
|
|||
import { z } from 'zod';
|
||||
|
||||
export const BrowserTabStateSchema = z.object({
|
||||
id: z.string(),
|
||||
url: z.string(),
|
||||
title: z.string(),
|
||||
canGoBack: z.boolean(),
|
||||
canGoForward: z.boolean(),
|
||||
loading: z.boolean(),
|
||||
});
|
||||
|
||||
export const BrowserStateSchema = z.object({
|
||||
activeTabId: z.string().nullable(),
|
||||
tabs: z.array(BrowserTabStateSchema),
|
||||
});
|
||||
|
||||
export const BrowserPageElementSchema = z.object({
|
||||
index: z.number().int().positive(),
|
||||
tagName: z.string(),
|
||||
role: z.string().nullable(),
|
||||
type: z.string().nullable(),
|
||||
label: z.string().nullable(),
|
||||
text: z.string().nullable(),
|
||||
placeholder: z.string().nullable(),
|
||||
href: z.string().nullable(),
|
||||
disabled: z.boolean(),
|
||||
});
|
||||
|
||||
export const BrowserPageSnapshotSchema = z.object({
|
||||
snapshotId: z.string(),
|
||||
url: z.string(),
|
||||
title: z.string(),
|
||||
loading: z.boolean(),
|
||||
text: z.string(),
|
||||
elements: z.array(BrowserPageElementSchema),
|
||||
});
|
||||
|
||||
export const BrowserControlActionSchema = z.enum([
|
||||
'open',
|
||||
'get-state',
|
||||
'new-tab',
|
||||
'switch-tab',
|
||||
'close-tab',
|
||||
'navigate',
|
||||
'back',
|
||||
'forward',
|
||||
'reload',
|
||||
'read-page',
|
||||
'click',
|
||||
'type',
|
||||
'press',
|
||||
'scroll',
|
||||
'wait',
|
||||
]);
|
||||
|
||||
const BrowserElementTargetFields = {
|
||||
index: z.number().int().positive().optional(),
|
||||
selector: z.string().min(1).optional(),
|
||||
snapshotId: z.string().optional(),
|
||||
} as const;
|
||||
|
||||
export const BrowserControlInputSchema = z.object({
|
||||
action: BrowserControlActionSchema,
|
||||
target: z.string().min(1).optional(),
|
||||
tabId: z.string().min(1).optional(),
|
||||
text: z.string().optional(),
|
||||
key: z.string().min(1).optional(),
|
||||
direction: z.enum(['up', 'down']).optional(),
|
||||
amount: z.number().int().positive().max(5000).optional(),
|
||||
ms: z.number().int().positive().max(30000).optional(),
|
||||
maxElements: z.number().int().positive().max(100).optional(),
|
||||
maxTextLength: z.number().int().positive().max(20000).optional(),
|
||||
...BrowserElementTargetFields,
|
||||
}).strict().superRefine((value, ctx) => {
|
||||
const needsElementTarget = value.action === 'click' || value.action === 'type';
|
||||
const hasElementTarget = value.index !== undefined || value.selector !== undefined;
|
||||
|
||||
if ((value.action === 'switch-tab' || value.action === 'close-tab') && !value.tabId) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
path: ['tabId'],
|
||||
message: 'tabId is required for this action.',
|
||||
});
|
||||
}
|
||||
|
||||
if ((value.action === 'navigate') && !value.target) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
path: ['target'],
|
||||
message: 'target is required for navigate.',
|
||||
});
|
||||
}
|
||||
|
||||
if (value.action === 'type' && value.text === undefined) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
path: ['text'],
|
||||
message: 'text is required for type.',
|
||||
});
|
||||
}
|
||||
|
||||
if (value.action === 'press' && !value.key) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
path: ['key'],
|
||||
message: 'key is required for press.',
|
||||
});
|
||||
}
|
||||
|
||||
if (needsElementTarget && !hasElementTarget) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
path: ['index'],
|
||||
message: 'Provide an element index or selector.',
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
export const BrowserControlResultSchema = z.object({
|
||||
success: z.boolean(),
|
||||
action: BrowserControlActionSchema,
|
||||
message: z.string().optional(),
|
||||
error: z.string().optional(),
|
||||
browser: BrowserStateSchema,
|
||||
page: BrowserPageSnapshotSchema.optional(),
|
||||
});
|
||||
|
||||
export type BrowserTabState = z.infer<typeof BrowserTabStateSchema>;
|
||||
export type BrowserState = z.infer<typeof BrowserStateSchema>;
|
||||
export type BrowserPageElement = z.infer<typeof BrowserPageElementSchema>;
|
||||
export type BrowserPageSnapshot = z.infer<typeof BrowserPageSnapshotSchema>;
|
||||
export type BrowserControlAction = z.infer<typeof BrowserControlActionSchema>;
|
||||
export type BrowserControlInput = z.infer<typeof BrowserControlInputSchema>;
|
||||
export type BrowserControlResult = z.infer<typeof BrowserControlResultSchema>;
|
||||
|
|
@ -12,4 +12,5 @@ export * as blocks from './blocks.js';
|
|||
export * as trackBlock from './track-block.js';
|
||||
export * as frontmatter from './frontmatter.js';
|
||||
export * as bases from './bases.js';
|
||||
export * as browserControl from './browser-control.js';
|
||||
export { PrefixLogger };
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import { TrackEvent } from './track-block.js';
|
|||
import { UserMessageContent } from './message.js';
|
||||
import { RowboatApiConfig } from './rowboat-account.js';
|
||||
import { ZListToolkitsResponse } from './composio.js';
|
||||
import { BrowserStateSchema } from './browser-control.js';
|
||||
|
||||
// ============================================================================
|
||||
// Runtime Validation Schemas (Single Source of Truth)
|
||||
|
|
@ -626,6 +627,87 @@ const ipcSchemas = {
|
|||
error: z.string().optional(),
|
||||
}),
|
||||
},
|
||||
// Embedded browser (WebContentsView) channels
|
||||
'browser:setBounds': {
|
||||
req: z.object({
|
||||
x: z.number().int(),
|
||||
y: z.number().int(),
|
||||
width: z.number().int().nonnegative(),
|
||||
height: z.number().int().nonnegative(),
|
||||
}),
|
||||
res: z.object({ ok: z.literal(true) }),
|
||||
},
|
||||
'browser:setVisible': {
|
||||
req: z.object({ visible: z.boolean() }),
|
||||
res: z.object({ ok: z.literal(true) }),
|
||||
},
|
||||
'browser:newTab': {
|
||||
req: z.object({
|
||||
url: z.string().min(1).refine(
|
||||
(u) => {
|
||||
const lower = u.trim().toLowerCase();
|
||||
if (lower.startsWith('javascript:')) return false;
|
||||
if (lower.startsWith('file://')) return false;
|
||||
if (lower.startsWith('chrome://')) return false;
|
||||
if (lower.startsWith('chrome-extension://')) return false;
|
||||
return true;
|
||||
},
|
||||
{ message: 'Unsafe URL scheme' },
|
||||
).optional(),
|
||||
}),
|
||||
res: z.object({
|
||||
ok: z.boolean(),
|
||||
tabId: z.string().optional(),
|
||||
error: z.string().optional(),
|
||||
}),
|
||||
},
|
||||
'browser:switchTab': {
|
||||
req: z.object({ tabId: z.string().min(1) }),
|
||||
res: z.object({ ok: z.boolean() }),
|
||||
},
|
||||
'browser:closeTab': {
|
||||
req: z.object({ tabId: z.string().min(1) }),
|
||||
res: z.object({ ok: z.boolean() }),
|
||||
},
|
||||
'browser:navigate': {
|
||||
req: z.object({
|
||||
url: z.string().min(1).refine(
|
||||
(u) => {
|
||||
const lower = u.trim().toLowerCase();
|
||||
if (lower.startsWith('javascript:')) return false;
|
||||
if (lower.startsWith('file://')) return false;
|
||||
if (lower.startsWith('chrome://')) return false;
|
||||
if (lower.startsWith('chrome-extension://')) return false;
|
||||
return true;
|
||||
},
|
||||
{ message: 'Unsafe URL scheme' },
|
||||
),
|
||||
}),
|
||||
res: z.object({
|
||||
ok: z.boolean(),
|
||||
error: z.string().optional(),
|
||||
}),
|
||||
},
|
||||
'browser:back': {
|
||||
req: z.null(),
|
||||
res: z.object({ ok: z.boolean() }),
|
||||
},
|
||||
'browser:forward': {
|
||||
req: z.null(),
|
||||
res: z.object({ ok: z.boolean() }),
|
||||
},
|
||||
'browser:reload': {
|
||||
req: z.null(),
|
||||
res: z.object({ ok: z.literal(true) }),
|
||||
},
|
||||
'browser:getState': {
|
||||
req: z.null(),
|
||||
res: BrowserStateSchema,
|
||||
},
|
||||
'browser:didUpdateState': {
|
||||
req: BrowserStateSchema,
|
||||
res: z.null(),
|
||||
},
|
||||
// Billing channels
|
||||
'billing:getInfo': {
|
||||
req: z.null(),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue