Browser2 (#495)

Add tabbed embedded browser and assistant browser control

apps/x/packages/shared/src/browser-control.ts

@@ -0,0 +1,134 @@
+import { z } from 'zod';
+
+export const BrowserTabStateSchema = z.object({
+  id: z.string(),
+  url: z.string(),
+  title: z.string(),
+  canGoBack: z.boolean(),
+  canGoForward: z.boolean(),
+  loading: z.boolean(),
+});
+
+export const BrowserStateSchema = z.object({
+  activeTabId: z.string().nullable(),
+  tabs: z.array(BrowserTabStateSchema),
+});
+
+export const BrowserPageElementSchema = z.object({
+  index: z.number().int().positive(),
+  tagName: z.string(),
+  role: z.string().nullable(),
+  type: z.string().nullable(),
+  label: z.string().nullable(),
+  text: z.string().nullable(),
+  placeholder: z.string().nullable(),
+  href: z.string().nullable(),
+  disabled: z.boolean(),
+});
+
+export const BrowserPageSnapshotSchema = z.object({
+  snapshotId: z.string(),
+  url: z.string(),
+  title: z.string(),
+  loading: z.boolean(),
+  text: z.string(),
+  elements: z.array(BrowserPageElementSchema),
+});
+
+export const BrowserControlActionSchema = z.enum([
+  'open',
+  'get-state',
+  'new-tab',
+  'switch-tab',
+  'close-tab',
+  'navigate',
+  'back',
+  'forward',
+  'reload',
+  'read-page',
+  'click',
+  'type',
+  'press',
+  'scroll',
+  'wait',
+]);
+
+const BrowserElementTargetFields = {
+  index: z.number().int().positive().optional(),
+  selector: z.string().min(1).optional(),
+  snapshotId: z.string().optional(),
+} as const;
+
+export const BrowserControlInputSchema = z.object({
+  action: BrowserControlActionSchema,
+  target: z.string().min(1).optional(),
+  tabId: z.string().min(1).optional(),
+  text: z.string().optional(),
+  key: z.string().min(1).optional(),
+  direction: z.enum(['up', 'down']).optional(),
+  amount: z.number().int().positive().max(5000).optional(),
+  ms: z.number().int().positive().max(30000).optional(),
+  maxElements: z.number().int().positive().max(100).optional(),
+  maxTextLength: z.number().int().positive().max(20000).optional(),
+  ...BrowserElementTargetFields,
+}).strict().superRefine((value, ctx) => {
+  const needsElementTarget = value.action === 'click' || value.action === 'type';
+  const hasElementTarget = value.index !== undefined || value.selector !== undefined;
+
+  if ((value.action === 'switch-tab' || value.action === 'close-tab') && !value.tabId) {
+    ctx.addIssue({
+      code: z.ZodIssueCode.custom,
+      path: ['tabId'],
+      message: 'tabId is required for this action.',
+    });
+  }
+
+  if ((value.action === 'navigate') && !value.target) {
+    ctx.addIssue({
+      code: z.ZodIssueCode.custom,
+      path: ['target'],
+      message: 'target is required for navigate.',
+    });
+  }
+
+  if (value.action === 'type' && value.text === undefined) {
+    ctx.addIssue({
+      code: z.ZodIssueCode.custom,
+      path: ['text'],
+      message: 'text is required for type.',
+    });
+  }
+
+  if (value.action === 'press' && !value.key) {
+    ctx.addIssue({
+      code: z.ZodIssueCode.custom,
+      path: ['key'],
+      message: 'key is required for press.',
+    });
+  }
+
+  if (needsElementTarget && !hasElementTarget) {
+    ctx.addIssue({
+      code: z.ZodIssueCode.custom,
+      path: ['index'],
+      message: 'Provide an element index or selector.',
+    });
+  }
+});
+
+export const BrowserControlResultSchema = z.object({
+  success: z.boolean(),
+  action: BrowserControlActionSchema,
+  message: z.string().optional(),
+  error: z.string().optional(),
+  browser: BrowserStateSchema,
+  page: BrowserPageSnapshotSchema.optional(),
+});
+
+export type BrowserTabState = z.infer<typeof BrowserTabStateSchema>;
+export type BrowserState = z.infer<typeof BrowserStateSchema>;
+export type BrowserPageElement = z.infer<typeof BrowserPageElementSchema>;
+export type BrowserPageSnapshot = z.infer<typeof BrowserPageSnapshotSchema>;
+export type BrowserControlAction = z.infer<typeof BrowserControlActionSchema>;
+export type BrowserControlInput = z.infer<typeof BrowserControlInputSchema>;
+export type BrowserControlResult = z.infer<typeof BrowserControlResultSchema>;

apps/x/packages/shared/src/index.ts

@@ -12,4 +12,5 @@ export * as blocks from './blocks.js';
 export * as trackBlock from './track-block.js';
 export * as frontmatter from './frontmatter.js';
 export * as bases from './bases.js';
+export * as browserControl from './browser-control.js';
 export { PrefixLogger };

apps/x/packages/shared/src/ipc.ts

@@ -10,6 +10,7 @@ import { TrackEvent } from './track-block.js';
 import { UserMessageContent } from './message.js';
 import { RowboatApiConfig } from './rowboat-account.js';
 import { ZListToolkitsResponse } from './composio.js';
+import { BrowserStateSchema } from './browser-control.js';
 
 // ============================================================================
 // Runtime Validation Schemas (Single Source of Truth)
@@ -626,6 +627,87 @@ const ipcSchemas = {
       error: z.string().optional(),
     }),
   },
+  // Embedded browser (WebContentsView) channels
+  'browser:setBounds': {
+    req: z.object({
+      x: z.number().int(),
+      y: z.number().int(),
+      width: z.number().int().nonnegative(),
+      height: z.number().int().nonnegative(),
+    }),
+    res: z.object({ ok: z.literal(true) }),
+  },
+  'browser:setVisible': {
+    req: z.object({ visible: z.boolean() }),
+    res: z.object({ ok: z.literal(true) }),
+  },
+  'browser:newTab': {
+    req: z.object({
+      url: z.string().min(1).refine(
+        (u) => {
+          const lower = u.trim().toLowerCase();
+          if (lower.startsWith('javascript:')) return false;
+          if (lower.startsWith('file://')) return false;
+          if (lower.startsWith('chrome://')) return false;
+          if (lower.startsWith('chrome-extension://')) return false;
+          return true;
+        },
+        { message: 'Unsafe URL scheme' },
+      ).optional(),
+    }),
+    res: z.object({
+      ok: z.boolean(),
+      tabId: z.string().optional(),
+      error: z.string().optional(),
+    }),
+  },
+  'browser:switchTab': {
+    req: z.object({ tabId: z.string().min(1) }),
+    res: z.object({ ok: z.boolean() }),
+  },
+  'browser:closeTab': {
+    req: z.object({ tabId: z.string().min(1) }),
+    res: z.object({ ok: z.boolean() }),
+  },
+  'browser:navigate': {
+    req: z.object({
+      url: z.string().min(1).refine(
+        (u) => {
+          const lower = u.trim().toLowerCase();
+          if (lower.startsWith('javascript:')) return false;
+          if (lower.startsWith('file://')) return false;
+          if (lower.startsWith('chrome://')) return false;
+          if (lower.startsWith('chrome-extension://')) return false;
+          return true;
+        },
+        { message: 'Unsafe URL scheme' },
+      ),
+    }),
+    res: z.object({
+      ok: z.boolean(),
+      error: z.string().optional(),
+    }),
+  },
+  'browser:back': {
+    req: z.null(),
+    res: z.object({ ok: z.boolean() }),
+  },
+  'browser:forward': {
+    req: z.null(),
+    res: z.object({ ok: z.boolean() }),
+  },
+  'browser:reload': {
+    req: z.null(),
+    res: z.object({ ok: z.literal(true) }),
+  },
+  'browser:getState': {
+    req: z.null(),
+    res: BrowserStateSchema,
+  },
+  'browser:didUpdateState': {
+    req: BrowserStateSchema,
+    res: z.null(),
+  },
   // Billing channels
   'billing:getInfo': {
     req: z.null(),