From 7943ec964f152383091249744e11d31b3a929780 Mon Sep 17 00:00:00 2001
From: Tony Fruzza <tony240zt@gmail.com>
Date: Mon, 15 Jun 2026 07:36:16 +0000
Subject: [PATCH] fix: connect to remote browser-node for JS-heavy retail sites
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Replaces the broken local Chromium launch (crashpad handler crash in k8s)
with connection to the shared browser-node Chromium via CDP.

Changes:
- Add getRemoteBrowserWSEndpoint() — fetches WebSocket URL from
  browser-node's /json/version endpoint, remaps localhost → k8s service
- scrapeWithBrowser tries remote browser first, falls back to local launch
- Use browser.disconnect() for remote, browser.close() for local
- REMOTE_BROWSER_URL env var overrides the default k8s service URL
---
 backend/src/services/scraper.ts | 85 ++++++++++++++++++++++++++-------
 1 file changed, 69 insertions(+), 16 deletions(-)

diff --git a/backend/src/services/scraper.ts b/backend/src/services/scraper.ts
index 60288ae..e766695 100644
--- a/backend/src/services/scraper.ts
+++ b/backend/src/services/scraper.ts
@@ -247,22 +247,71 @@ function extractGenericCssCandidates($: CheerioAPI): PriceCandidate[] {
 }
 
 // Browser-based scraping for sites that block HTTP requests (e.g., Cloudflare)
+/**
+ * Get the remote browser WebSocket endpoint from browser-node service
+ */
+async function getRemoteBrowserWSEndpoint(): Promise<string | null> {
+  const remoteUrl = process.env.REMOTE_BROWSER_URL || 'http://browser-node.openclaw.svc.cluster.local:9222';
+  try {
+    const response = await axios.get<string>(`${remoteUrl}/json/version`, { timeout: 5000 });
+    const data = JSON.parse(response.data);
+    const wsUrl = data.webSocketDebuggerUrl;
+    if (wsUrl) {
+      // Replace localhost with the actual remote host
+      const remoteHost = new URL(remoteUrl).hostname;
+      const remotePort = new URL(remoteUrl).port || '9222';
+      return wsUrl.replace(/ws:\/\/localhost(:\d+)?/, `ws://${remoteHost}:${remotePort}`);
+    }
+  } catch (e) {
+    console.log(`[Browser] Remote browser not available at ${remoteUrl}: ${(e as Error).message}`);
+  }
+  return null;
+}
+
 async function scrapeWithBrowser(url: string): Promise<string> {
-  const browser = await puppeteer.launch({
-    headless: true,
-    args: [
-      '--no-sandbox',
-      '--disable-setuid-sandbox',
-      '--disable-dev-shm-usage',
-      '--disable-blink-features=AutomationControlled',
-      '--disable-infobars',
-      '--disable-crash-reporter',
-      '--window-size=1920,1080',
-      '--start-maximized',
-    ],
-    executablePath: process.env.PUPPETEER_EXECUTABLE_PATH || undefined,
-    ignoreDefaultArgs: ['--enable-automation'],
-  });
+  let browser;
+  let connected = false;
+
+  // Try remote browser first
+  const wsEndpoint = await getRemoteBrowserWSEndpoint();
+  if (wsEndpoint) {
+    try {
+      browser = await puppeteer.connect({
+        browserWSEndpoint: wsEndpoint,
+        defaultViewport: { width: 1920, height: 1080 },
+      });
+      connected = true;
+      console.log(`[Browser] Connected to remote browser at ${wsEndpoint}`);
+    } catch (e) {
+      console.log(`[Browser] Remote connection failed: ${(e as Error).message}`);
+    }
+  }
+
+  if (!connected) {
+    try {
+      browser = await puppeteer.launch({
+        headless: true,
+        args: [
+          '--no-sandbox',
+          '--disable-setuid-sandbox',
+          '--disable-dev-shm-usage',
+          '--disable-blink-features=AutomationControlled',
+          '--disable-infobars',
+          '--disable-crash-reporter',
+          '--window-size=1920,1080',
+          '--start-maximized',
+        ],
+        executablePath: process.env.PUPPETEER_EXECUTABLE_PATH || undefined,
+        ignoreDefaultArgs: ['--enable-automation'],
+      });
+    } catch (e) {
+      throw new Error(`No browser available: remote unavailable and local launch failed: ${(e as Error).message}`);
+    }
+  }
+
+  if (!browser) {
+    throw new Error('Failed to acquire browser instance');
+  }
 
   try {
     const page = await browser.newPage();
@@ -311,7 +360,11 @@ async function scrapeWithBrowser(url: string): Promise<string> {
     const html = await page.content();
     return html;
   } finally {
-    await browser.close();
+    if (connected) {
+      await browser.disconnect();
+    } else {
+      await browser.close();
+    }
   }
 }