From 99a133453f2e201b1798c3b968a3720ec2831239 Mon Sep 17 00:00:00 2001
From: Tony Fruzza <tony240zt@gmail.com>
Date: Mon, 15 Jun 2026 07:28:34 +0000
Subject: [PATCH] fix: try HTTP before browser for JS-heavy retail sites
 (Target, Walmart, Best Buy, Costco)

The scraper was going straight to Puppeteer for JS-heavy sites, but
Chromium fails to launch in the container (crashpad issue). These sites
actually serve product data via JSON-LD in the initial HTML, so plain
HTTP requests work fine.

Changes:
- Try axios HTTP fetch FIRST for ALL sites (including JS-heavy)
- Fall back to Puppeteer only on 403 or empty response
- Add size check: if HTML < 50KB and site is JS-heavy, try browser
---
 backend/src/services/scraper.ts | 42 +++++++++++++++++++++++----------
 1 file changed, 30 insertions(+), 12 deletions(-)

diff --git a/backend/src/services/scraper.ts b/backend/src/services/scraper.ts
index 475f189..60288ae 100644
--- a/backend/src/services/scraper.ts
+++ b/backend/src/services/scraper.ts
@@ -1382,15 +1382,9 @@ export async function scrapeProductWithVoting(
   try {
     let usedBrowser = false;
 
-    // For JS-heavy sites, go straight to browser
-    if (requiresBrowser) {
-      console.log(`[Voting] ${new URL(url).hostname} requires browser rendering, using Puppeteer...`);
-      html = await scrapeWithBrowser(url);
-      usedBrowser = true;
-    } else {
-      // Fetch HTML
-      try {
-        const response = await axios.get<string>(url, {
+    // Try plain HTTP first for all sites (includes JSON-LD data even for JS-heavy sites)
+    try {
+      const response = await axios.get<string>(url, {
         headers: {
           'User-Agent':
             'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
@@ -1403,17 +1397,41 @@ export async function scrapeProductWithVoting(
         maxRedirects: 5,
       });
       html = response.data;
-      } catch (axiosError) {
-        if (axiosError instanceof AxiosError && axiosError.response?.status === 403) {
-          console.log(`[Voting] HTTP blocked (403) for ${url}, using browser...`);
+      console.log(`[Voting] HTTP fetch OK: ${url} (${html.length} bytes)`);
+    } catch (axiosError) {
+      if (axiosError instanceof AxiosError && axiosError.response?.status === 403) {
+        console.log(`[Voting] HTTP blocked (403) for ${url}`);
+        if (requiresBrowser) {
+          console.log(`[Voting] Falling back to browser rendering...`);
           html = await scrapeWithBrowser(url);
           usedBrowser = true;
+        }
+      } else {
+        if (requiresBrowser) {
+          console.log(`[Voting] HTTP error for JS-heavy site, trying browser...`);
+          try {
+            html = await scrapeWithBrowser(url);
+            usedBrowser = true;
+          } catch (browserError) {
+            throw axiosError; // Throw the original error
+          }
         } else {
           throw axiosError;
         }
       }
     }
 
+    // If we got HTML but it's suspiciously small (e.g., blocking page) and browser is available, try browser
+    if (!usedBrowser && requiresBrowser && html.length < 50000) {
+      console.log(`[Voting] HTTP response too small (${html.length}b), content may be blocked. Trying browser...`);
+      try {
+        html = await scrapeWithBrowser(url);
+        usedBrowser = true;
+      } catch (browserError) {
+        console.log(`[Voting] Browser also failed, keeping HTTP result.`);
+      }
+    }
+
     let $ = load(html);
 
     // Collect candidates from all methods