From c9a9e390242dcf0ca03e02a9a706a5e78aaffab6 Mon Sep 17 00:00:00 2001
From: Tony Fruzza <tony240zt@gmail.com>
Date: Mon, 15 Jun 2026 06:43:56 +0000
Subject: [PATCH] fix: remove bot-triggering headers from Amazon HTTP requests

The Sec-* and Cache-Control headers in the axios request config trigger
Amazon's bot detection, returning a 5KB blocking page instead of the
full product page. This breaks ALL Amazon scraping, not just a.co links.

Removed headers: Cache-Control, Pragma, Sec-Ch-Ua, Sec-Ch-Ua-Mobile,
Sec-Ch-Ua-Platform, Sec-Fetch-Dest, Sec-Fetch-Mode, Sec-Fetch-Site,
Sec-Fetch-User, Upgrade-Insecure-Requests
---
 backend/src/services/scraper.ts | 20 --------------------
 1 file changed, 20 deletions(-)

diff --git a/backend/src/services/scraper.ts b/backend/src/services/scraper.ts
index 1c829e6..475f189 100644
--- a/backend/src/services/scraper.ts
+++ b/backend/src/services/scraper.ts
@@ -1147,16 +1147,6 @@ export async function scrapeProduct(url: string, userId?: number): Promise<Scrap
             'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
           'Accept-Language': 'en-US,en;q=0.9',
           'Accept-Encoding': 'gzip, deflate, br',
-          'Cache-Control': 'no-cache',
-          Pragma: 'no-cache',
-          'Sec-Ch-Ua': '"Not A(Brand";v="99", "Google Chrome";v="121", "Chromium";v="121"',
-          'Sec-Ch-Ua-Mobile': '?0',
-          'Sec-Ch-Ua-Platform': '"Windows"',
-          'Sec-Fetch-Dest': 'document',
-          'Sec-Fetch-Mode': 'navigate',
-          'Sec-Fetch-Site': 'none',
-          'Sec-Fetch-User': '?1',
-          'Upgrade-Insecure-Requests': '1',
         },
         timeout: 20000,
         maxRedirects: 5,
@@ -1408,16 +1398,6 @@ export async function scrapeProductWithVoting(
             'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
           'Accept-Language': 'en-US,en;q=0.9',
           'Accept-Encoding': 'gzip, deflate, br',
-          'Cache-Control': 'no-cache',
-          Pragma: 'no-cache',
-          'Sec-Ch-Ua': '"Not A(Brand";v="99", "Google Chrome";v="121", "Chromium";v="121"',
-          'Sec-Ch-Ua-Mobile': '?0',
-          'Sec-Ch-Ua-Platform': '"Windows"',
-          'Sec-Fetch-Dest': 'document',
-          'Sec-Fetch-Mode': 'navigate',
-          'Sec-Fetch-Site': 'none',
-          'Sec-Fetch-User': '?1',
-          'Upgrade-Insecure-Requests': '1',
         },
         timeout: 20000,
         maxRedirects: 5,