apunkt/PriceGhost
1
0
Fork 0
mirror of https://github.com/clucraft/PriceGhost.git synced 2026-07-14 16:32:14 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: try HTTP before browser for JS-heavy retail sites (Target, Walmart, Best Buy, Costco)

Browse source 
The scraper was going straight to Puppeteer for JS-heavy sites, but
Chromium fails to launch in the container (crashpad issue). These sites
actually serve product data via JSON-LD in the initial HTML, so plain
HTTP requests work fine.

Changes:
- Try axios HTTP fetch FIRST for ALL sites (including JS-heavy)
- Fall back to Puppeteer only on 403 or empty response
- Add size check: if HTML < 50KB and site is JS-heavy, try browser
This commit is contained in:
Tony Fruzza 2026-06-15 07:28:34 +00:00
parent a3ad25a4ae
commit 99a133453f
1 changed files with 30 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

42
backend/src/services/scraper.ts
View file

@ -1382,15 +1382,9 @@ export async function scrapeProductWithVoting(
try {
let usedBrowser = false;
// For JS-heavy sites, go straight to browser
if (requiresBrowser) {
console.log(`[Voting] ${new URL(url).hostname} requires browser rendering, using Puppeteer...`);
html = await scrapeWithBrowser(url);
usedBrowser = true;
} else {
// Fetch HTML
try {
const response = await axios.get<string>(url, {
// Try plain HTTP first for all sites (includes JSON-LD data even for JS-heavy sites)
try {
const response = await axios.get<string>(url, {
headers: {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
@ -1403,17 +1397,41 @@ export async function scrapeProductWithVoting(
maxRedirects: 5,
});
html = response.data;
} catch (axiosError) {
if (axiosError instanceof AxiosError && axiosError.response?.status === 403) {
console.log(`[Voting] HTTP blocked (403) for ${url}, using browser...`);
console.log(`[Voting] HTTP fetch OK: ${url} (${html.length} bytes)`);
} catch (axiosError) {
if (axiosError instanceof AxiosError && axiosError.response?.status === 403) {
console.log(`[Voting] HTTP blocked (403) for ${url}`);
if (requiresBrowser) {
console.log(`[Voting] Falling back to browser rendering...`);
html = await scrapeWithBrowser(url);
usedBrowser = true;
}
} else {
if (requiresBrowser) {
console.log(`[Voting] HTTP error for JS-heavy site, trying browser...`);
try {
html = await scrapeWithBrowser(url);
usedBrowser = true;
} catch (browserError) {
throw axiosError; // Throw the original error
}
} else {
throw axiosError;
}
}
}
// If we got HTML but it's suspiciously small (e.g., blocking page) and browser is available, try browser
if (!usedBrowser && requiresBrowser && html.length < 50000) {
console.log(`[Voting] HTTP response too small (${html.length}b), content may be blocked. Trying browser...`);
try {
html = await scrapeWithBrowser(url);
usedBrowser = true;
} catch (browserError) {
console.log(`[Voting] Browser also failed, keeping HTTP result.`);
}
}
let $ = load(html);
// Collect candidates from all methods