apunkt/PriceGhost
1
0
Fork 0
mirror of https://github.com/clucraft/PriceGhost.git synced 2026-04-25 00:36:32 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix Best Buy price extraction and add browser rendering for JS-heavy sites

Browse source 
- Fix TypeScript narrowing issue in Best Buy scraper
- Add browser rendering for JS-heavy sites (Best Buy, Target, Walmart, Costco)
- Improve Best Buy site-specific scraper with better selectors and logging
- Skip payment plan prices (/mo, per month, etc.)
- Enhance AI HTML preparation:
  - Extract JSON-LD data before removing scripts
  - Add price element extraction for better context
  - Increase character limit from 15000 to 25000

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
clucraft 2026-01-24 15:09:42 -05:00
parent cf23ac9db1
commit 196ff8fd0b
2 changed files with 54 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

46
backend/src/services/ai-extractor.ts
View file

@ -87,6 +87,7 @@ function prepareHtmlForAI(html: string): string {
if (scriptContent) {
// Include any JSON-LD that might be product-related
if (scriptContent.includes('price') ||
scriptContent.includes('Price') ||
scriptContent.includes('Product') ||
scriptContent.includes('Offer')) {
jsonLdScripts.push(scriptContent);
@ -94,6 +95,26 @@ function prepareHtmlForAI(html: string): string {
}
});
// Extract price-related elements specifically
const priceElements: string[] = [];
const priceSelectors = [
'[class*="price"]',
'[class*="Price"]',
'[data-testid*="price"]',
'[itemprop="price"]',
'[data-price]',
];
for (const selector of priceSelectors) {
$(selector).each((_, el) => {
const text = $(el).text().trim();
const parent = $(el).parent().text().trim().slice(0, 200);
if (text && text.match(/\$[\d,]+\.?\d*/)) {
priceElements.push(`Price element: "${text}" (context: "${parent.slice(0, 100)}")`);
}
});
}
// Now remove script, style, and other non-content elements
$('script, style, noscript, iframe, svg, path, meta, link, comment').remove();
@ -103,9 +124,10 @@ function prepareHtmlForAI(html: string): string {
// Try to focus on product-related sections if possible
const productSelectors = [
'[itemtype*="Product"]',
'[class*="product"]',
'[class*="product-detail"]',
'[class*="productDetail"]',
'[class*="pdp-"]',
'[id*="product"]',
'[class*="pdp"]',
'main',
'[role="main"]',
];
@ -118,16 +140,24 @@ function prepareHtmlForAI(html: string): string {
}
}
// Combine JSON-LD data with HTML content
let finalContent = content;
// Build final content with all price-related info at the top
let finalContent = '';
if (jsonLdScripts.length > 0) {
finalContent = `JSON-LD Structured Data:\n${jsonLdScripts.join('\n')}\n\nHTML Content:\n${content}`;
finalContent += `=== JSON-LD Structured Data (MOST RELIABLE) ===\n${jsonLdScripts.join('\n')}\n\n`;
console.log(`[AI] Found ${jsonLdScripts.length} JSON-LD scripts with product data`);
}
// Truncate to ~15000 characters to stay within token limits
if (finalContent.length > 15000) {
finalContent = finalContent.substring(0, 15000) + '\n... [truncated]';
if (priceElements.length > 0) {
finalContent += `=== Price Elements Found ===\n${priceElements.slice(0, 10).join('\n')}\n\n`;
console.log(`[AI] Found ${priceElements.length} price elements`);
}
finalContent += `=== HTML Content ===\n${content}`;
// Truncate to ~25000 characters to stay within token limits but capture more content
if (finalContent.length > 25000) {
finalContent = finalContent.substring(0, 25000) + '\n... [truncated]';
}
console.log(`[AI] Prepared HTML content: ${finalContent.length} characters`);

17
backend/src/services/scraper.ts
View file

@ -591,25 +591,35 @@ const siteScrapers: SiteScraper[] = [
'.priceView-customer-price span',
'.priceView-hero-price span',
'[class*="customerPrice"]',
'[class*="priceView"] span[aria-hidden="true"]',
'.pricing-price__regular-price',
'[data-testid="product-price"]',
'.price-box span',
];
let price: ParsedPrice | null = null;
for (const selector of priceSelectors) {
const elements = $(selector);
console.log(`[BestBuy] Selector "${selector}" found ${elements.length} elements`);
// Check each element, skip payment plan prices (contain "/mo", "per month", etc.)
elements.each((_, el) => {
if (price) return false; // Already found a valid price
const text = $(el).text().trim();
if (!text) return true;
console.log(`[BestBuy] Found text: "${text.slice(0, 50)}"`);
const lowerText = text.toLowerCase();
// Skip if it looks like a monthly payment plan
if (lowerText.includes('/mo') ||
lowerText.includes('per month') ||
lowerText.includes('monthly') ||
lowerText.includes('financing')) {
lowerText.includes('financing') ||
lowerText.includes('payment')) {
console.log(`[BestBuy] Skipping payment plan price: "${text.slice(0, 30)}"`);
return true; // Continue to next element
}
const parsed = parsePrice(text);
if (parsed) {
console.log(`[BestBuy] Parsed price: ${parsed.price} ${parsed.currency}`);
price = parsed;
return false; // Break the loop
}
@ -619,12 +629,17 @@ const siteScrapers: SiteScraper[] = [
const name = $('h1.heading-5').text().trim() ||
$('.sku-title h1').text().trim() ||
$('[data-testid="product-title"]').text().trim() ||
$('h1').first().text().trim() ||
null;
console.log(`[BestBuy] Found name: "${name?.slice(0, 50)}"`);
const imageUrl = $('img.primary-image').attr('src') ||
$('[data-testid="image-gallery-image"]').attr('src') ||
$('img[class*="product-image"]').attr('src') ||
null;
console.log(`[BestBuy] Final result - name: ${!!name}, price: ${price ? (price as ParsedPrice).price : null}, image: ${!!imageUrl}`);
return { name, price, imageUrl };
},
},