diff --git a/backend/src/models/index.ts b/backend/src/models/index.ts index c06f474..3d0e2ae 100644 --- a/backend/src/models/index.ts +++ b/backend/src/models/index.ts @@ -35,6 +35,8 @@ export const userQueries = { }; // Product types and queries +export type StockStatus = 'in_stock' | 'out_of_stock' | 'unknown'; + export interface Product { id: number; user_id: number; @@ -43,6 +45,7 @@ export interface Product { image_url: string | null; refresh_interval: number; last_checked: Date | null; + stock_status: StockStatus; created_at: Date; } @@ -159,13 +162,14 @@ export const productQueries = { url: string, name: string | null, imageUrl: string | null, - refreshInterval: number = 3600 + refreshInterval: number = 3600, + stockStatus: StockStatus = 'unknown' ): Promise => { const result = await pool.query( - `INSERT INTO products (user_id, url, name, image_url, refresh_interval) - VALUES ($1, $2, $3, $4, $5) + `INSERT INTO products (user_id, url, name, image_url, refresh_interval, stock_status) + VALUES ($1, $2, $3, $4, $5, $6) RETURNING *`, - [userId, url, name, imageUrl, refreshInterval] + [userId, url, name, imageUrl, refreshInterval, stockStatus] ); return result.rows[0]; }, @@ -215,6 +219,13 @@ export const productQueries = { ); }, + updateStockStatus: async (id: number, stockStatus: StockStatus): Promise => { + await pool.query( + 'UPDATE products SET stock_status = $1 WHERE id = $2', + [stockStatus, id] + ); + }, + findDueForRefresh: async (): Promise => { const result = await pool.query( `SELECT * FROM products diff --git a/backend/src/routes/prices.ts b/backend/src/routes/prices.ts index 51aa22f..4b632f0 100644 --- a/backend/src/routes/prices.ts +++ b/backend/src/routes/prices.ts @@ -1,7 +1,7 @@ import { Router, Response } from 'express'; import { AuthRequest, authMiddleware } from '../middleware/auth'; import { productQueries, priceHistoryQueries } from '../models'; -import { scrapePrice } from '../services/scraper'; +import { scrapeProduct } from '../services/scraper'; const router = Router(); @@ -62,27 +62,31 @@ router.post('/:productId/refresh', async (req: AuthRequest, res: Response) => { return; } - // Scrape new price - const priceData = await scrapePrice(product.url); + // Scrape product data including price and stock status + const scrapedData = await scrapeProduct(product.url); - if (!priceData) { - res.status(400).json({ error: 'Could not extract price from URL' }); - return; + // Update stock status + await productQueries.updateStockStatus(productId, scrapedData.stockStatus); + + // Record new price if available + let newPrice = null; + if (scrapedData.price) { + newPrice = await priceHistoryQueries.create( + productId, + scrapedData.price.price, + scrapedData.price.currency + ); } - // Record new price - const newPrice = await priceHistoryQueries.create( - productId, - priceData.price, - priceData.currency - ); - // Update last_checked timestamp await productQueries.updateLastChecked(productId); res.json({ - message: 'Price refreshed successfully', + message: scrapedData.stockStatus === 'out_of_stock' + ? 'Product is currently out of stock' + : 'Price refreshed successfully', price: newPrice, + stockStatus: scrapedData.stockStatus, }); } catch (error) { console.error('Error refreshing price:', error); diff --git a/backend/src/routes/products.ts b/backend/src/routes/products.ts index e47b0b7..7600c28 100644 --- a/backend/src/routes/products.ts +++ b/backend/src/routes/products.ts @@ -42,28 +42,32 @@ router.post('/', async (req: AuthRequest, res: Response) => { // Scrape product info const scrapedData = await scrapeProduct(url); - if (!scrapedData.price) { + // Allow adding out-of-stock products, but require a price for in-stock ones + if (!scrapedData.price && scrapedData.stockStatus !== 'out_of_stock') { res.status(400).json({ error: 'Could not extract price from the provided URL', }); return; } - // Create product + // Create product with stock status const product = await productQueries.create( userId, url, scrapedData.name, scrapedData.imageUrl, - refresh_interval || 3600 + refresh_interval || 3600, + scrapedData.stockStatus ); - // Record initial price - await priceHistoryQueries.create( - product.id, - scrapedData.price.price, - scrapedData.price.currency - ); + // Record initial price if available + if (scrapedData.price) { + await priceHistoryQueries.create( + product.id, + scrapedData.price.price, + scrapedData.price.currency + ); + } // Update last_checked timestamp await productQueries.updateLastChecked(product.id); diff --git a/backend/src/services/scheduler.ts b/backend/src/services/scheduler.ts index 8bdcb41..74069db 100644 --- a/backend/src/services/scheduler.ts +++ b/backend/src/services/scheduler.ts @@ -1,6 +1,6 @@ import cron from 'node-cron'; import { productQueries, priceHistoryQueries } from '../models'; -import { scrapePrice } from './scraper'; +import { scrapeProduct } from './scraper'; let isRunning = false; @@ -22,25 +22,35 @@ async function checkPrices(): Promise { try { console.log(`Checking price for product ${product.id}: ${product.url}`); - const priceData = await scrapePrice(product.url); + const scrapedData = await scrapeProduct(product.url); - if (priceData) { + // Update stock status + if (scrapedData.stockStatus !== product.stock_status) { + await productQueries.updateStockStatus(product.id, scrapedData.stockStatus); + console.log( + `Stock status changed for product ${product.id}: ${product.stock_status} -> ${scrapedData.stockStatus}` + ); + } + + if (scrapedData.price) { // Get the latest recorded price to compare const latestPrice = await priceHistoryQueries.getLatest(product.id); // Only record if price has changed or it's the first entry - if (!latestPrice || latestPrice.price !== priceData.price) { + if (!latestPrice || latestPrice.price !== scrapedData.price.price) { await priceHistoryQueries.create( product.id, - priceData.price, - priceData.currency + scrapedData.price.price, + scrapedData.price.currency ); console.log( - `Recorded new price for product ${product.id}: ${priceData.currency} ${priceData.price}` + `Recorded new price for product ${product.id}: ${scrapedData.price.currency} ${scrapedData.price.price}` ); } else { console.log(`Price unchanged for product ${product.id}`); } + } else if (scrapedData.stockStatus === 'out_of_stock') { + console.log(`Product ${product.id} is out of stock, no price available`); } else { console.warn(`Could not extract price for product ${product.id}`); } diff --git a/backend/src/services/scraper.ts b/backend/src/services/scraper.ts index f9a6190..754390e 100644 --- a/backend/src/services/scraper.ts +++ b/backend/src/services/scraper.ts @@ -6,17 +6,20 @@ import { findMostLikelyPrice, } from '../utils/priceParser'; +export type StockStatus = 'in_stock' | 'out_of_stock' | 'unknown'; + export interface ScrapedProduct { name: string | null; price: ParsedPrice | null; imageUrl: string | null; url: string; + stockStatus: StockStatus; } // Site-specific scraper configurations interface SiteScraper { match: (url: string) => boolean; - scrape: ($: CheerioAPI, url: string) => Partial; + scrape: ($: CheerioAPI, url: string) => Partial>; } const siteScrapers: SiteScraper[] = [ @@ -133,7 +136,38 @@ const siteScrapers: SiteScraper[] = [ $('img[data-a-dynamic-image]').attr('src') || null; - return { name, price, imageUrl }; + // Stock status detection + let stockStatus: StockStatus = 'unknown'; + const availabilityText = $('#availability').text().toLowerCase(); + const outOfStockDiv = $('#outOfStock').length > 0; + const unavailableText = $('body').text().toLowerCase(); + + // Check for out of stock indicators + if ( + outOfStockDiv || + availabilityText.includes('currently unavailable') || + availabilityText.includes('out of stock') || + availabilityText.includes('not available') || + $('#add-to-cart-button').length === 0 && $('#buy-now-button').length === 0 + ) { + // Verify it's truly out of stock by checking for unavailable messaging + if ( + unavailableText.includes('currently unavailable') || + unavailableText.includes("we don't know when or if this item will be back in stock") || + outOfStockDiv || + availabilityText.includes('out of stock') + ) { + stockStatus = 'out_of_stock'; + } + } else if ( + availabilityText.includes('in stock') || + availabilityText.includes('available') || + $('#add-to-cart-button').length > 0 + ) { + stockStatus = 'in_stock'; + } + + return { name, price, imageUrl, stockStatus }; }, }, @@ -407,6 +441,7 @@ export async function scrapeProduct(url: string): Promise { price: null, imageUrl: null, url, + stockStatus: 'unknown', }; try { @@ -442,6 +477,7 @@ export async function scrapeProduct(url: string): Promise { if (siteResult.name) result.name = siteResult.name; if (siteResult.price) result.price = siteResult.price; if (siteResult.imageUrl) result.imageUrl = siteResult.imageUrl; + if (siteResult.stockStatus) result.stockStatus = siteResult.stockStatus; } // Try JSON-LD structured data @@ -467,6 +503,11 @@ export async function scrapeProduct(url: string): Promise { result.imageUrl = extractGenericImage($, url); } + // Generic stock status detection if not already set + if (result.stockStatus === 'unknown') { + result.stockStatus = extractGenericStockStatus($); + } + // Try Open Graph meta tags as last resort if (!result.name) { result.name = $('meta[property="og:title"]').attr('content') || null; @@ -635,6 +676,68 @@ function extractGenericImage($: CheerioAPI, baseUrl: string): string | null { return null; } +function extractGenericStockStatus($: CheerioAPI): StockStatus { + const bodyText = $('body').text().toLowerCase(); + + // Common out-of-stock indicators + const outOfStockPatterns = [ + 'out of stock', + 'currently unavailable', + 'not available', + 'sold out', + 'no longer available', + 'temporarily out of stock', + 'unavailable', + 'back in stock soon', + 'notify me when available', + 'out-of-stock', + ]; + + // Common in-stock indicators + const inStockPatterns = [ + 'in stock', + 'add to cart', + 'add to basket', + 'buy now', + 'available', + 'ships from', + 'ready to ship', + ]; + + // Check for out-of-stock patterns + for (const pattern of outOfStockPatterns) { + if (bodyText.includes(pattern)) { + // Make sure it's not negated (e.g., "not out of stock") + const index = bodyText.indexOf(pattern); + const before = bodyText.slice(Math.max(0, index - 10), index); + if (!before.includes('not ') && !before.includes("isn't ") && !before.includes("won't be ")) { + return 'out_of_stock'; + } + } + } + + // Check for in-stock indicators + for (const pattern of inStockPatterns) { + if (bodyText.includes(pattern)) { + return 'in_stock'; + } + } + + // Check for schema.org availability + const availability = $('[itemprop="availability"]').attr('content') || + $('[itemprop="availability"]').attr('href') || ''; + if (availability.toLowerCase().includes('outofstock') || + availability.toLowerCase().includes('discontinued')) { + return 'out_of_stock'; + } + if (availability.toLowerCase().includes('instock') || + availability.toLowerCase().includes('available')) { + return 'in_stock'; + } + + return 'unknown'; +} + export async function scrapePrice(url: string): Promise { const product = await scrapeProduct(url); return product.price; diff --git a/database/init.sql b/database/init.sql index 1219589..b0820ce 100644 --- a/database/init.sql +++ b/database/init.sql @@ -17,10 +17,22 @@ CREATE TABLE IF NOT EXISTS products ( image_url TEXT, refresh_interval INTEGER DEFAULT 3600, last_checked TIMESTAMP, + stock_status VARCHAR(20) DEFAULT 'unknown', created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, UNIQUE(user_id, url) ); +-- Migration: Add stock_status column if it doesn't exist (for existing databases) +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_name = 'products' AND column_name = 'stock_status' + ) THEN + ALTER TABLE products ADD COLUMN stock_status VARCHAR(20) DEFAULT 'unknown'; + END IF; +END $$; + -- Price history table CREATE TABLE IF NOT EXISTS price_history ( id SERIAL PRIMARY KEY, diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index b6680e4..086c676 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -41,6 +41,8 @@ export const authApi = { }; // Products API +export type StockStatus = 'in_stock' | 'out_of_stock' | 'unknown'; + export interface SparklinePoint { price: number; recorded_at: string; @@ -54,6 +56,7 @@ export interface Product { image_url: string | null; refresh_interval: number; last_checked: string | null; + stock_status: StockStatus; created_at: string; current_price: number | null; currency: string | null; diff --git a/frontend/src/components/ProductCard.tsx b/frontend/src/components/ProductCard.tsx index ac17719..c81964a 100644 --- a/frontend/src/components/ProductCard.tsx +++ b/frontend/src/components/ProductCard.tsx @@ -40,8 +40,10 @@ export default function ProductCard({ product, onDelete }: ProductCardProps) { : '' : ''; + const isOutOfStock = product.stock_status === 'out_of_stock'; + return ( -
+