diff --git a/backend/src/models/index.ts b/backend/src/models/index.ts index 6347475..8df98cf 100644 --- a/backend/src/models/index.ts +++ b/backend/src/models/index.ts @@ -94,12 +94,18 @@ export interface Product { image_url: string | null; refresh_interval: number; last_checked: Date | null; + next_check_at: Date | null; stock_status: StockStatus; price_drop_threshold: number | null; notify_back_in_stock: boolean; created_at: Date; } +// Generate jitter between -5 and +5 minutes (in seconds) +function getJitterSeconds(): number { + return Math.floor(Math.random() * 600) - 300; +} + export interface ProductWithLatestPrice extends Product { current_price: number | null; currency: string | null; @@ -216,11 +222,14 @@ export const productQueries = { refreshInterval: number = 3600, stockStatus: StockStatus = 'unknown' ): Promise => { + // Set initial next_check_at to a random time within the refresh interval + // This spreads out new products so they don't all check at once + const randomDelaySeconds = Math.floor(Math.random() * refreshInterval); const result = await pool.query( - `INSERT INTO products (user_id, url, name, image_url, refresh_interval, stock_status) - VALUES ($1, $2, $3, $4, $5, $6) + `INSERT INTO products (user_id, url, name, image_url, refresh_interval, stock_status, next_check_at) + VALUES ($1, $2, $3, $4, $5, $6, CURRENT_TIMESTAMP + ($7 || ' seconds')::interval) RETURNING *`, - [userId, url, name, imageUrl, refreshInterval, stockStatus] + [userId, url, name, imageUrl, refreshInterval, stockStatus, randomDelaySeconds] ); return result.rows[0]; }, @@ -276,10 +285,16 @@ export const productQueries = { return (result.rowCount ?? 0) > 0; }, - updateLastChecked: async (id: number): Promise => { + updateLastChecked: async (id: number, refreshInterval: number): Promise => { + // Add jitter of ±5 minutes to spread out checks over time + const jitterSeconds = getJitterSeconds(); + const nextCheckSeconds = refreshInterval + jitterSeconds; await pool.query( - 'UPDATE products SET last_checked = CURRENT_TIMESTAMP WHERE id = $1', - [id] + `UPDATE products + SET last_checked = CURRENT_TIMESTAMP, + next_check_at = CURRENT_TIMESTAMP + ($2 || ' seconds')::interval + WHERE id = $1`, + [id, nextCheckSeconds] ); }, @@ -293,8 +308,8 @@ export const productQueries = { findDueForRefresh: async (): Promise => { const result = await pool.query( `SELECT * FROM products - WHERE last_checked IS NULL - OR last_checked + (refresh_interval || ' seconds')::interval < CURRENT_TIMESTAMP` + WHERE next_check_at IS NULL + OR next_check_at < CURRENT_TIMESTAMP` ); return result.rows; }, diff --git a/backend/src/services/scheduler.ts b/backend/src/services/scheduler.ts index 4bccf6f..a42467e 100644 --- a/backend/src/services/scheduler.ts +++ b/backend/src/services/scheduler.ts @@ -108,11 +108,12 @@ async function checkPrices(): Promise { console.warn(`Could not extract price for product ${product.id}`); } - // Update last_checked even if price extraction failed - await productQueries.updateLastChecked(product.id); + // Update last_checked and schedule next check with jitter + await productQueries.updateLastChecked(product.id, product.refresh_interval); - // Add a small delay between requests to avoid rate limiting - await new Promise((resolve) => setTimeout(resolve, 2000)); + // Add a randomized delay between requests (2-5 seconds) to avoid rate limiting + const delay = 2000 + Math.floor(Math.random() * 3000); + await new Promise((resolve) => setTimeout(resolve, delay)); } catch (error) { console.error(`Error checking product ${product.id}:`, error); // Continue with next product even if one fails diff --git a/database/init.sql b/database/init.sql index 93f81d5..0aa7551 100644 --- a/database/init.sql +++ b/database/init.sql @@ -33,6 +33,7 @@ CREATE TABLE IF NOT EXISTS products ( image_url TEXT, refresh_interval INTEGER DEFAULT 3600, last_checked TIMESTAMP, + next_check_at TIMESTAMP, stock_status VARCHAR(20) DEFAULT 'unknown', price_drop_threshold DECIMAL(10,2), notify_back_in_stock BOOLEAN DEFAULT false, @@ -63,6 +64,17 @@ BEGIN END IF; END $$; +-- Migration: Add next_check_at column for staggered checking +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_name = 'products' AND column_name = 'next_check_at' + ) THEN + ALTER TABLE products ADD COLUMN next_check_at TIMESTAMP; + END IF; +END $$; + -- Price history table CREATE TABLE IF NOT EXISTS price_history ( id SERIAL PRIMARY KEY,