Add staggered checking with jitter to prevent rate limiting

- Add next_check_at column to track when each product should be checked
- New products get random initial delay (0 to refresh_interval) to spread them out
- Each check adds ±5 minute jitter so products naturally drift apart over time
- Randomize delay between requests (2-5 seconds instead of fixed 2s)

This prevents all products from being checked at the same time,
reducing the risk of being rate-limited or blocked by retailers.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
clucraft 2026-01-20 21:34:28 -05:00
parent a6928a0c17
commit a2b632d35b
3 changed files with 40 additions and 12 deletions

View file

@ -33,6 +33,7 @@ CREATE TABLE IF NOT EXISTS products (
image_url TEXT,
refresh_interval INTEGER DEFAULT 3600,
last_checked TIMESTAMP,
next_check_at TIMESTAMP,
stock_status VARCHAR(20) DEFAULT 'unknown',
price_drop_threshold DECIMAL(10,2),
notify_back_in_stock BOOLEAN DEFAULT false,
@ -63,6 +64,17 @@ BEGIN
END IF;
END $$;
-- Migration: Add next_check_at column for staggered checking
DO $$
BEGIN
IF NOT EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_name = 'products' AND column_name = 'next_check_at'
) THEN
ALTER TABLE products ADD COLUMN next_check_at TIMESTAMP;
END IF;
END $$;
-- Price history table
CREATE TABLE IF NOT EXISTS price_history (
id SERIAL PRIMARY KEY,