mirror of
https://github.com/clucraft/PriceGhost.git
synced 2026-05-18 13:55:16 +02:00
Add staggered checking with jitter to prevent rate limiting
- Add next_check_at column to track when each product should be checked - New products get random initial delay (0 to refresh_interval) to spread them out - Each check adds ±5 minute jitter so products naturally drift apart over time - Randomize delay between requests (2-5 seconds instead of fixed 2s) This prevents all products from being checked at the same time, reducing the risk of being rate-limited or blocked by retailers. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
parent
a6928a0c17
commit
a2b632d35b
3 changed files with 40 additions and 12 deletions
|
|
@ -108,11 +108,12 @@ async function checkPrices(): Promise<void> {
|
|||
console.warn(`Could not extract price for product ${product.id}`);
|
||||
}
|
||||
|
||||
// Update last_checked even if price extraction failed
|
||||
await productQueries.updateLastChecked(product.id);
|
||||
// Update last_checked and schedule next check with jitter
|
||||
await productQueries.updateLastChecked(product.id, product.refresh_interval);
|
||||
|
||||
// Add a small delay between requests to avoid rate limiting
|
||||
await new Promise((resolve) => setTimeout(resolve, 2000));
|
||||
// Add a randomized delay between requests (2-5 seconds) to avoid rate limiting
|
||||
const delay = 2000 + Math.floor(Math.random() * 3000);
|
||||
await new Promise((resolve) => setTimeout(resolve, delay));
|
||||
} catch (error) {
|
||||
console.error(`Error checking product ${product.id}:`, error);
|
||||
// Continue with next product even if one fails
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue