apunkt/ShopExtract
1
0
Fork 0
mirror of https://github.com/Coding-Doctor-Omar/ShopExtract.git synced 2026-06-22 08:38:20 +02:00
Code Issues Projects Releases Packages Wiki Activity

Add timeout to the main scraping wreq client.

Browse source
This commit is contained in:
Omar 2026-05-30 08:57:12 +03:00
parent e1a408e2a9
commit 73f3a18554
1 changed files with 2 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

3
main.py
View file

@ -1,6 +1,7 @@
from wreq.exceptions import DecodingError, TimeoutError, StatusError, BuilderError from wreq.exceptions import DecodingError, TimeoutError, StatusError, BuilderError
from wreq import Client, Emulation, Response from wreq import Client, Emulation, Response
from collections.abc import AsyncGenerator from collections.abc import AsyncGenerator
from datetime import timedelta
from asyncio import Semaphore from asyncio import Semaphore
from functools import wraps from functools import wraps
import pandas as pd import pandas as pd
@ -495,7 +496,7 @@ async def initiate_scraping_operation(store_url: str, output_csv_name: str="shop
if not output_csv_name: if not output_csv_name:
output_csv_name = "shopify" output_csv_name = "shopify"
scraping_client = Client(emulation=Emulation.Chrome147, cookie_store=True) scraping_client = Client(emulation=Emulation.Chrome147, cookie_store=True, timeout=timedelta(seconds=10))
print(f"Initializing scraping operation...\n") print(f"Initializing scraping operation...\n")
scrape_url = await get_scrape_url(store_url=store_url, client=scraping_client) scrape_url = await get_scrape_url(store_url=store_url, client=scraping_client)