mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-22 23:31:12 +02:00
test(walmart): add parser and flow tests with e2e harness
This commit is contained in:
CREDO23
parent
a3f8a7e741
commit
7b5e09c528
8 changed files with 321 additions and 0 deletions
80
surfsense_backend/scripts/e2e_walmart_scraper.py
Normal file
80
surfsense_backend/scripts/e2e_walmart_scraper.py
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
"""Manual end-to-end check for the public Walmart scraper.
|
||||
|
||||
Run from the backend directory:
|
||||
|
||||
uv run python scripts/e2e_walmart_scraper.py
|
||||
|
||||
The script requires live network access and the configured residential proxy
|
||||
(US exit). It is intentionally excluded from pytest.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
_BACKEND_ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(_BACKEND_ROOT))
|
||||
for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"):
|
||||
if _candidate.exists():
|
||||
load_dotenv(_candidate)
|
||||
break
|
||||
|
||||
from app.proprietary.platforms.walmart import ( # noqa: E402
|
||||
WalmartReviewsInput,
|
||||
WalmartScrapeInput,
|
||||
scrape_products,
|
||||
scrape_reviews,
|
||||
)
|
||||
|
||||
_PRODUCT_URL = "https://www.walmart.com/ip/212092810"
|
||||
_SEARCH_URL = "https://www.walmart.com/search?q=air+fryer"
|
||||
|
||||
|
||||
def _check(label: str, passed: bool) -> bool:
|
||||
print(f"[{'PASS' if passed else 'FAIL'}] {label}")
|
||||
return passed
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
ok = True
|
||||
|
||||
product_items = await scrape_products(
|
||||
WalmartScrapeInput(startUrls=[_PRODUCT_URL]), limit=1
|
||||
)
|
||||
product = product_items[0] if product_items else {}
|
||||
print(json.dumps(product, indent=2, ensure_ascii=False)[:2500])
|
||||
ok &= _check(
|
||||
"product detail has id and name",
|
||||
bool(product.get("usItemId") and product.get("name")),
|
||||
)
|
||||
|
||||
search_items = await scrape_products(
|
||||
WalmartScrapeInput(
|
||||
startUrls=[_SEARCH_URL], maxItemsPerStartUrl=3, includeDetails=False
|
||||
),
|
||||
limit=3,
|
||||
)
|
||||
ok &= _check(
|
||||
"search returns product cards",
|
||||
bool(search_items) and any("name" in item for item in search_items),
|
||||
)
|
||||
|
||||
item_id = product.get("usItemId") or "212092810"
|
||||
reviews = await scrape_reviews(
|
||||
WalmartReviewsInput(itemIds=[item_id], maxReviews=15), limit=15
|
||||
)
|
||||
ok &= _check(
|
||||
"reviews returned with rating and text",
|
||||
bool(reviews) and any(r.get("rating") and r.get("text") for r in reviews),
|
||||
)
|
||||
|
||||
return 0 if ok else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(asyncio.run(main()))
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
<!doctype html><html><head><title>Robot or human?</title></head><body>
|
||||
<div id="px-captcha">Activate and hold the button to confirm that you're human.</div>
|
||||
<p>Robot or human?</p>
|
||||
</body></html>
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
<!doctype html><html><head><title>laptop - Walmart.com</title></head><body>
|
||||
<script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"initialData":{"searchResult":{"itemStacks":[{"count":2,"items":[{"__typename":"Product","usItemId":"791595618","name":"Restored Acer Chromebook","brand":null,"canonicalUrl":"/ip/Acer-Chromebook/791595618","averageRating":3.5,"numberOfReviews":46,"sellerId":"AB4FE","sellerName":"Ellison ProTech","availabilityStatusV2":{"value":"IN_STOCK"},"imageInfo":{"thumbnailUrl":"https://i5.walmartimages.com/a.jpg"},"priceInfo":{"currentPrice":{"price":79.99,"currencyUnit":"USD"}},"isSponsoredFlag":false},{"__typename":"Product","usItemId":"999001","name":"HP Laptop 15","canonicalUrl":"/ip/HP-Laptop/999001","isOutOfStock":true,"price":499.0}]}]}}}}}</script>
|
||||
</body></html>
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
<!doctype html><html><head><title>Midea AC</title></head><body>
|
||||
<script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"initialData":{"data":{"product":{"usItemId":"212092810","id":"ABC123","name":"Midea 5,000 BTU Window Air Conditioner","brand":"Midea","availabilityStatus":"IN_STOCK","averageRating":4.5,"numberOfReviews":6287,"manufacturerName":"Midea","shortDescription":"Cools a small room fast.","sellerName":"Walmart.com","sellerId":"F55CT","priceInfo":{"currentPrice":{"price":149.0,"currencyUnit":"USD"},"wasPrice":{"price":199.0,"currencyUnit":"USD"}},"imageInfo":{"thumbnailUrl":"https://i5.walmartimages.com/t.jpg","allImages":[{"url":"https://i5.walmartimages.com/1.jpg"},{"url":"https://i5.walmartimages.com/2.jpg"}]},"category":{"path":"Home/Air Conditioners"},"variantCriteria":[{"name":"capacity"}]},"idml":{"longDescription":"<p>A powerful window unit.</p>"},"reviews":{"averageOverallRating":4.5,"totalReviewCount":6287,"aspects":[{"name":"Value","score":90}],"customerReviews":[{"reviewId":"r1","rating":5,"reviewTitle":"Great AC","reviewText":"Very cold air.","reviewSubmissionTime":"2024-04-28","userNickname":"Anne","positiveFeedback":3,"negativeFeedback":0,"badges":[{"id":"VerifiedPurchaser"}]}]}}}}}}</script>
|
||||
</body></html>
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
<!doctype html><html><head><title>Reviews</title></head><body>
|
||||
<script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"initialData":{"data":{"reviews":{"customerReviews":[{"reviewId":"296013686","rating":5,"reviewTitle":"Awesome","reviewText":"The PS5 is awesome!","reviewSubmissionTime":"12/20/2022","userNickname":"JohnPaul","positiveFeedback":10,"negativeFeedback":3,"badges":[{"id":"VerifiedPurchaser"}],"photos":[{"normalUrl":"https://i5.walmartimages.com/r.jpg"}],"clientResponses":[{"response":"Thanks for the review!"}],"syndicationSource":null},{"reviewId":"296013687","rating":4,"reviewTitle":"Good","reviewText":"Solid console.","reviewSubmissionTime":"12/21/2022","userNickname":"Sam","positiveFeedback":1,"negativeFeedback":0,"badges":[]}]}}}}}}</script>
|
||||
</body></html>
|
||||
138
surfsense_backend/tests/unit/platforms/walmart/test_flows.py
Normal file
138
surfsense_backend/tests/unit/platforms/walmart/test_flows.py
Normal file
|
|
@ -0,0 +1,138 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from app.proprietary.platforms.walmart import (
|
||||
WalmartReviewsInput,
|
||||
WalmartScrapeInput,
|
||||
scrape_products,
|
||||
scrape_reviews,
|
||||
scraper,
|
||||
)
|
||||
from app.proprietary.platforms.walmart.fetch import FetchResult
|
||||
|
||||
_FIXTURES = Path(__file__).parent / "fixtures"
|
||||
|
||||
|
||||
def _fixture(name: str) -> str:
|
||||
return (_FIXTURES / name).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def _response(url: str, html: str, status: int = 200) -> FetchResult:
|
||||
return FetchResult(status=status, html=html, url=url, cookies={})
|
||||
|
||||
|
||||
async def test_product_flow_emits_parsed_item(monkeypatch):
|
||||
async def fetch_page(url: str, **_kwargs):
|
||||
return _response(url, _fixture("product.html"))
|
||||
|
||||
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
|
||||
items = await scrape_products(
|
||||
WalmartScrapeInput(startUrls=["https://www.walmart.com/ip/212092810"])
|
||||
)
|
||||
|
||||
assert len(items) == 1
|
||||
assert items[0]["usItemId"] == "212092810"
|
||||
assert items[0]["name"].startswith("Midea")
|
||||
|
||||
|
||||
async def test_product_flow_maps_not_found_to_error_item(monkeypatch):
|
||||
async def fetch_page(url: str, **_kwargs):
|
||||
return _response(url, "", status=404)
|
||||
|
||||
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
|
||||
items = await scrape_products(
|
||||
WalmartScrapeInput(startUrls=["https://www.walmart.com/ip/212092810"])
|
||||
)
|
||||
|
||||
assert items[0]["error"] == "product_not_found"
|
||||
|
||||
|
||||
async def test_listing_flow_card_only_honors_cap(monkeypatch):
|
||||
calls: list[str] = []
|
||||
|
||||
async def fetch_page(url: str, **_kwargs):
|
||||
calls.append(url)
|
||||
html = _fixture("listing.html") if "page=1" in url else "<html></html>"
|
||||
return _response(url, html)
|
||||
|
||||
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
|
||||
items = await scrape_products(
|
||||
WalmartScrapeInput(
|
||||
startUrls=["https://www.walmart.com/search?q=laptop"],
|
||||
maxItemsPerStartUrl=1,
|
||||
includeDetails=False,
|
||||
)
|
||||
)
|
||||
|
||||
assert len(items) == 1
|
||||
assert items[0]["usItemId"] == "791595618"
|
||||
assert len(calls) == 1
|
||||
|
||||
|
||||
async def test_listing_flow_enriches_with_detail_pages(monkeypatch):
|
||||
async def fetch_page(url: str, **_kwargs):
|
||||
if "/ip/" in url:
|
||||
return _response(url, _fixture("product.html"))
|
||||
html = _fixture("listing.html") if "page=1" in url else "<html></html>"
|
||||
return _response(url, html)
|
||||
|
||||
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
|
||||
items = await scrape_products(
|
||||
WalmartScrapeInput(
|
||||
startUrls=["https://www.walmart.com/search?q=laptop"],
|
||||
maxItemsPerStartUrl=2,
|
||||
includeDetails=True,
|
||||
)
|
||||
)
|
||||
|
||||
# Both cards enrich to the same product fixture (detail fetch wins).
|
||||
assert all(item["longDescription"] for item in items)
|
||||
|
||||
|
||||
async def test_invalid_url_yields_error_item(monkeypatch):
|
||||
items = await scrape_products(
|
||||
WalmartScrapeInput(startUrls=["https://example.com/not-walmart"])
|
||||
)
|
||||
assert items[0]["error"] == "invalid_url"
|
||||
|
||||
|
||||
async def test_reviews_flow_paginates_until_empty(monkeypatch):
|
||||
calls: list[str] = []
|
||||
|
||||
async def fetch_page(url: str, **_kwargs):
|
||||
calls.append(url)
|
||||
html = _fixture("reviews.html") if "page=1" in url else "<html></html>"
|
||||
return _response(url, html)
|
||||
|
||||
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
|
||||
items = await scrape_reviews(
|
||||
WalmartReviewsInput(itemIds=["212092810"], maxReviews=100)
|
||||
)
|
||||
|
||||
assert len(items) == 2
|
||||
assert items[0]["reviewId"] == "296013686"
|
||||
# page=1 returned records, page=2 empty → stop. Two fetches total.
|
||||
assert len(calls) == 2
|
||||
|
||||
|
||||
async def test_reviews_flow_honors_max_reviews(monkeypatch):
|
||||
async def fetch_page(url: str, **_kwargs):
|
||||
return _response(url, _fixture("reviews.html"))
|
||||
|
||||
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
|
||||
items = await scrape_reviews(
|
||||
WalmartReviewsInput(itemIds=["212092810"], maxReviews=1)
|
||||
)
|
||||
|
||||
assert len(items) == 1
|
||||
|
||||
|
||||
async def test_reviews_flow_maps_empty_to_error_item(monkeypatch):
|
||||
async def fetch_page(url: str, **_kwargs):
|
||||
return _response(url, "<html></html>")
|
||||
|
||||
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
|
||||
items = await scrape_reviews(WalmartReviewsInput(itemIds=["212092810"]))
|
||||
|
||||
assert items[0]["error"] == "reviews_not_found"
|
||||
|
|
@ -0,0 +1,90 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from app.proprietary.platforms.walmart.fetch import is_blocked
|
||||
from app.proprietary.platforms.walmart.next_data import extract_next_data
|
||||
from app.proprietary.platforms.walmart.parsers import (
|
||||
parse_listing_page,
|
||||
parse_product,
|
||||
parse_reviews_page,
|
||||
)
|
||||
from app.proprietary.platforms.walmart.url_resolver import extract_item_id, resolve_url
|
||||
|
||||
_FIXTURES = Path(__file__).parent / "fixtures"
|
||||
|
||||
|
||||
def _fixture(name: str) -> str:
|
||||
return (_FIXTURES / name).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_product_parser_extracts_core_fields_and_review_sample():
|
||||
data = extract_next_data(_fixture("product.html"))
|
||||
item = parse_product(data, url="https://www.walmart.com/ip/212092810")
|
||||
|
||||
assert item["usItemId"] == "212092810"
|
||||
assert item["name"].startswith("Midea")
|
||||
assert item["price"] == {"value": 149.0, "currency": "USD"}
|
||||
assert item["listPrice"] == {"value": 199.0, "currency": "USD"}
|
||||
assert item["stars"] == 4.5
|
||||
assert item["reviewsCount"] == 6287
|
||||
assert item["inStock"] is True
|
||||
assert item["seller"] == {"id": "F55CT", "name": "Walmart.com", "type": "WALMART"}
|
||||
assert item["longDescription"] == "<p>A powerful window unit.</p>"
|
||||
assert item["images"] == [
|
||||
"https://i5.walmartimages.com/1.jpg",
|
||||
"https://i5.walmartimages.com/2.jpg",
|
||||
]
|
||||
assert item["reviewsSample"]["totalReviewCount"] == 6287
|
||||
assert item["reviewsSample"]["topReviews"][0]["verifiedPurchase"] is True
|
||||
|
||||
|
||||
def test_listing_parser_normalizes_cards_and_marketplace_seller():
|
||||
data = extract_next_data(_fixture("listing.html"))
|
||||
cards = parse_listing_page(data)
|
||||
|
||||
assert [c["usItemId"] for c in cards] == ["791595618", "999001"]
|
||||
first = cards[0]
|
||||
assert first["url"] == "https://www.walmart.com/ip/Acer-Chromebook/791595618"
|
||||
assert first["price"] == {"value": 79.99, "currency": "USD"}
|
||||
assert first["seller"]["type"] == "MARKETPLACE"
|
||||
assert first["inStock"] is True
|
||||
# Fallback price + out-of-stock derivation for the second card.
|
||||
assert cards[1]["price"] == {"value": 499.0}
|
||||
assert cards[1]["inStock"] is False
|
||||
|
||||
|
||||
def test_reviews_parser_extracts_all_records():
|
||||
data = extract_next_data(_fixture("reviews.html"))
|
||||
reviews = parse_reviews_page(data)
|
||||
|
||||
assert len(reviews) == 2
|
||||
assert reviews[0]["reviewId"] == "296013686"
|
||||
assert reviews[0]["author"] == "JohnPaul"
|
||||
assert reviews[0]["verifiedPurchase"] is True
|
||||
assert reviews[0]["images"] == ["https://i5.walmartimages.com/r.jpg"]
|
||||
assert reviews[0]["sellerResponse"] == "Thanks for the review!"
|
||||
assert reviews[1]["verifiedPurchase"] is False
|
||||
|
||||
|
||||
def test_block_detection_handles_status_and_body():
|
||||
assert is_blocked(_fixture("blocked.html"), 200)
|
||||
assert is_blocked("", 412)
|
||||
assert is_blocked("", 429)
|
||||
assert not is_blocked(_fixture("product.html"), 200)
|
||||
|
||||
|
||||
def test_missing_next_data_yields_none():
|
||||
assert extract_next_data("<html><body>no script here</body></html>") is None
|
||||
assert parse_product(extract_next_data("") or {}, url="x") == {}
|
||||
|
||||
|
||||
def test_url_resolver_classifies_and_extracts_ids():
|
||||
assert resolve_url("https://www.walmart.com/ip/Foo/123456789").kind == "product"
|
||||
assert (
|
||||
extract_item_id("https://www.walmart.com/reviews/product/212092810")
|
||||
== "212092810"
|
||||
)
|
||||
assert resolve_url("https://www.walmart.com/search?q=tv").kind == "listing"
|
||||
assert resolve_url("https://www.walmart.com/cp/tvs/3944").kind == "listing"
|
||||
assert resolve_url("https://example.com/ip/1") is None
|
||||
Loading…
Add table
Add a link
Reference in a new issue