apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-22 23:31:12 +02:00
Code Issues Projects Releases Packages Wiki Activity

test(walmart): add parser and flow tests with e2e harness

Browse source
This commit is contained in:
CREDO23 2026-07-19 08:19:22 +02:00
parent a3f8a7e741
commit 7b5e09c528
8 changed files with 321 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

80
surfsense_backend/scripts/e2e_walmart_scraper.py Normal file
View file

@ -0,0 +1,80 @@
"""Manual end-to-end check for the public Walmart scraper.
Run from the backend directory:
uv run python scripts/e2e_walmart_scraper.py
The script requires live network access and the configured residential proxy
(US exit). It is intentionally excluded from pytest.
"""
from __future__ import annotations
import asyncio
import json
import sys
from pathlib import Path
from dotenv import load_dotenv
_BACKEND_ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(_BACKEND_ROOT))
for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"):
if _candidate.exists():
load_dotenv(_candidate)
break
from app.proprietary.platforms.walmart import ( # noqa: E402
WalmartReviewsInput,
WalmartScrapeInput,
scrape_products,
scrape_reviews,
)
_PRODUCT_URL = "https://www.walmart.com/ip/212092810"
_SEARCH_URL = "https://www.walmart.com/search?q=air+fryer"
def _check(label: str, passed: bool) -> bool:
print(f"[{'PASS' if passed else 'FAIL'}] {label}")
return passed
async def main() -> int:
ok = True
product_items = await scrape_products(
WalmartScrapeInput(startUrls=[_PRODUCT_URL]), limit=1
)
product = product_items[0] if product_items else {}
print(json.dumps(product, indent=2, ensure_ascii=False)[:2500])
ok &= _check(
"product detail has id and name",
bool(product.get("usItemId") and product.get("name")),
)
search_items = await scrape_products(
WalmartScrapeInput(
startUrls=[_SEARCH_URL], maxItemsPerStartUrl=3, includeDetails=False
),
limit=3,
)
ok &= _check(
"search returns product cards",
bool(search_items) and any("name" in item for item in search_items),
)
item_id = product.get("usItemId") or "212092810"
reviews = await scrape_reviews(
WalmartReviewsInput(itemIds=[item_id], maxReviews=15), limit=15
)
ok &= _check(
"reviews returned with rating and text",
bool(reviews) and any(r.get("rating") and r.get("text") for r in reviews),
)
return 0 if ok else 1
if __name__ == "__main__":
raise SystemExit(asyncio.run(main()))

0
surfsense_backend/tests/unit/platforms/walmart/__init__.py Normal file
View file

4
surfsense_backend/tests/unit/platforms/walmart/fixtures/blocked.html Normal file
View file

@ -0,0 +1,4 @@
<!doctype html><html><head><title>Robot or human?</title></head><body>
<div id="px-captcha">Activate and hold the button to confirm that you're human.</div>
<p>Robot or human?</p>
</body></html>

3
surfsense_backend/tests/unit/platforms/walmart/fixtures/listing.html Normal file
View file

@ -0,0 +1,3 @@
<!doctype html><html><head><title>laptop - Walmart.com</title></head><body>
<script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"initialData":{"searchResult":{"itemStacks":[{"count":2,"items":[{"__typename":"Product","usItemId":"791595618","name":"Restored Acer Chromebook","brand":null,"canonicalUrl":"/ip/Acer-Chromebook/791595618","averageRating":3.5,"numberOfReviews":46,"sellerId":"AB4FE","sellerName":"Ellison ProTech","availabilityStatusV2":{"value":"IN_STOCK"},"imageInfo":{"thumbnailUrl":"https://i5.walmartimages.com/a.jpg"},"priceInfo":{"currentPrice":{"price":79.99,"currencyUnit":"USD"}},"isSponsoredFlag":false},{"__typename":"Product","usItemId":"999001","name":"HP Laptop 15","canonicalUrl":"/ip/HP-Laptop/999001","isOutOfStock":true,"price":499.0}]}]}}}}}</script>
</body></html>

3
surfsense_backend/tests/unit/platforms/walmart/fixtures/product.html Normal file
View file

@ -0,0 +1,3 @@
<!doctype html><html><head><title>Midea AC</title></head><body>
<script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"initialData":{"data":{"product":{"usItemId":"212092810","id":"ABC123","name":"Midea 5,000 BTU Window Air Conditioner","brand":"Midea","availabilityStatus":"IN_STOCK","averageRating":4.5,"numberOfReviews":6287,"manufacturerName":"Midea","shortDescription":"Cools a small room fast.","sellerName":"Walmart.com","sellerId":"F55CT","priceInfo":{"currentPrice":{"price":149.0,"currencyUnit":"USD"},"wasPrice":{"price":199.0,"currencyUnit":"USD"}},"imageInfo":{"thumbnailUrl":"https://i5.walmartimages.com/t.jpg","allImages":[{"url":"https://i5.walmartimages.com/1.jpg"},{"url":"https://i5.walmartimages.com/2.jpg"}]},"category":{"path":"Home/Air Conditioners"},"variantCriteria":[{"name":"capacity"}]},"idml":{"longDescription":"<p>A powerful window unit.</p>"},"reviews":{"averageOverallRating":4.5,"totalReviewCount":6287,"aspects":[{"name":"Value","score":90}],"customerReviews":[{"reviewId":"r1","rating":5,"reviewTitle":"Great AC","reviewText":"Very cold air.","reviewSubmissionTime":"2024-04-28","userNickname":"Anne","positiveFeedback":3,"negativeFeedback":0,"badges":[{"id":"VerifiedPurchaser"}]}]}}}}}}</script>
</body></html>

3
surfsense_backend/tests/unit/platforms/walmart/fixtures/reviews.html Normal file
View file

@ -0,0 +1,3 @@
<!doctype html><html><head><title>Reviews</title></head><body>
<script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"initialData":{"data":{"reviews":{"customerReviews":[{"reviewId":"296013686","rating":5,"reviewTitle":"Awesome","reviewText":"The PS5 is awesome!","reviewSubmissionTime":"12/20/2022","userNickname":"JohnPaul","positiveFeedback":10,"negativeFeedback":3,"badges":[{"id":"VerifiedPurchaser"}],"photos":[{"normalUrl":"https://i5.walmartimages.com/r.jpg"}],"clientResponses":[{"response":"Thanks for the review!"}],"syndicationSource":null},{"reviewId":"296013687","rating":4,"reviewTitle":"Good","reviewText":"Solid console.","reviewSubmissionTime":"12/21/2022","userNickname":"Sam","positiveFeedback":1,"negativeFeedback":0,"badges":[]}]}}}}}}</script>
</body></html>

138
surfsense_backend/tests/unit/platforms/walmart/test_flows.py Normal file
View file

@ -0,0 +1,138 @@
from __future__ import annotations
from pathlib import Path
from app.proprietary.platforms.walmart import (
WalmartReviewsInput,
WalmartScrapeInput,
scrape_products,
scrape_reviews,
scraper,
)
from app.proprietary.platforms.walmart.fetch import FetchResult
_FIXTURES = Path(__file__).parent / "fixtures"
def _fixture(name: str) -> str:
return (_FIXTURES / name).read_text(encoding="utf-8")
def _response(url: str, html: str, status: int = 200) -> FetchResult:
return FetchResult(status=status, html=html, url=url, cookies={})
async def test_product_flow_emits_parsed_item(monkeypatch):
async def fetch_page(url: str, **_kwargs):
return _response(url, _fixture("product.html"))
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
items = await scrape_products(
WalmartScrapeInput(startUrls=["https://www.walmart.com/ip/212092810"])
)
assert len(items) == 1
assert items[0]["usItemId"] == "212092810"
assert items[0]["name"].startswith("Midea")
async def test_product_flow_maps_not_found_to_error_item(monkeypatch):
async def fetch_page(url: str, **_kwargs):
return _response(url, "", status=404)
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
items = await scrape_products(
WalmartScrapeInput(startUrls=["https://www.walmart.com/ip/212092810"])
)
assert items[0]["error"] == "product_not_found"
async def test_listing_flow_card_only_honors_cap(monkeypatch):
calls: list[str] = []
async def fetch_page(url: str, **_kwargs):
calls.append(url)
html = _fixture("listing.html") if "page=1" in url else "<html></html>"
return _response(url, html)
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
items = await scrape_products(
WalmartScrapeInput(
startUrls=["https://www.walmart.com/search?q=laptop"],
maxItemsPerStartUrl=1,
includeDetails=False,
)
)
assert len(items) == 1
assert items[0]["usItemId"] == "791595618"
assert len(calls) == 1
async def test_listing_flow_enriches_with_detail_pages(monkeypatch):
async def fetch_page(url: str, **_kwargs):
if "/ip/" in url:
return _response(url, _fixture("product.html"))
html = _fixture("listing.html") if "page=1" in url else "<html></html>"
return _response(url, html)
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
items = await scrape_products(
WalmartScrapeInput(
startUrls=["https://www.walmart.com/search?q=laptop"],
maxItemsPerStartUrl=2,
includeDetails=True,
)
)
# Both cards enrich to the same product fixture (detail fetch wins).
assert all(item["longDescription"] for item in items)
async def test_invalid_url_yields_error_item(monkeypatch):
items = await scrape_products(
WalmartScrapeInput(startUrls=["https://example.com/not-walmart"])
)
assert items[0]["error"] == "invalid_url"
async def test_reviews_flow_paginates_until_empty(monkeypatch):
calls: list[str] = []
async def fetch_page(url: str, **_kwargs):
calls.append(url)
html = _fixture("reviews.html") if "page=1" in url else "<html></html>"
return _response(url, html)
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
items = await scrape_reviews(
WalmartReviewsInput(itemIds=["212092810"], maxReviews=100)
)
assert len(items) == 2
assert items[0]["reviewId"] == "296013686"
# page=1 returned records, page=2 empty → stop. Two fetches total.
assert len(calls) == 2
async def test_reviews_flow_honors_max_reviews(monkeypatch):
async def fetch_page(url: str, **_kwargs):
return _response(url, _fixture("reviews.html"))
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
items = await scrape_reviews(
WalmartReviewsInput(itemIds=["212092810"], maxReviews=1)
)
assert len(items) == 1
async def test_reviews_flow_maps_empty_to_error_item(monkeypatch):
async def fetch_page(url: str, **_kwargs):
return _response(url, "<html></html>")
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
items = await scrape_reviews(WalmartReviewsInput(itemIds=["212092810"]))
assert items[0]["error"] == "reviews_not_found"

90
surfsense_backend/tests/unit/platforms/walmart/test_parsers.py Normal file
View file

@ -0,0 +1,90 @@
from __future__ import annotations
from pathlib import Path
from app.proprietary.platforms.walmart.fetch import is_blocked
from app.proprietary.platforms.walmart.next_data import extract_next_data
from app.proprietary.platforms.walmart.parsers import (
parse_listing_page,
parse_product,
parse_reviews_page,
)
from app.proprietary.platforms.walmart.url_resolver import extract_item_id, resolve_url
_FIXTURES = Path(__file__).parent / "fixtures"
def _fixture(name: str) -> str:
return (_FIXTURES / name).read_text(encoding="utf-8")
def test_product_parser_extracts_core_fields_and_review_sample():
data = extract_next_data(_fixture("product.html"))
item = parse_product(data, url="https://www.walmart.com/ip/212092810")
assert item["usItemId"] == "212092810"
assert item["name"].startswith("Midea")
assert item["price"] == {"value": 149.0, "currency": "USD"}
assert item["listPrice"] == {"value": 199.0, "currency": "USD"}
assert item["stars"] == 4.5
assert item["reviewsCount"] == 6287
assert item["inStock"] is True
assert item["seller"] == {"id": "F55CT", "name": "Walmart.com", "type": "WALMART"}
assert item["longDescription"] == "<p>A powerful window unit.</p>"
assert item["images"] == [
"https://i5.walmartimages.com/1.jpg",
"https://i5.walmartimages.com/2.jpg",
]
assert item["reviewsSample"]["totalReviewCount"] == 6287
assert item["reviewsSample"]["topReviews"][0]["verifiedPurchase"] is True
def test_listing_parser_normalizes_cards_and_marketplace_seller():
data = extract_next_data(_fixture("listing.html"))
cards = parse_listing_page(data)
assert [c["usItemId"] for c in cards] == ["791595618", "999001"]
first = cards[0]
assert first["url"] == "https://www.walmart.com/ip/Acer-Chromebook/791595618"
assert first["price"] == {"value": 79.99, "currency": "USD"}
assert first["seller"]["type"] == "MARKETPLACE"
assert first["inStock"] is True
# Fallback price + out-of-stock derivation for the second card.
assert cards[1]["price"] == {"value": 499.0}
assert cards[1]["inStock"] is False
def test_reviews_parser_extracts_all_records():
data = extract_next_data(_fixture("reviews.html"))
reviews = parse_reviews_page(data)
assert len(reviews) == 2
assert reviews[0]["reviewId"] == "296013686"
assert reviews[0]["author"] == "JohnPaul"
assert reviews[0]["verifiedPurchase"] is True
assert reviews[0]["images"] == ["https://i5.walmartimages.com/r.jpg"]
assert reviews[0]["sellerResponse"] == "Thanks for the review!"
assert reviews[1]["verifiedPurchase"] is False
def test_block_detection_handles_status_and_body():
assert is_blocked(_fixture("blocked.html"), 200)
assert is_blocked("", 412)
assert is_blocked("", 429)
assert not is_blocked(_fixture("product.html"), 200)
def test_missing_next_data_yields_none():
assert extract_next_data("<html><body>no script here</body></html>") is None
assert parse_product(extract_next_data("") or {}, url="x") == {}
def test_url_resolver_classifies_and_extracts_ids():
assert resolve_url("https://www.walmart.com/ip/Foo/123456789").kind == "product"
assert (
extract_item_id("https://www.walmart.com/reviews/product/212092810")
== "212092810"
)
assert resolve_url("https://www.walmart.com/search?q=tv").kind == "listing"
assert resolve_url("https://www.walmart.com/cp/tvs/3944").kind == "listing"
assert resolve_url("https://example.com/ip/1") is None