diff --git a/surfsense_backend/scripts/e2e_walmart_scraper.py b/surfsense_backend/scripts/e2e_walmart_scraper.py new file mode 100644 index 000000000..28ed3a344 --- /dev/null +++ b/surfsense_backend/scripts/e2e_walmart_scraper.py @@ -0,0 +1,80 @@ +"""Manual end-to-end check for the public Walmart scraper. + +Run from the backend directory: + + uv run python scripts/e2e_walmart_scraper.py + +The script requires live network access and the configured residential proxy +(US exit). It is intentionally excluded from pytest. +""" + +from __future__ import annotations + +import asyncio +import json +import sys +from pathlib import Path + +from dotenv import load_dotenv + +_BACKEND_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(_BACKEND_ROOT)) +for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"): + if _candidate.exists(): + load_dotenv(_candidate) + break + +from app.proprietary.platforms.walmart import ( # noqa: E402 + WalmartReviewsInput, + WalmartScrapeInput, + scrape_products, + scrape_reviews, +) + +_PRODUCT_URL = "https://www.walmart.com/ip/212092810" +_SEARCH_URL = "https://www.walmart.com/search?q=air+fryer" + + +def _check(label: str, passed: bool) -> bool: + print(f"[{'PASS' if passed else 'FAIL'}] {label}") + return passed + + +async def main() -> int: + ok = True + + product_items = await scrape_products( + WalmartScrapeInput(startUrls=[_PRODUCT_URL]), limit=1 + ) + product = product_items[0] if product_items else {} + print(json.dumps(product, indent=2, ensure_ascii=False)[:2500]) + ok &= _check( + "product detail has id and name", + bool(product.get("usItemId") and product.get("name")), + ) + + search_items = await scrape_products( + WalmartScrapeInput( + startUrls=[_SEARCH_URL], maxItemsPerStartUrl=3, includeDetails=False + ), + limit=3, + ) + ok &= _check( + "search returns product cards", + bool(search_items) and any("name" in item for item in search_items), + ) + + item_id = product.get("usItemId") or "212092810" + reviews = await scrape_reviews( + WalmartReviewsInput(itemIds=[item_id], maxReviews=15), limit=15 + ) + ok &= _check( + "reviews returned with rating and text", + bool(reviews) and any(r.get("rating") and r.get("text") for r in reviews), + ) + + return 0 if ok else 1 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) diff --git a/surfsense_backend/tests/unit/platforms/walmart/__init__.py b/surfsense_backend/tests/unit/platforms/walmart/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/platforms/walmart/fixtures/blocked.html b/surfsense_backend/tests/unit/platforms/walmart/fixtures/blocked.html new file mode 100644 index 000000000..a9e9d606e --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/walmart/fixtures/blocked.html @@ -0,0 +1,4 @@ +Robot or human? +
Activate and hold the button to confirm that you're human.
+

Robot or human?

+ diff --git a/surfsense_backend/tests/unit/platforms/walmart/fixtures/listing.html b/surfsense_backend/tests/unit/platforms/walmart/fixtures/listing.html new file mode 100644 index 000000000..0ebb7e682 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/walmart/fixtures/listing.html @@ -0,0 +1,3 @@ +laptop - Walmart.com + + diff --git a/surfsense_backend/tests/unit/platforms/walmart/fixtures/product.html b/surfsense_backend/tests/unit/platforms/walmart/fixtures/product.html new file mode 100644 index 000000000..3db17b152 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/walmart/fixtures/product.html @@ -0,0 +1,3 @@ +Midea AC + + diff --git a/surfsense_backend/tests/unit/platforms/walmart/fixtures/reviews.html b/surfsense_backend/tests/unit/platforms/walmart/fixtures/reviews.html new file mode 100644 index 000000000..b0d27e5b6 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/walmart/fixtures/reviews.html @@ -0,0 +1,3 @@ +Reviews + + diff --git a/surfsense_backend/tests/unit/platforms/walmart/test_flows.py b/surfsense_backend/tests/unit/platforms/walmart/test_flows.py new file mode 100644 index 000000000..045aa99d5 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/walmart/test_flows.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +from pathlib import Path + +from app.proprietary.platforms.walmart import ( + WalmartReviewsInput, + WalmartScrapeInput, + scrape_products, + scrape_reviews, + scraper, +) +from app.proprietary.platforms.walmart.fetch import FetchResult + +_FIXTURES = Path(__file__).parent / "fixtures" + + +def _fixture(name: str) -> str: + return (_FIXTURES / name).read_text(encoding="utf-8") + + +def _response(url: str, html: str, status: int = 200) -> FetchResult: + return FetchResult(status=status, html=html, url=url, cookies={}) + + +async def test_product_flow_emits_parsed_item(monkeypatch): + async def fetch_page(url: str, **_kwargs): + return _response(url, _fixture("product.html")) + + monkeypatch.setattr(scraper, "fetch_page", fetch_page) + items = await scrape_products( + WalmartScrapeInput(startUrls=["https://www.walmart.com/ip/212092810"]) + ) + + assert len(items) == 1 + assert items[0]["usItemId"] == "212092810" + assert items[0]["name"].startswith("Midea") + + +async def test_product_flow_maps_not_found_to_error_item(monkeypatch): + async def fetch_page(url: str, **_kwargs): + return _response(url, "", status=404) + + monkeypatch.setattr(scraper, "fetch_page", fetch_page) + items = await scrape_products( + WalmartScrapeInput(startUrls=["https://www.walmart.com/ip/212092810"]) + ) + + assert items[0]["error"] == "product_not_found" + + +async def test_listing_flow_card_only_honors_cap(monkeypatch): + calls: list[str] = [] + + async def fetch_page(url: str, **_kwargs): + calls.append(url) + html = _fixture("listing.html") if "page=1" in url else "" + return _response(url, html) + + monkeypatch.setattr(scraper, "fetch_page", fetch_page) + items = await scrape_products( + WalmartScrapeInput( + startUrls=["https://www.walmart.com/search?q=laptop"], + maxItemsPerStartUrl=1, + includeDetails=False, + ) + ) + + assert len(items) == 1 + assert items[0]["usItemId"] == "791595618" + assert len(calls) == 1 + + +async def test_listing_flow_enriches_with_detail_pages(monkeypatch): + async def fetch_page(url: str, **_kwargs): + if "/ip/" in url: + return _response(url, _fixture("product.html")) + html = _fixture("listing.html") if "page=1" in url else "" + return _response(url, html) + + monkeypatch.setattr(scraper, "fetch_page", fetch_page) + items = await scrape_products( + WalmartScrapeInput( + startUrls=["https://www.walmart.com/search?q=laptop"], + maxItemsPerStartUrl=2, + includeDetails=True, + ) + ) + + # Both cards enrich to the same product fixture (detail fetch wins). + assert all(item["longDescription"] for item in items) + + +async def test_invalid_url_yields_error_item(monkeypatch): + items = await scrape_products( + WalmartScrapeInput(startUrls=["https://example.com/not-walmart"]) + ) + assert items[0]["error"] == "invalid_url" + + +async def test_reviews_flow_paginates_until_empty(monkeypatch): + calls: list[str] = [] + + async def fetch_page(url: str, **_kwargs): + calls.append(url) + html = _fixture("reviews.html") if "page=1" in url else "" + return _response(url, html) + + monkeypatch.setattr(scraper, "fetch_page", fetch_page) + items = await scrape_reviews( + WalmartReviewsInput(itemIds=["212092810"], maxReviews=100) + ) + + assert len(items) == 2 + assert items[0]["reviewId"] == "296013686" + # page=1 returned records, page=2 empty → stop. Two fetches total. + assert len(calls) == 2 + + +async def test_reviews_flow_honors_max_reviews(monkeypatch): + async def fetch_page(url: str, **_kwargs): + return _response(url, _fixture("reviews.html")) + + monkeypatch.setattr(scraper, "fetch_page", fetch_page) + items = await scrape_reviews( + WalmartReviewsInput(itemIds=["212092810"], maxReviews=1) + ) + + assert len(items) == 1 + + +async def test_reviews_flow_maps_empty_to_error_item(monkeypatch): + async def fetch_page(url: str, **_kwargs): + return _response(url, "") + + monkeypatch.setattr(scraper, "fetch_page", fetch_page) + items = await scrape_reviews(WalmartReviewsInput(itemIds=["212092810"])) + + assert items[0]["error"] == "reviews_not_found" diff --git a/surfsense_backend/tests/unit/platforms/walmart/test_parsers.py b/surfsense_backend/tests/unit/platforms/walmart/test_parsers.py new file mode 100644 index 000000000..da66064ab --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/walmart/test_parsers.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +from pathlib import Path + +from app.proprietary.platforms.walmart.fetch import is_blocked +from app.proprietary.platforms.walmart.next_data import extract_next_data +from app.proprietary.platforms.walmart.parsers import ( + parse_listing_page, + parse_product, + parse_reviews_page, +) +from app.proprietary.platforms.walmart.url_resolver import extract_item_id, resolve_url + +_FIXTURES = Path(__file__).parent / "fixtures" + + +def _fixture(name: str) -> str: + return (_FIXTURES / name).read_text(encoding="utf-8") + + +def test_product_parser_extracts_core_fields_and_review_sample(): + data = extract_next_data(_fixture("product.html")) + item = parse_product(data, url="https://www.walmart.com/ip/212092810") + + assert item["usItemId"] == "212092810" + assert item["name"].startswith("Midea") + assert item["price"] == {"value": 149.0, "currency": "USD"} + assert item["listPrice"] == {"value": 199.0, "currency": "USD"} + assert item["stars"] == 4.5 + assert item["reviewsCount"] == 6287 + assert item["inStock"] is True + assert item["seller"] == {"id": "F55CT", "name": "Walmart.com", "type": "WALMART"} + assert item["longDescription"] == "

A powerful window unit.

" + assert item["images"] == [ + "https://i5.walmartimages.com/1.jpg", + "https://i5.walmartimages.com/2.jpg", + ] + assert item["reviewsSample"]["totalReviewCount"] == 6287 + assert item["reviewsSample"]["topReviews"][0]["verifiedPurchase"] is True + + +def test_listing_parser_normalizes_cards_and_marketplace_seller(): + data = extract_next_data(_fixture("listing.html")) + cards = parse_listing_page(data) + + assert [c["usItemId"] for c in cards] == ["791595618", "999001"] + first = cards[0] + assert first["url"] == "https://www.walmart.com/ip/Acer-Chromebook/791595618" + assert first["price"] == {"value": 79.99, "currency": "USD"} + assert first["seller"]["type"] == "MARKETPLACE" + assert first["inStock"] is True + # Fallback price + out-of-stock derivation for the second card. + assert cards[1]["price"] == {"value": 499.0} + assert cards[1]["inStock"] is False + + +def test_reviews_parser_extracts_all_records(): + data = extract_next_data(_fixture("reviews.html")) + reviews = parse_reviews_page(data) + + assert len(reviews) == 2 + assert reviews[0]["reviewId"] == "296013686" + assert reviews[0]["author"] == "JohnPaul" + assert reviews[0]["verifiedPurchase"] is True + assert reviews[0]["images"] == ["https://i5.walmartimages.com/r.jpg"] + assert reviews[0]["sellerResponse"] == "Thanks for the review!" + assert reviews[1]["verifiedPurchase"] is False + + +def test_block_detection_handles_status_and_body(): + assert is_blocked(_fixture("blocked.html"), 200) + assert is_blocked("", 412) + assert is_blocked("", 429) + assert not is_blocked(_fixture("product.html"), 200) + + +def test_missing_next_data_yields_none(): + assert extract_next_data("no script here") is None + assert parse_product(extract_next_data("") or {}, url="x") == {} + + +def test_url_resolver_classifies_and_extracts_ids(): + assert resolve_url("https://www.walmart.com/ip/Foo/123456789").kind == "product" + assert ( + extract_item_id("https://www.walmart.com/reviews/product/212092810") + == "212092810" + ) + assert resolve_url("https://www.walmart.com/search?q=tv").kind == "listing" + assert resolve_url("https://www.walmart.com/cp/tvs/3944").kind == "listing" + assert resolve_url("https://example.com/ip/1") is None