apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-22 23:31:12 +02:00
Code Issues Projects Releases Packages Wiki Activity

fix(walmart): parse category breadcrumb path into breadCrumbs and category

Browse source
This commit is contained in:
CREDO23 2026-07-19 09:10:03 +02:00
parent bfde1b8741
commit f6970b90e8
3 changed files with 20 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

17
surfsense_backend/app/proprietary/platforms/walmart/parsers.py
View file

@ -122,6 +122,19 @@ def _images(image_info: dict[str, Any]) -> list[str]:
return images
def _breadcrumbs(product: dict[str, Any]) -> list[str]:
"""Category breadcrumb names, root→leaf.
Walmart ships ``category.path`` as a list of ``{name, url}`` nodes, e.g.
``[{"name": "Home Improvement"}, ..., {"name": "Air Conditioners"}]`` — not
a string.
"""
path = dig(product, "category", "path")
if not isinstance(path, list):
return []
return [node["name"] for node in path if isinstance(node, dict) and node.get("name")]
def _reviews_sample(
reviews: dict[str, Any] | None, limit: int = 10
) -> dict[str, Any] | None:
@ -151,6 +164,7 @@ def parse_product(
price_info = product.get("priceInfo") or {}
image_info = product.get("imageInfo") or {}
availability = product.get("availabilityStatus")
breadcrumbs = _breadcrumbs(product)
fields: dict[str, Any] = {
"usItemId": str(product.get("usItemId") or product.get("id") or "") or None,
@ -172,7 +186,8 @@ def parse_product(
else None,
"thumbnailImage": image_info.get("thumbnailUrl"),
"images": _images(image_info),
"category": dig(product, "category", "path"),
"category": breadcrumbs[-1] if breadcrumbs else None,
"breadCrumbs": breadcrumbs,
"variants": product.get("variantCriteria") or [],
}
if include_reviews_sample:

2
surfsense_backend/tests/unit/platforms/walmart/fixtures/product.html
View file

@ -1,3 +1,3 @@
<!doctype html><html><head><title>Midea AC</title></head><body>
<script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"initialData":{"data":{"product":{"usItemId":"212092810","id":"ABC123","name":"Midea 5,000 BTU Window Air Conditioner","brand":"Midea","availabilityStatus":"IN_STOCK","averageRating":4.5,"numberOfReviews":6287,"manufacturerName":"Midea","shortDescription":"Cools a small room fast.","sellerName":"Walmart.com","sellerId":"F55CT","priceInfo":{"currentPrice":{"price":149.0,"currencyUnit":"USD"},"wasPrice":{"price":199.0,"currencyUnit":"USD"}},"imageInfo":{"thumbnailUrl":"https://i5.walmartimages.com/t.jpg","allImages":[{"url":"https://i5.walmartimages.com/1.jpg"},{"url":"https://i5.walmartimages.com/2.jpg"}]},"category":{"path":"Home/Air Conditioners"},"variantCriteria":[{"name":"capacity"}]},"idml":{"longDescription":"<p>A powerful window unit.</p>"},"reviews":{"averageOverallRating":4.5,"totalReviewCount":6287,"aspects":[{"name":"Value","score":90}],"customerReviews":[{"reviewId":"r1","rating":5,"reviewTitle":"Great AC","reviewText":"Very cold air.","reviewSubmissionTime":"2024-04-28","userNickname":"Anne","positiveFeedback":3,"negativeFeedback":0,"badges":[{"id":"VerifiedPurchaser"}]}]}}}}}}</script>
<script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"initialData":{"data":{"product":{"usItemId":"212092810","id":"ABC123","name":"Midea 5,000 BTU Window Air Conditioner","brand":"Midea","availabilityStatus":"IN_STOCK","averageRating":4.5,"numberOfReviews":6287,"manufacturerName":"Midea","shortDescription":"Cools a small room fast.","sellerName":"Walmart.com","sellerId":"F55CT","priceInfo":{"currentPrice":{"price":149.0,"currencyUnit":"USD"},"wasPrice":{"price":199.0,"currencyUnit":"USD"}},"imageInfo":{"thumbnailUrl":"https://i5.walmartimages.com/t.jpg","allImages":[{"url":"https://i5.walmartimages.com/1.jpg"},{"url":"https://i5.walmartimages.com/2.jpg"}]},"category":{"categoryPathId":"0:1072864:133026","path":[{"name":"Home","url":"/cp/home/1072864"},{"name":"Air Conditioners","url":"/cp/air-conditioners/133026"}]},"variantCriteria":[{"name":"capacity"}]},"idml":{"longDescription":"<p>A powerful window unit.</p>"},"reviews":{"averageOverallRating":4.5,"totalReviewCount":6287,"aspects":[{"name":"Value","score":90}],"customerReviews":[{"reviewId":"r1","rating":5,"reviewTitle":"Great AC","reviewText":"Very cold air.","reviewSubmissionTime":"2024-04-28","userNickname":"Anne","positiveFeedback":3,"negativeFeedback":0,"badges":[{"id":"VerifiedPurchaser"}]}]}}}}}}</script>
</body></html>

3
surfsense_backend/tests/unit/platforms/walmart/test_parsers.py
View file

@ -35,6 +35,9 @@ def test_product_parser_extracts_core_fields_and_review_sample():
"https://i5.walmartimages.com/1.jpg",
"https://i5.walmartimages.com/2.jpg",
]
# Walmart ships category.path as breadcrumb objects, not a string.
assert item["breadCrumbs"] == ["Home", "Air Conditioners"]
assert item["category"] == "Air Conditioners"
assert item["reviewsSample"]["totalReviewCount"] == 6287
assert item["reviewsSample"]["topReviews"][0]["verifiedPurchase"] is True