From 0d2c8df5bd6b74a2a1c4e9d1ee97dddeae38e39a Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 19 Jul 2026 08:19:21 +0200 Subject: [PATCH 01/21] feat(walmart): add billing units and rate config --- surfsense_backend/app/capabilities/core/billing.py | 4 ++++ surfsense_backend/app/capabilities/core/types.py | 2 ++ surfsense_backend/app/config/__init__.py | 6 ++++++ 3 files changed, 12 insertions(+) diff --git a/surfsense_backend/app/capabilities/core/billing.py b/surfsense_backend/app/capabilities/core/billing.py index c2c7f6b5b..1d08a8bc7 100644 --- a/surfsense_backend/app/capabilities/core/billing.py +++ b/surfsense_backend/app/capabilities/core/billing.py @@ -41,6 +41,8 @@ _PLATFORM_RATE_KEYS: dict[BillingUnit, str] = { BillingUnit.TIKTOK_VIDEO: "TIKTOK_MICROS_PER_VIDEO", BillingUnit.TIKTOK_USER: "TIKTOK_MICROS_PER_USER", BillingUnit.TIKTOK_COMMENT: "TIKTOK_MICROS_PER_COMMENT", + BillingUnit.WALMART_PRODUCT: "WALMART_MICROS_PER_PRODUCT", + BillingUnit.WALMART_REVIEW: "WALMART_MICROS_PER_REVIEW", } @@ -63,6 +65,8 @@ _UNIT_NOUNS: dict[BillingUnit, str] = { BillingUnit.TIKTOK_VIDEO: "video", BillingUnit.TIKTOK_USER: "profile", BillingUnit.TIKTOK_COMMENT: "comment", + BillingUnit.WALMART_PRODUCT: "product", + BillingUnit.WALMART_REVIEW: "review", } diff --git a/surfsense_backend/app/capabilities/core/types.py b/surfsense_backend/app/capabilities/core/types.py index 2e8d3b6cc..ba63a0908 100644 --- a/surfsense_backend/app/capabilities/core/types.py +++ b/surfsense_backend/app/capabilities/core/types.py @@ -31,6 +31,8 @@ class BillingUnit(StrEnum): TIKTOK_VIDEO = "tiktok_video" TIKTOK_USER = "tiktok_user" TIKTOK_COMMENT = "tiktok_comment" + WALMART_PRODUCT = "walmart_product" + WALMART_REVIEW = "walmart_review" class BillableInput(Protocol): diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 20497bb15..d90aa2542 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -736,6 +736,12 @@ class Config: # Comments are the cheapest per-item TikTok data, matching the per-comment # market (and YouTube's comment meter). TIKTOK_MICROS_PER_COMMENT = int(os.getenv("TIKTOK_MICROS_PER_COMMENT", "1500")) + # Walmart products come from server-rendered JSON behind residential proxies, + # priced alongside Amazon's per-product meter. + WALMART_MICROS_PER_PRODUCT = int(os.getenv("WALMART_MICROS_PER_PRODUCT", "3500")) + # Reviews are 10 per page (many light requests per product), priced on the + # cheaper per-review market like the Google Maps review meter. + WALMART_MICROS_PER_REVIEW = int(os.getenv("WALMART_MICROS_PER_REVIEW", "1500")) # Retry an empty listing draw on a fresh rotating IP. Set to 1 for a static # proxy, where every retry re-hits the same exit. TIKTOK_LISTING_MAX_ATTEMPTS = int(os.getenv("TIKTOK_LISTING_MAX_ATTEMPTS", "3")) From a72cc3436cff02a341c1059c96b071a1b06242ec Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 19 Jul 2026 08:19:21 +0200 Subject: [PATCH 02/21] feat(walmart): add product and review schemas --- .../proprietary/platforms/walmart/schemas.py | 188 ++++++++++++++++++ 1 file changed, 188 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/walmart/schemas.py diff --git a/surfsense_backend/app/proprietary/platforms/walmart/schemas.py b/surfsense_backend/app/proprietary/platforms/walmart/schemas.py new file mode 100644 index 000000000..ea25f2eff --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/walmart/schemas.py @@ -0,0 +1,188 @@ +# ruff: noqa: N815 +"""Input/output models for the public Walmart scraper. + +Two verbs share this module: ``walmart.scrape`` (products + listings) and +``walmart.reviews`` (deep paginated reviews). Walmart is a Next.js app that +ships its data as JSON in a ``', re.DOTALL) +_APP_DATA_RE = re.compile(r'', re.DOTALL) + + +def extract_next_data(html: str | None) -> dict[str, Any] | None: + """Return the parsed Next.js state object, or ``None`` when absent/invalid. + + Tries ``__NEXT_DATA__`` first, then the ``__APP_DATA__`` fallback so a single + Walmart layout experiment does not blank the whole extractor. + """ + if not html: + return None + for pattern in (_NEXT_DATA_RE, _APP_DATA_RE): + match = pattern.search(html) + if not match: + continue + try: + data = json.loads(match.group(1)) + except (ValueError, TypeError): + logger.warning("Walmart hidden JSON present but did not parse") + continue + if isinstance(data, dict): + return data + return None + + +def dig(obj: Any, *keys: str | int) -> Any: + """Walk nested dict/list keys, returning ``None`` on any miss. + + Tolerates the layout drift between Walmart's ``initialData`` variants without + a cascade of ``if key in ...`` guards at every call site. + """ + current = obj + for key in keys: + if isinstance(key, int): + if not isinstance(current, list) or not -len(current) <= key < len(current): + return None + current = current[key] + else: + if not isinstance(current, dict) or key not in current: + return None + current = current[key] + return current + + +def initial_data(next_data: dict[str, Any]) -> dict[str, Any] | None: + """The ``props.pageProps.initialData`` node shared by every page type.""" + node = dig(next_data, "props", "pageProps", "initialData") + return node if isinstance(node, dict) else None From 34b6885ba220988140135f96e38d33f5cf1afc26 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 19 Jul 2026 08:19:21 +0200 Subject: [PATCH 05/21] feat(walmart): add fetch with proxy rotation and block detection --- .../proprietary/platforms/walmart/fetch.py | 171 ++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/walmart/fetch.py diff --git a/surfsense_backend/app/proprietary/platforms/walmart/fetch.py b/surfsense_backend/app/proprietary/platforms/walmart/fetch.py new file mode 100644 index 000000000..851b6a674 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/walmart/fetch.py @@ -0,0 +1,171 @@ +"""Network access for public Walmart pages. + +Mirrors the Amazon fetch layer: every request goes through the configured +residential proxy (pinned to the US, since Walmart geo-locks inventory), and any +response that looks like an anti-bot interstitial is retried on a fresh proxy +exit. Ordinary HTTP failures are returned to the caller for domain-specific +handling. + +Walmart runs Akamai (edge/TLS) + PerimeterX/HUMAN (behavioral JS). Two Walmart +specifics differ from Amazon: + +* Walmart serves CAPTCHA with an HTTP ``200`` body ("Robot or human?"), so block + detection scans the body, never the status alone. +* ``412`` is PerimeterX's rejection code and is treated as blocked → rotate. + +``ponytail:`` MVP hits only the server-rendered ``__NEXT_DATA__`` pages, which +TLS impersonation + residential proxies clear without seeding PerimeterX cookies. +If block rates on those pages climb, the upgrade path is a warmed sticky session +(seed ``_px3``/``_pxhd``/``ACID`` from a homepage fetch, reuse exit + cookies) — +the same shape as Amazon's ``get_location_session``. +""" + +from __future__ import annotations + +import asyncio +import logging +import time +from collections.abc import Awaitable, Callable +from dataclasses import dataclass, field +from typing import Any + +from scrapling.fetchers import AsyncFetcher + +from app.utils.proxy import get_geo_proxy_url, get_sticky_proxy_url + +logger = logging.getLogger(__name__) + +_MAX_IP_ATTEMPTS = 6 +_REQUEST_TIMEOUT_S = 30 +_HEADERS = { + "Accept-Language": "en-US,en;q=0.9", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", +} +_BLOCK_MARKERS = ( + "robot or human", + "px-captcha", + "/blocked", + "verify you are a human", + "access to this page has been denied", +) + + +@dataclass(frozen=True) +class FetchResult: + """The response details needed by scraper flows.""" + + status: int + html: str + url: str + cookies: dict[str, str] + headers: dict[str, str] = field(default_factory=dict) + + +async def gather_bounded[T]( + factories: list[Callable[[], Awaitable[T]]], *, concurrency: int +) -> list[T]: + """Run async factories concurrently while preserving input order.""" + if not factories: + return [] + semaphore = asyncio.Semaphore(max(1, concurrency)) + + async def run(factory: Callable[[], Awaitable[T]]) -> T: + async with semaphore: + return await factory() + + return await asyncio.gather(*(run(factory) for factory in factories)) + + +def is_blocked( + html: str | None, status: int, headers: dict[str, str] | None = None +) -> bool: + """Return whether a response is a Walmart anti-bot interstitial. + + ``412`` is PerimeterX's rejection; ``429``/``503`` are throttles. Walmart also + serves CAPTCHA with a ``200`` body, so the body is scanned regardless of + status. + """ + if status in {412, 429, 503}: + return True + text = (html or "")[:200_000].lower() + return any(marker in text for marker in _BLOCK_MARKERS) + + +def _response_url(page: Any, fallback: str) -> str: + value = getattr(page, "url", None) + return str(value) if value else fallback + + +def _response_cookies(page: Any) -> dict[str, str]: + cookies = getattr(page, "cookies", None) + return dict(cookies) if isinstance(cookies, dict) else {} + + +def _response_headers(page: Any) -> dict[str, str]: + headers = getattr(page, "headers", None) + return dict(headers) if isinstance(headers, dict) else {} + + +def _selected_proxy( + proxy: str | None, country: str, attempt: int, url: str +) -> str | None: + if proxy is not None: + return proxy + if attempt > 1: + session_id = f"walmart-{attempt}-{abs(hash((url, time.time_ns()))):x}" + return get_sticky_proxy_url(session_id, country) + return get_geo_proxy_url(country) + + +async def fetch_page( + url: str, + *, + cookies: dict[str, str] | None = None, + proxy: str | None = None, + country: str = "us", + rotate_on_block: bool = True, +) -> FetchResult | None: + """Fetch a page and retry blocked responses with fresh proxy exits.""" + attempts = _MAX_IP_ATTEMPTS if rotate_on_block else 1 + for attempt in range(1, attempts + 1): + selected_proxy = _selected_proxy(proxy, country, attempt, url) + started = time.perf_counter() + try: + page = await AsyncFetcher.get( + url, + headers={**_HEADERS}, + cookies=cookies or {}, + proxy=selected_proxy, + stealthy_headers=True, + timeout=_REQUEST_TIMEOUT_S, + ) + except Exception as exc: + logger.warning("Walmart request failed for %s: %s", url, exc) + if proxy is not None: + return None + continue + + status = int(getattr(page, "status", 0) or 0) + html = getattr(page, "html_content", None) or "" + response_headers = _response_headers(page) + logger.info( + "[walmart][perf] status=%s attempt=%s fetch_ms=%.1f url=%s", + status, + attempt, + (time.perf_counter() - started) * 1000, + url, + ) + if rotate_on_block and is_blocked(html, status, response_headers): + logger.info( + "Walmart blocked proxy attempt %s/%s for %s", attempt, attempts, url + ) + continue + return FetchResult( + status=status, + html=html, + url=_response_url(page, url), + cookies=_response_cookies(page), + headers=response_headers, + ) + logger.warning("Walmart exhausted %s proxy attempts for %s", attempts, url) + return None From 661bb8c1872210bd28bc04a2efad83cad8307717 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 19 Jul 2026 08:19:22 +0200 Subject: [PATCH 06/21] feat(walmart): add JSON parsers --- .../proprietary/platforms/walmart/parsers.py | 231 ++++++++++++++++++ 1 file changed, 231 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/walmart/parsers.py diff --git a/surfsense_backend/app/proprietary/platforms/walmart/parsers.py b/surfsense_backend/app/proprietary/platforms/walmart/parsers.py new file mode 100644 index 000000000..9a0d070a1 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/walmart/parsers.py @@ -0,0 +1,231 @@ +"""Pure parsers for Walmart's hidden ``__NEXT_DATA__`` JSON. + +Product, listing, and review data all live in the same Next.js state tree; these +functions navigate it defensively (via :func:`.next_data.dig`) and normalize the +fields into the stable output shape. Missing sections yield ``None``/``[]`` so an +isolated schema change never discards an otherwise usable record. +""" + +from __future__ import annotations + +from typing import Any +from urllib.parse import urljoin + +from .next_data import dig, initial_data + +_WALMART_ORIGIN = "https://www.walmart.com" + + +# --------------------------------------------------------------------------- # +# Shared helpers. # +# --------------------------------------------------------------------------- # + + +def _price(node: Any) -> dict[str, Any] | None: + """Normalize a Walmart price node into ``{value, currency}``.""" + if not isinstance(node, dict): + return None + value = node.get("price") + currency = node.get("currencyUnit") + if value is None and currency is None: + return None + return {"value": value, "currency": currency} + + +def _seller(raw: dict[str, Any]) -> dict[str, Any] | None: + name = raw.get("sellerName") or raw.get("sellerDisplayName") + seller_id = raw.get("sellerId") + if not name and not seller_id: + return None + is_walmart = bool(name) and "walmart" in name.lower() + return { + "id": seller_id, + "name": name, + "type": "WALMART" if is_walmart else "MARKETPLACE", + } + + +def _absolute(url: str | None) -> str | None: + return urljoin(_WALMART_ORIGIN, url) if url else None + + +# --------------------------------------------------------------------------- # +# Listing / search cards. # +# --------------------------------------------------------------------------- # + + +def _listing_card(raw: dict[str, Any]) -> dict[str, Any] | None: + """Normalize one search/category result item into a product card.""" + item_id = raw.get("usItemId") or raw.get("id") + name = raw.get("name") + if not item_id or not name: + return None + price_info = raw.get("priceInfo") or {} + availability = raw.get("availabilityStatusV2") or {} + image = raw.get("imageInfo") or {} + return { + "usItemId": str(item_id), + "name": name, + "brand": raw.get("brand"), + "url": _absolute(raw.get("canonicalUrl")), + "price": _price(price_info.get("currentPrice")) + or ({"value": raw.get("price")} if raw.get("price") is not None else None), + "listPrice": _price(price_info.get("wasPrice")), + "stars": raw.get("averageRating"), + "reviewsCount": raw.get("numberOfReviews"), + "seller": _seller(raw), + "availabilityStatus": availability.get("value") + or ("OUT_OF_STOCK" if raw.get("isOutOfStock") else None), + "inStock": (availability.get("value") == "IN_STOCK") + if availability.get("value") + else (raw.get("isOutOfStock") is False if "isOutOfStock" in raw else None), + "thumbnailImage": image.get("thumbnailUrl") or raw.get("image"), + "sponsored": raw.get("isSponsoredFlag"), + } + + +def parse_listing_page(next_data: dict[str, Any]) -> list[dict[str, Any]]: + """Extract normalized product cards from a search/category/browse page.""" + data = initial_data(next_data) + if data is None: + return [] + stacks = dig(data, "searchResult", "itemStacks") + if not isinstance(stacks, list): + return [] + cards: list[dict[str, Any]] = [] + seen: set[str] = set() + for stack in stacks: + for raw in (stack or {}).get("items") or []: + if not isinstance(raw, dict) or raw.get("__typename") not in ( + None, + "Product", + ): + continue + card = _listing_card(raw) + if card and card["usItemId"] not in seen: + seen.add(card["usItemId"]) + cards.append(card) + return cards + + +# --------------------------------------------------------------------------- # +# Product detail. # +# --------------------------------------------------------------------------- # + + +def _images(image_info: dict[str, Any]) -> list[str]: + images: list[str] = [] + for entry in image_info.get("allImages") or []: + url = entry.get("url") if isinstance(entry, dict) else None + if url and url not in images: + images.append(url) + return images + + +def _reviews_sample( + reviews: dict[str, Any] | None, limit: int = 10 +) -> dict[str, Any] | None: + if not isinstance(reviews, dict): + return None + customer = reviews.get("customerReviews") or [] + return { + "averageOverallRating": reviews.get("averageOverallRating"), + "totalReviewCount": reviews.get("totalReviewCount"), + "aspects": reviews.get("aspects") or [], + "topReviews": [ + normalize_review(r) for r in customer[:limit] if isinstance(r, dict) + ], + } + + +def parse_product( + next_data: dict[str, Any], *, url: str, include_reviews_sample: bool = True +) -> dict[str, Any]: + """Extract normalized product fields from a product detail page.""" + data = initial_data(next_data) + product = dig(data, "data", "product") if data else None + if not isinstance(product, dict): + return {} + idml = dig(data, "data", "idml") if data else None + reviews = dig(data, "data", "reviews") if data else None + price_info = product.get("priceInfo") or {} + image_info = product.get("imageInfo") or {} + availability = product.get("availabilityStatus") + + fields: dict[str, Any] = { + "usItemId": str(product.get("usItemId") or product.get("id") or "") or None, + "name": product.get("name"), + "brand": product.get("brand"), + "url": url, + "price": _price(price_info.get("currentPrice")), + "listPrice": _price(price_info.get("wasPrice")), + "currency": dig(price_info, "currentPrice", "currencyUnit"), + "availabilityStatus": availability, + "inStock": availability == "IN_STOCK" if availability else None, + "stars": product.get("averageRating"), + "reviewsCount": product.get("numberOfReviews"), + "seller": _seller(product), + "manufacturerName": product.get("manufacturerName"), + "shortDescription": product.get("shortDescription"), + "longDescription": (idml or {}).get("longDescription") + if isinstance(idml, dict) + else None, + "thumbnailImage": image_info.get("thumbnailUrl"), + "images": _images(image_info), + "category": dig(product, "category", "path"), + "variants": product.get("variantCriteria") or [], + } + if include_reviews_sample: + fields["reviewsSample"] = _reviews_sample(reviews) + return {key: value for key, value in fields.items() if value is not None} + + +# --------------------------------------------------------------------------- # +# Reviews (deep pagination). # +# --------------------------------------------------------------------------- # + + +def normalize_review(raw: dict[str, Any]) -> dict[str, Any]: + """Normalize one Walmart ``customerReviews`` record into a ``ReviewItem`` dict.""" + badges = raw.get("badges") or [] + verified = any( + isinstance(b, dict) and b.get("id") == "VerifiedPurchaser" for b in badges + ) + photos = raw.get("photos") or raw.get("media") or [] + images: list[str] = [] + for photo in photos: + if not isinstance(photo, dict): + continue + url = photo.get("normalUrl") or dig(photo, "sizes", "normal", "url") + if url and url not in images: + images.append(url) + responses = raw.get("clientResponses") or [] + seller_response = None + if responses and isinstance(responses[0], dict): + seller_response = responses[0].get("response") or responses[0].get( + "responseText" + ) + return { + "reviewId": raw.get("reviewId"), + "rating": raw.get("rating"), + "title": raw.get("reviewTitle"), + "text": raw.get("reviewText"), + "submissionTime": raw.get("reviewSubmissionTime"), + "author": raw.get("userNickname"), + "verifiedPurchase": verified, + "positiveFeedback": raw.get("positiveFeedback"), + "negativeFeedback": raw.get("negativeFeedback"), + "images": images, + "syndicated": bool(raw.get("syndicationSource")), + "sellerResponse": seller_response, + } + + +def parse_reviews_page(next_data: dict[str, Any]) -> list[dict[str, Any]]: + """Extract normalized reviews from one ``/reviews/product/{id}`` page.""" + data = initial_data(next_data) + reviews = dig(data, "data", "reviews") if data else None + customer = reviews.get("customerReviews") if isinstance(reviews, dict) else None + if not isinstance(customer, list): + return [] + return [normalize_review(r) for r in customer if isinstance(r, dict)] From d446aa32ec083c0b481837f9eb0fbdc55b5880ea Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 19 Jul 2026 08:19:22 +0200 Subject: [PATCH 07/21] feat(walmart): add scrape and review orchestration --- .../proprietary/platforms/walmart/README.md | 52 ++++ .../proprietary/platforms/walmart/__init__.py | 27 ++ .../proprietary/platforms/walmart/scraper.py | 288 ++++++++++++++++++ 3 files changed, 367 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/walmart/README.md create mode 100644 surfsense_backend/app/proprietary/platforms/walmart/__init__.py create mode 100644 surfsense_backend/app/proprietary/platforms/walmart/scraper.py diff --git a/surfsense_backend/app/proprietary/platforms/walmart/README.md b/surfsense_backend/app/proprietary/platforms/walmart/README.md new file mode 100644 index 000000000..982ee5400 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/walmart/README.md @@ -0,0 +1,52 @@ +# Walmart Scraper + +Two verbs read Walmart's public, anonymous pages: `walmart.scrape` (products + +search/category/browse listings, per-product billing) and `walmart.reviews` +(deep paginated reviews, per-review billing). + +## Scope + +The scraper reads public pages available to anonymous visitors — no login, no +account cookies. Data is extracted from the Next.js `__NEXT_DATA__` JSON blob +embedded in each page (with an `__APP_DATA__` fallback), not from the rendered +DOM, because Walmart obfuscates CSS classes and A/B-tests layout constantly. + +`walmart.scrape` returns a free sample of on-page reviews (rating distribution, +aspects, top reviews) under `reviewsSample`. `walmart.reviews` fetches the full +review history from the public `/reviews/product/{usItemId}` page, which +robots.txt permits (unlike `/search`). + +## Architecture + +- `schemas.py` defines the stable input, product, review, and error models. +- `url_resolver.py` classifies product (`/ip/`) vs listing (`/search`, `/cp/`, + `/browse/`) URLs and extracts the numeric `usItemId`. +- `next_data.py` extracts and navigates the hidden Next.js JSON state. +- `fetch.py` owns proxy-aware HTTP access (US-pinned), block detection, and + retries. +- `parsers.py` contains pure, defensive JSON parsers. +- `scraper.py` coordinates discovery, enrichment, pagination, concurrency, + limits, and in-stream error items. + +## Anti-bot + +Walmart runs Akamai (edge/TLS) + PerimeterX/HUMAN (behavioral JS). Requests go +through US residential proxies with TLS-impersonated headers; blocked responses +(body markers, `412`/`429`/`503`, or the `200`-OK CAPTCHA body) rotate to a +fresh proxy exit. + +Known ceilings and upgrade paths (see `fetch.py` / `scraper.py` `ponytail:` +notes): reviews page at 10/page; search capped at Walmart's 25-page limit; +session warming (seed `_px3`/`_pxhd` on a sticky exit) is the next lever if +block rates on the SSR pages climb, and the `/orchestra/*` GraphQL API is +deliberately avoided (rotating persisted-query hashes make it brittle). + +## Verification + +Offline fixtures cover the parsers and both flows: + + uv run pytest tests/unit/platforms/walmart/ + +A manual live check (requires network + residential proxy): + + uv run python scripts/e2e_walmart_scraper.py diff --git a/surfsense_backend/app/proprietary/platforms/walmart/__init__.py b/surfsense_backend/app/proprietary/platforms/walmart/__init__.py new file mode 100644 index 000000000..83b7cf277 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/walmart/__init__.py @@ -0,0 +1,27 @@ +"""Platform-native Walmart scraper (products, listings, reviews).""" + +from .schemas import ( + ErrorItem, + ProductItem, + ReviewItem, + WalmartReviewsInput, + WalmartScrapeInput, +) +from .scraper import ( + iter_products, + iter_reviews, + scrape_products, + scrape_reviews, +) + +__all__ = [ + "ErrorItem", + "ProductItem", + "ReviewItem", + "WalmartReviewsInput", + "WalmartScrapeInput", + "iter_products", + "iter_reviews", + "scrape_products", + "scrape_reviews", +] diff --git a/surfsense_backend/app/proprietary/platforms/walmart/scraper.py b/surfsense_backend/app/proprietary/platforms/walmart/scraper.py new file mode 100644 index 000000000..e883efdbc --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/walmart/scraper.py @@ -0,0 +1,288 @@ +"""Orchestrate public Walmart product discovery, enrichment, and reviews. + +Two streaming cores: :func:`iter_products` dispatches each start URL to a product +or listing flow, and :func:`iter_reviews` paginates the public reviews page per +item id. Network and parsing concerns stay isolated in their own modules so +markup changes and retry policy can be tested independently. + +The failure model is in-stream error items (dicts with an ``error`` key), never +exceptions — identical to the Amazon scraper. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from typing import Any +from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse + +from .fetch import fetch_page, gather_bounded +from .next_data import extract_next_data +from .parsers import parse_listing_page, parse_product, parse_reviews_page +from .schemas import ( + ErrorItem, + ProductItem, + ReviewItem, + WalmartReviewsInput, + WalmartScrapeInput, +) +from .url_resolver import ResolvedUrl, extract_item_id, resolve_url + +__all__ = ["iter_products", "iter_reviews", "scrape_products", "scrape_reviews"] + +_DETAIL_CONCURRENCY = 6 +_SEARCH_PAGE_LIMIT = 25 # Walmart caps result pagination at 25 pages. +_REVIEWS_PAGE_LIMIT = 500 # 10 reviews/page → 5000-review safety ceiling. +_DEFAULT_ITEMS_PER_START_URL = 40 + +# Friendly sort → Walmart's ``sort`` query value on the reviews page. +_REVIEW_SORT = { + "most-recent": "submission-desc", + "most-helpful": "helpful", + "rating-high": "rating-desc", + "rating-low": "rating-asc", +} + + +def _error( + code: str, description: str, *, input_url: str | None, url: str | None = None +) -> dict[str, Any]: + return ErrorItem( + error=code, + errorDescription=description, + input=input_url, + url=url or input_url, + ).model_dump() + + +def _page_url(url: str, page: int) -> str: + parsed = urlparse(url) + query = dict(parse_qsl(parsed.query, keep_blank_values=True)) + query["page"] = str(page) + return urlunparse(parsed._replace(query=urlencode(query))) + + +def _product_url(item_id: str) -> str: + return f"https://www.walmart.com/ip/{item_id}" + + +def _reviews_url(item_id: str, page: int, sort: str) -> str: + query = urlencode( + {"page": page, "sort": sort, "entryPoint": "viewAllReviewsBottom"} + ) + return f"https://www.walmart.com/reviews/product/{item_id}?{query}" + + +# --------------------------------------------------------------------------- # +# Product / listing flows. # +# --------------------------------------------------------------------------- # + + +async def _product_flow( + resolved: ResolvedUrl, input_model: WalmartScrapeInput +) -> AsyncIterator[dict[str, Any]]: + """Fetch and parse one product detail page.""" + url = _product_url(resolved.item_id) if resolved.item_id else resolved.url + response = await fetch_page(url, country=input_model.country) + if response is None: + yield _error( + "product_not_found", + "The product page could not be loaded after retrying proxy exits.", + input_url=resolved.url, + ) + return + if response.status in {404, 410}: + yield _error( + "product_not_found", + "The product page was not found.", + input_url=resolved.url, + ) + return + + next_data = extract_next_data(response.html) + fields = ( + parse_product( + next_data, + url=response.url, + include_reviews_sample=input_model.includeReviewsSample, + ) + if next_data + else {} + ) + if not fields.get("name"): + yield _error( + "product_not_found", + "The response did not contain a recognizable product.", + input_url=resolved.url, + url=response.url, + ) + return + fields["input"] = resolved.url + yield ProductItem(**fields).to_output() + + +async def _listing_flow( + resolved: ResolvedUrl, input_model: WalmartScrapeInput +) -> AsyncIterator[dict[str, Any]]: + """Page through a search/category/browse listing and optionally enrich.""" + cap = ( + input_model.maxItemsPerStartUrl + if input_model.maxItemsPerStartUrl is not None + else _DEFAULT_ITEMS_PER_START_URL + ) + max_pages = min(input_model.maxSearchPagesPerStartUrl, _SEARCH_PAGE_LIMIT) + seen: set[str] = set() + emitted = 0 + for page in range(1, max_pages + 1): + response = await fetch_page( + _page_url(resolved.url, page), country=input_model.country + ) + next_data = extract_next_data(response.html) if response else None + cards = parse_listing_page(next_data) if next_data else [] + cards = [c for c in cards if c["usItemId"] not in seen] + for card in cards: + seen.add(card["usItemId"]) + if not cards: + if page == 1: + yield _error( + "no_results_found", + "The listing page did not contain any products.", + input_url=resolved.url, + ) + return + cards = cards[: max(0, cap - emitted)] + + if not input_model.includeDetails: + for card in cards: + card["input"] = resolved.url + yield ProductItem(**card).to_output() + emitted += 1 + else: + + async def load_card(card: dict[str, Any]) -> list[dict[str, Any]]: + product = resolve_url(card["url"]) if card.get("url") else None + if product is None or product.item_id is None: + card["input"] = resolved.url + return [ProductItem(**card).to_output()] + return [item async for item in _product_flow(product, input_model)] + + for batch in await gather_bounded( + [lambda card=card: load_card(card) for card in cards], + concurrency=_DETAIL_CONCURRENCY, + ): + for item in batch: + if "error" not in item and emitted >= cap: + return + yield item + if "error" not in item: + emitted += 1 + if emitted >= cap: + return + + +_FLOWS = {"product": _product_flow, "listing": _listing_flow} + + +async def iter_products( + input_model: WalmartScrapeInput, +) -> AsyncIterator[dict[str, Any]]: + """Yield product items for every start URL. + + Each URL is classified and dispatched to its per-kind flow. An unrecognized + URL yields an ``invalid_url`` error item. + """ + for url in input_model.startUrls: + resolved = resolve_url(url) + if resolved is None: + yield _error( + "invalid_url", + "Start URL was malformed or not a recognized Walmart URL.", + input_url=url, + ) + continue + async for item in _FLOWS[resolved.kind](resolved, input_model): + yield item + + +async def scrape_products( + input_model: WalmartScrapeInput, *, limit: int | None = None +) -> list[dict[str, Any]]: + """Collect :func:`iter_products` into a list, honoring an optional ``limit``.""" + from app.capabilities.core.progress import emit_progress + + results: list[dict[str, Any]] = [] + async for item in iter_products(input_model): + results.append(item) + emit_progress("scraping", current=len(results), total=limit, unit="product") + if limit is not None and len(results) >= limit: + break + return results + + +# --------------------------------------------------------------------------- # +# Reviews flow (deep pagination). # +# --------------------------------------------------------------------------- # + + +async def _reviews_flow( + item_id: str, input_model: WalmartReviewsInput +) -> AsyncIterator[dict[str, Any]]: + """Paginate the public reviews page until empty or ``maxReviews`` reached. + + Walmart does not expose a reliable total up front, so the loop stops on the + first empty page rather than trusting a page count. + """ + sort = _REVIEW_SORT[input_model.sort] + emitted = 0 + for page in range(1, _REVIEWS_PAGE_LIMIT + 1): + response = await fetch_page( + _reviews_url(item_id, page, sort), country=input_model.country + ) + next_data = extract_next_data(response.html) if response else None + reviews = parse_reviews_page(next_data) if next_data else [] + if not reviews: + if page == 1 and emitted == 0: + yield _error( + "reviews_not_found", + "The product has no public reviews or could not be loaded.", + input_url=item_id, + ) + return + for raw in reviews: + raw["usItemId"] = item_id + raw["input"] = item_id + yield ReviewItem(**raw).to_output() + emitted += 1 + if emitted >= input_model.maxReviews: + return + + +async def iter_reviews( + input_model: WalmartReviewsInput, +) -> AsyncIterator[dict[str, Any]]: + """Yield review items for every requested item id (or resolvable URL).""" + for raw_id in input_model.itemIds: + item_id = extract_item_id(raw_id) + if item_id is None: + yield _error( + "invalid_url", + "Could not extract a Walmart item id from the input.", + input_url=raw_id, + ) + continue + async for item in _reviews_flow(item_id, input_model): + yield item + + +async def scrape_reviews( + input_model: WalmartReviewsInput, *, limit: int | None = None +) -> list[dict[str, Any]]: + """Collect :func:`iter_reviews` into a list, honoring an optional ``limit``.""" + from app.capabilities.core.progress import emit_progress + + results: list[dict[str, Any]] = [] + async for item in iter_reviews(input_model): + results.append(item) + emit_progress("scraping", current=len(results), total=limit, unit="review") + if limit is not None and len(results) >= limit: + break + return results From 9b9091ae97cf9385510e7e43671547f54137ac5d Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 19 Jul 2026 08:19:22 +0200 Subject: [PATCH 08/21] feat(walmart): add scrape and reviews capabilities --- .../app/capabilities/walmart/__init__.py | 6 ++ .../capabilities/walmart/reviews/__init__.py | 3 + .../walmart/reviews/definition.py | 24 +++++++ .../capabilities/walmart/reviews/executor.py | 40 +++++++++++ .../capabilities/walmart/reviews/schemas.py | 72 +++++++++++++++++++ .../capabilities/walmart/scrape/__init__.py | 3 + .../capabilities/walmart/scrape/definition.py | 22 ++++++ .../capabilities/walmart/scrape/executor.py | 46 ++++++++++++ .../capabilities/walmart/scrape/schemas.py | 64 +++++++++++++++++ 9 files changed, 280 insertions(+) create mode 100644 surfsense_backend/app/capabilities/walmart/__init__.py create mode 100644 surfsense_backend/app/capabilities/walmart/reviews/__init__.py create mode 100644 surfsense_backend/app/capabilities/walmart/reviews/definition.py create mode 100644 surfsense_backend/app/capabilities/walmart/reviews/executor.py create mode 100644 surfsense_backend/app/capabilities/walmart/reviews/schemas.py create mode 100644 surfsense_backend/app/capabilities/walmart/scrape/__init__.py create mode 100644 surfsense_backend/app/capabilities/walmart/scrape/definition.py create mode 100644 surfsense_backend/app/capabilities/walmart/scrape/executor.py create mode 100644 surfsense_backend/app/capabilities/walmart/scrape/schemas.py diff --git a/surfsense_backend/app/capabilities/walmart/__init__.py b/surfsense_backend/app/capabilities/walmart/__init__.py new file mode 100644 index 000000000..1031f4230 --- /dev/null +++ b/surfsense_backend/app/capabilities/walmart/__init__.py @@ -0,0 +1,6 @@ +"""``walmart.*`` namespace: platform-native Walmart data verbs.""" + +from __future__ import annotations + +from app.capabilities.walmart.reviews import definition as _reviews # noqa: F401 +from app.capabilities.walmart.scrape import definition as _scrape # noqa: F401 diff --git a/surfsense_backend/app/capabilities/walmart/reviews/__init__.py b/surfsense_backend/app/capabilities/walmart/reviews/__init__.py new file mode 100644 index 000000000..473d60402 --- /dev/null +++ b/surfsense_backend/app/capabilities/walmart/reviews/__init__.py @@ -0,0 +1,3 @@ +"""Walmart deep review scraping capability.""" + +from __future__ import annotations diff --git a/surfsense_backend/app/capabilities/walmart/reviews/definition.py b/surfsense_backend/app/capabilities/walmart/reviews/definition.py new file mode 100644 index 000000000..6984ee031 --- /dev/null +++ b/surfsense_backend/app/capabilities/walmart/reviews/definition.py @@ -0,0 +1,24 @@ +"""``walmart.reviews`` capability registration (billed per review; see config +``WALMART_MICROS_PER_REVIEW``).""" + +from __future__ import annotations + +from app.capabilities.core import BillingUnit, Capability, register_capability +from app.capabilities.walmart.reviews.executor import build_reviews_executor +from app.capabilities.walmart.reviews.schemas import ReviewsInput, ReviewsOutput + +WALMART_REVIEWS = Capability( + name="walmart.reviews", + description=( + "Fetch deep paginated public Walmart product reviews with ratings, text, " + "authors, verified-purchase flags, images, and seller responses. Use " + "product urls or item ids." + ), + input_schema=ReviewsInput, + output_schema=ReviewsOutput, + executor=build_reviews_executor(), + billing_unit=BillingUnit.WALMART_REVIEW, + docs_url="/docs/connectors/native/walmart", +) + +register_capability(WALMART_REVIEWS) diff --git a/surfsense_backend/app/capabilities/walmart/reviews/executor.py b/surfsense_backend/app/capabilities/walmart/reviews/executor.py new file mode 100644 index 000000000..35f88e11a --- /dev/null +++ b/surfsense_backend/app/capabilities/walmart/reviews/executor.py @@ -0,0 +1,40 @@ +"""``walmart.reviews`` executor: verb input → scraper → review items.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable + +from app.capabilities.core import Executor +from app.capabilities.core.progress import emit_progress +from app.capabilities.walmart.reviews.schemas import ReviewsInput, ReviewsOutput +from app.proprietary.platforms.walmart import WalmartReviewsInput, scrape_reviews + +ReviewsFn = Callable[..., Awaitable[list[dict]]] + + +def build_reviews_executor(scrape_fn: ReviewsFn | None = None) -> Executor: + """Bind the executor to a reviews scraper fn (defaults to the proprietary actor).""" + scrape_fn = scrape_fn or scrape_reviews + + async def execute(payload: ReviewsInput) -> ReviewsOutput: + input_model = WalmartReviewsInput( + itemIds=payload.sources(), + maxReviews=payload.max_reviews, + sort=payload.sort_by, + ) + emit_progress( + "starting", + "Fetching Walmart reviews", + total=payload.estimated_units, + unit="review", + ) + items = await scrape_fn(input_model, limit=payload.estimated_units) + emit_progress( + "done", + f"Scraped {sum('error' not in item for item in items)} review(s)", + current=len(items), + unit="review", + ) + return ReviewsOutput(items=items) + + return execute diff --git a/surfsense_backend/app/capabilities/walmart/reviews/schemas.py b/surfsense_backend/app/capabilities/walmart/reviews/schemas.py new file mode 100644 index 000000000..faf56ed5b --- /dev/null +++ b/surfsense_backend/app/capabilities/walmart/reviews/schemas.py @@ -0,0 +1,72 @@ +"""``walmart.reviews`` I/O contracts. + +A lean surface over ``WalmartReviewsInput``; the scraper's ``ReviewItem`` is +reused verbatim as the output element. Accepts product URLs or bare item ids — +both resolve to a ``usItemId`` the reviews page is keyed on. +""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, Field, model_validator + +from app.proprietary.platforms.walmart import ReviewItem + +MAX_WALMART_REVIEW_SOURCES = 20 + + +class ReviewsInput(BaseModel): + urls: list[str] = Field( + default_factory=list, + max_length=MAX_WALMART_REVIEW_SOURCES, + description=( + "Walmart product URLs (/ip/...) or reviews URLs to fetch reviews for. " + "Provide these OR item_ids (at least one is required)." + ), + ) + item_ids: list[str] = Field( + default_factory=list, + max_length=MAX_WALMART_REVIEW_SOURCES, + description="Walmart numeric item ids (usItemId) to fetch reviews for.", + ) + max_reviews: int = Field( + default=200, + ge=1, + le=5000, + description="Max reviews to return per product (10 per page).", + ) + sort_by: Literal["most-recent", "most-helpful", "rating-high", "rating-low"] = ( + Field(default="most-recent", description="Review ordering.") + ) + + @model_validator(mode="after") + def _require_a_source(self) -> ReviewsInput: + if not (self.urls or self.item_ids): + raise ValueError("Provide at least one of 'urls' or 'item_ids'.") + return self + + def sources(self) -> list[str]: + """URLs and item ids merged; the scraper resolves each to a usItemId.""" + return [*self.urls, *self.item_ids] + + @property + def estimated_units(self) -> int: + """Worst-case billable reviews: up to ``max_reviews`` per source.""" + return (len(self.urls) + len(self.item_ids)) * self.max_reviews + + +class ReviewsOutput(BaseModel): + items: list[ReviewItem] = Field( + default_factory=list, + description="One item per review, in the scraper's emission order.", + ) + + @property + def billable_units(self) -> int: + """One returned review = one billable unit; error items are not billed.""" + return sum( + 1 + for item in self.items + if not (item.model_extra and item.model_extra.get("error")) + ) diff --git a/surfsense_backend/app/capabilities/walmart/scrape/__init__.py b/surfsense_backend/app/capabilities/walmart/scrape/__init__.py new file mode 100644 index 000000000..bb3e8f794 --- /dev/null +++ b/surfsense_backend/app/capabilities/walmart/scrape/__init__.py @@ -0,0 +1,3 @@ +"""Walmart product scraping capability.""" + +from __future__ import annotations diff --git a/surfsense_backend/app/capabilities/walmart/scrape/definition.py b/surfsense_backend/app/capabilities/walmart/scrape/definition.py new file mode 100644 index 000000000..6165a8992 --- /dev/null +++ b/surfsense_backend/app/capabilities/walmart/scrape/definition.py @@ -0,0 +1,22 @@ +"""Registration for the ``walmart.scrape`` capability.""" + +from __future__ import annotations + +from app.capabilities.core import BillingUnit, Capability, register_capability +from app.capabilities.walmart.scrape.executor import build_scrape_executor +from app.capabilities.walmart.scrape.schemas import ScrapeInput, ScrapeOutput + +WALMART_SCRAPE = Capability( + name="walmart.scrape", + description=( + "Scrape public Walmart product details, search/category listings, " + "prices, sellers, variants, availability, and a sample of on-page reviews." + ), + input_schema=ScrapeInput, + output_schema=ScrapeOutput, + executor=build_scrape_executor(), + billing_unit=BillingUnit.WALMART_PRODUCT, + docs_url="/docs/connectors/native/walmart", +) + +register_capability(WALMART_SCRAPE) diff --git a/surfsense_backend/app/capabilities/walmart/scrape/executor.py b/surfsense_backend/app/capabilities/walmart/scrape/executor.py new file mode 100644 index 000000000..4c4b073ee --- /dev/null +++ b/surfsense_backend/app/capabilities/walmart/scrape/executor.py @@ -0,0 +1,46 @@ +"""Executor for the ``walmart.scrape`` capability.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable + +from app.capabilities.core import Executor +from app.capabilities.core.progress import emit_progress +from app.capabilities.walmart.scrape.schemas import ( + MAX_WALMART_RESULTS, + ScrapeInput, + ScrapeOutput, +) +from app.proprietary.platforms.walmart import WalmartScrapeInput, scrape_products + +ScrapeFn = Callable[..., Awaitable[list[dict]]] + + +def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor: + """Bind the capability input mapping to a replaceable scraper function.""" + scrape_fn = scrape_fn or scrape_products + + async def execute(payload: ScrapeInput) -> ScrapeOutput: + input_model = WalmartScrapeInput( + startUrls=payload.start_urls(), + maxItemsPerStartUrl=payload.max_items, + includeDetails=payload.include_details, + includeReviewsSample=payload.include_reviews_sample, + ) + emit_progress( + "starting", + "Scraping Walmart products", + total=payload.estimated_units, + unit="product", + ) + items = await scrape_fn(input_model, limit=MAX_WALMART_RESULTS) + emit_progress( + "done", + f"Scraped {sum('error' not in item for item in items)} product(s)", + current=len(items), + total=payload.estimated_units, + unit="product", + ) + return ScrapeOutput(items=items) + + return execute diff --git a/surfsense_backend/app/capabilities/walmart/scrape/schemas.py b/surfsense_backend/app/capabilities/walmart/scrape/schemas.py new file mode 100644 index 000000000..2d42dc371 --- /dev/null +++ b/surfsense_backend/app/capabilities/walmart/scrape/schemas.py @@ -0,0 +1,64 @@ +"""Input and output contracts for ``walmart.scrape``.""" + +from __future__ import annotations + +from urllib.parse import quote_plus + +from pydantic import BaseModel, Field, model_validator + +from app.proprietary.platforms.walmart import ProductItem + +MAX_WALMART_SOURCES = 20 +MAX_WALMART_RESULTS = 1000 + + +class ScrapeInput(BaseModel): + """Agent-facing controls for public Walmart product discovery and enrichment.""" + + urls: list[str] = Field(default_factory=list, max_length=MAX_WALMART_SOURCES) + search_terms: list[str] = Field( + default_factory=list, max_length=MAX_WALMART_SOURCES + ) + max_items: int = Field(default=10, ge=1, le=100) + include_details: bool = True + include_reviews_sample: bool = True + + @model_validator(mode="after") + def _require_source(self) -> ScrapeInput: + if not (self.urls or self.search_terms): + raise ValueError("Provide at least one URL or search term.") + if len(self.urls) + len(self.search_terms) > MAX_WALMART_SOURCES: + raise ValueError( + f"Provide no more than {MAX_WALMART_SOURCES} combined sources." + ) + return self + + def start_urls(self) -> list[str]: + """Direct URLs plus a search URL synthesized per search term.""" + searches = [ + f"https://www.walmart.com/search?q={quote_plus(term)}" + for term in self.search_terms + ] + return [*self.urls, *searches] + + @property + def estimated_units(self) -> int: + """Worst-case returned products within the hard per-run ceiling.""" + search_products = len(self.search_terms) * self.max_items + direct_products = len(self.urls) * self.max_items + return min(search_products + direct_products, MAX_WALMART_RESULTS) + + +class ScrapeOutput(BaseModel): + """Products and structured per-input errors in emission order.""" + + items: list[ProductItem] = Field(default_factory=list) + + @property + def billable_units(self) -> int: + """Count successful products; error items are never billed.""" + return sum( + 1 + for item in self.items + if not (item.model_extra and item.model_extra.get("error")) + ) From a3f8a7e7413e1405ae793a890c7f07ec27056b67 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 19 Jul 2026 08:19:22 +0200 Subject: [PATCH 09/21] feat(walmart): register routes and MCP tools --- surfsense_backend/app/routes/__init__.py | 1 + .../mcp_server/features/scrapers/__init__.py | 7 +- .../features/scrapers/platforms/walmart.py | 140 ++++++++++++++++++ 3 files changed, 146 insertions(+), 2 deletions(-) create mode 100644 surfsense_mcp/mcp_server/features/scrapers/platforms/walmart.py diff --git a/surfsense_backend/app/routes/__init__.py b/surfsense_backend/app/routes/__init__.py index 509233e15..e76d3ea1d 100644 --- a/surfsense_backend/app/routes/__init__.py +++ b/surfsense_backend/app/routes/__init__.py @@ -7,6 +7,7 @@ import app.capabilities.google_search import app.capabilities.instagram import app.capabilities.reddit import app.capabilities.tiktok +import app.capabilities.walmart import app.capabilities.web import app.capabilities.youtube # noqa: F401 from app.automations.api import router as automations_router diff --git a/surfsense_mcp/mcp_server/features/scrapers/__init__.py b/surfsense_mcp/mcp_server/features/scrapers/__init__.py index e6f1055cd..524250060 100644 --- a/surfsense_mcp/mcp_server/features/scrapers/__init__.py +++ b/surfsense_mcp/mcp_server/features/scrapers/__init__.py @@ -1,7 +1,8 @@ """Scraper tools: one MCP surface per SurfSense platform capability. -Web crawl, Google Search, Reddit, YouTube, and Google Maps each get a tool that -maps a natural-language request to the workspace's scraper. Two run-history tools +Web crawl, Google Search, Reddit, YouTube, Google Maps, Amazon, and Walmart each +get a tool that maps a natural-language request to the workspace's scraper. Two +run-history tools list and fetch past runs, so a large result truncated inline can be retrieved in full later. Each platform lives in its own module under platforms/. """ @@ -20,6 +21,7 @@ from .platforms import ( instagram, reddit, tiktok, + walmart, web, youtube, ) @@ -33,6 +35,7 @@ _REGISTRARS = ( tiktok, google_maps, amazon, + walmart, run_history, ) diff --git a/surfsense_mcp/mcp_server/features/scrapers/platforms/walmart.py b/surfsense_mcp/mcp_server/features/scrapers/platforms/walmart.py new file mode 100644 index 000000000..0c11af653 --- /dev/null +++ b/surfsense_mcp/mcp_server/features/scrapers/platforms/walmart.py @@ -0,0 +1,140 @@ +"""Walmart scraper tools: products/listings and deep reviews.""" + +from __future__ import annotations + +from typing import Annotated, Literal + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from ....core.client import SurfSenseClient +from ....core.rendering import ResponseFormatParam +from ....core.workspace_context import WorkspaceContext, WorkspaceParam +from ..annotations import SCRAPE +from ..capability import run_scraper + +ReviewSort = Literal["most-recent", "most-helpful", "rating-high", "rating-low"] + + +def register(mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext) -> None: + """Register the Walmart product and review tools.""" + + @mcp.tool( + name="surfsense_walmart_scrape", + title="Scrape Walmart products", + annotations=SCRAPE, + structured_output=False, + ) + async def walmart_scrape( + urls: Annotated[ + list[str] | None, + Field( + description="Walmart product (/ip/), search (/search), category " + "(/cp/), or browse (/browse/) URLs. Provide urls OR search_terms." + ), + ] = None, + search_terms: Annotated[ + list[str] | None, + Field( + description="Search phrases run on walmart.com, e.g. ['air fryer']. " + "Provide search_terms OR urls." + ), + ] = None, + max_items: Annotated[ + int, + Field(ge=1, le=100, description="Max products per search term or listing URL."), + ] = 10, + include_details: Annotated[ + bool, + Field( + description="Fetch full product detail pages. False returns faster " + "card-only results from listings." + ), + ] = True, + include_reviews_sample: Annotated[ + bool, + Field( + description="Include the free on-page review sample (rating " + "distribution, aspects, top reviews) on detail pages." + ), + ] = True, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Scrape public Walmart product data by URL or search term. + + Use this for product research: title, price, list price, rating and + review count, availability, seller (Walmart 1P vs marketplace), images, + description, variants, and a sample of on-page reviews. Only public, + anonymous data — no login. For a product's full review history use + surfsense_walmart_reviews instead. + Example: search_terms=['air fryer'], max_items=5. + """ + return await run_scraper( + client, + context, + platform="walmart", + verb="scrape", + payload={ + "urls": urls, + "search_terms": search_terms, + "max_items": max_items, + "include_details": include_details, + "include_reviews_sample": include_reviews_sample, + }, + workspace=workspace, + response_format=response_format, + ) + + @mcp.tool( + name="surfsense_walmart_reviews", + title="Fetch Walmart reviews", + annotations=SCRAPE, + structured_output=False, + ) + async def walmart_reviews( + urls: Annotated[ + list[str] | None, + Field( + description="Walmart product URLs (/ip/...). Provide urls OR item_ids." + ), + ] = None, + item_ids: Annotated[ + list[str] | None, + Field( + description="Walmart numeric item ids (usItemId) from " + "surfsense_walmart_scrape." + ), + ] = None, + max_reviews: Annotated[ + int, + Field(ge=1, le=5000, description="Max reviews per product (10 per page)."), + ] = 200, + sort_by: Annotated[ + ReviewSort, Field(description="Review ordering.") + ] = "most-recent", + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Fetch deep paginated customer reviews for Walmart products. + + Use this to read the full review history on specific products; get urls + or item_ids from surfsense_walmart_scrape first if you only have a name. + Returns rating, title, text, author, verified-purchase flag, images, and + seller response per review. + Example: item_ids=['212092810'], sort_by='most-helpful', max_reviews=100. + """ + return await run_scraper( + client, + context, + platform="walmart", + verb="reviews", + payload={ + "urls": urls, + "item_ids": item_ids, + "max_reviews": max_reviews, + "sort_by": sort_by, + }, + workspace=workspace, + response_format=response_format, + ) From 7b5e09c528c2aebbe1ba861624ca48bd00c8e648 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 19 Jul 2026 08:19:22 +0200 Subject: [PATCH 10/21] test(walmart): add parser and flow tests with e2e harness --- .../scripts/e2e_walmart_scraper.py | 80 ++++++++++ .../tests/unit/platforms/walmart/__init__.py | 0 .../platforms/walmart/fixtures/blocked.html | 4 + .../platforms/walmart/fixtures/listing.html | 3 + .../platforms/walmart/fixtures/product.html | 3 + .../platforms/walmart/fixtures/reviews.html | 3 + .../unit/platforms/walmart/test_flows.py | 138 ++++++++++++++++++ .../unit/platforms/walmart/test_parsers.py | 90 ++++++++++++ 8 files changed, 321 insertions(+) create mode 100644 surfsense_backend/scripts/e2e_walmart_scraper.py create mode 100644 surfsense_backend/tests/unit/platforms/walmart/__init__.py create mode 100644 surfsense_backend/tests/unit/platforms/walmart/fixtures/blocked.html create mode 100644 surfsense_backend/tests/unit/platforms/walmart/fixtures/listing.html create mode 100644 surfsense_backend/tests/unit/platforms/walmart/fixtures/product.html create mode 100644 surfsense_backend/tests/unit/platforms/walmart/fixtures/reviews.html create mode 100644 surfsense_backend/tests/unit/platforms/walmart/test_flows.py create mode 100644 surfsense_backend/tests/unit/platforms/walmart/test_parsers.py diff --git a/surfsense_backend/scripts/e2e_walmart_scraper.py b/surfsense_backend/scripts/e2e_walmart_scraper.py new file mode 100644 index 000000000..28ed3a344 --- /dev/null +++ b/surfsense_backend/scripts/e2e_walmart_scraper.py @@ -0,0 +1,80 @@ +"""Manual end-to-end check for the public Walmart scraper. + +Run from the backend directory: + + uv run python scripts/e2e_walmart_scraper.py + +The script requires live network access and the configured residential proxy +(US exit). It is intentionally excluded from pytest. +""" + +from __future__ import annotations + +import asyncio +import json +import sys +from pathlib import Path + +from dotenv import load_dotenv + +_BACKEND_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(_BACKEND_ROOT)) +for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"): + if _candidate.exists(): + load_dotenv(_candidate) + break + +from app.proprietary.platforms.walmart import ( # noqa: E402 + WalmartReviewsInput, + WalmartScrapeInput, + scrape_products, + scrape_reviews, +) + +_PRODUCT_URL = "https://www.walmart.com/ip/212092810" +_SEARCH_URL = "https://www.walmart.com/search?q=air+fryer" + + +def _check(label: str, passed: bool) -> bool: + print(f"[{'PASS' if passed else 'FAIL'}] {label}") + return passed + + +async def main() -> int: + ok = True + + product_items = await scrape_products( + WalmartScrapeInput(startUrls=[_PRODUCT_URL]), limit=1 + ) + product = product_items[0] if product_items else {} + print(json.dumps(product, indent=2, ensure_ascii=False)[:2500]) + ok &= _check( + "product detail has id and name", + bool(product.get("usItemId") and product.get("name")), + ) + + search_items = await scrape_products( + WalmartScrapeInput( + startUrls=[_SEARCH_URL], maxItemsPerStartUrl=3, includeDetails=False + ), + limit=3, + ) + ok &= _check( + "search returns product cards", + bool(search_items) and any("name" in item for item in search_items), + ) + + item_id = product.get("usItemId") or "212092810" + reviews = await scrape_reviews( + WalmartReviewsInput(itemIds=[item_id], maxReviews=15), limit=15 + ) + ok &= _check( + "reviews returned with rating and text", + bool(reviews) and any(r.get("rating") and r.get("text") for r in reviews), + ) + + return 0 if ok else 1 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) diff --git a/surfsense_backend/tests/unit/platforms/walmart/__init__.py b/surfsense_backend/tests/unit/platforms/walmart/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/platforms/walmart/fixtures/blocked.html b/surfsense_backend/tests/unit/platforms/walmart/fixtures/blocked.html new file mode 100644 index 000000000..a9e9d606e --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/walmart/fixtures/blocked.html @@ -0,0 +1,4 @@ +Robot or human? +
Activate and hold the button to confirm that you're human.
+

Robot or human?

+ diff --git a/surfsense_backend/tests/unit/platforms/walmart/fixtures/listing.html b/surfsense_backend/tests/unit/platforms/walmart/fixtures/listing.html new file mode 100644 index 000000000..0ebb7e682 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/walmart/fixtures/listing.html @@ -0,0 +1,3 @@ +laptop - Walmart.com + + diff --git a/surfsense_backend/tests/unit/platforms/walmart/fixtures/product.html b/surfsense_backend/tests/unit/platforms/walmart/fixtures/product.html new file mode 100644 index 000000000..3db17b152 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/walmart/fixtures/product.html @@ -0,0 +1,3 @@ +Midea AC + + diff --git a/surfsense_backend/tests/unit/platforms/walmart/fixtures/reviews.html b/surfsense_backend/tests/unit/platforms/walmart/fixtures/reviews.html new file mode 100644 index 000000000..b0d27e5b6 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/walmart/fixtures/reviews.html @@ -0,0 +1,3 @@ +Reviews + + diff --git a/surfsense_backend/tests/unit/platforms/walmart/test_flows.py b/surfsense_backend/tests/unit/platforms/walmart/test_flows.py new file mode 100644 index 000000000..045aa99d5 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/walmart/test_flows.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +from pathlib import Path + +from app.proprietary.platforms.walmart import ( + WalmartReviewsInput, + WalmartScrapeInput, + scrape_products, + scrape_reviews, + scraper, +) +from app.proprietary.platforms.walmart.fetch import FetchResult + +_FIXTURES = Path(__file__).parent / "fixtures" + + +def _fixture(name: str) -> str: + return (_FIXTURES / name).read_text(encoding="utf-8") + + +def _response(url: str, html: str, status: int = 200) -> FetchResult: + return FetchResult(status=status, html=html, url=url, cookies={}) + + +async def test_product_flow_emits_parsed_item(monkeypatch): + async def fetch_page(url: str, **_kwargs): + return _response(url, _fixture("product.html")) + + monkeypatch.setattr(scraper, "fetch_page", fetch_page) + items = await scrape_products( + WalmartScrapeInput(startUrls=["https://www.walmart.com/ip/212092810"]) + ) + + assert len(items) == 1 + assert items[0]["usItemId"] == "212092810" + assert items[0]["name"].startswith("Midea") + + +async def test_product_flow_maps_not_found_to_error_item(monkeypatch): + async def fetch_page(url: str, **_kwargs): + return _response(url, "", status=404) + + monkeypatch.setattr(scraper, "fetch_page", fetch_page) + items = await scrape_products( + WalmartScrapeInput(startUrls=["https://www.walmart.com/ip/212092810"]) + ) + + assert items[0]["error"] == "product_not_found" + + +async def test_listing_flow_card_only_honors_cap(monkeypatch): + calls: list[str] = [] + + async def fetch_page(url: str, **_kwargs): + calls.append(url) + html = _fixture("listing.html") if "page=1" in url else "" + return _response(url, html) + + monkeypatch.setattr(scraper, "fetch_page", fetch_page) + items = await scrape_products( + WalmartScrapeInput( + startUrls=["https://www.walmart.com/search?q=laptop"], + maxItemsPerStartUrl=1, + includeDetails=False, + ) + ) + + assert len(items) == 1 + assert items[0]["usItemId"] == "791595618" + assert len(calls) == 1 + + +async def test_listing_flow_enriches_with_detail_pages(monkeypatch): + async def fetch_page(url: str, **_kwargs): + if "/ip/" in url: + return _response(url, _fixture("product.html")) + html = _fixture("listing.html") if "page=1" in url else "" + return _response(url, html) + + monkeypatch.setattr(scraper, "fetch_page", fetch_page) + items = await scrape_products( + WalmartScrapeInput( + startUrls=["https://www.walmart.com/search?q=laptop"], + maxItemsPerStartUrl=2, + includeDetails=True, + ) + ) + + # Both cards enrich to the same product fixture (detail fetch wins). + assert all(item["longDescription"] for item in items) + + +async def test_invalid_url_yields_error_item(monkeypatch): + items = await scrape_products( + WalmartScrapeInput(startUrls=["https://example.com/not-walmart"]) + ) + assert items[0]["error"] == "invalid_url" + + +async def test_reviews_flow_paginates_until_empty(monkeypatch): + calls: list[str] = [] + + async def fetch_page(url: str, **_kwargs): + calls.append(url) + html = _fixture("reviews.html") if "page=1" in url else "" + return _response(url, html) + + monkeypatch.setattr(scraper, "fetch_page", fetch_page) + items = await scrape_reviews( + WalmartReviewsInput(itemIds=["212092810"], maxReviews=100) + ) + + assert len(items) == 2 + assert items[0]["reviewId"] == "296013686" + # page=1 returned records, page=2 empty → stop. Two fetches total. + assert len(calls) == 2 + + +async def test_reviews_flow_honors_max_reviews(monkeypatch): + async def fetch_page(url: str, **_kwargs): + return _response(url, _fixture("reviews.html")) + + monkeypatch.setattr(scraper, "fetch_page", fetch_page) + items = await scrape_reviews( + WalmartReviewsInput(itemIds=["212092810"], maxReviews=1) + ) + + assert len(items) == 1 + + +async def test_reviews_flow_maps_empty_to_error_item(monkeypatch): + async def fetch_page(url: str, **_kwargs): + return _response(url, "") + + monkeypatch.setattr(scraper, "fetch_page", fetch_page) + items = await scrape_reviews(WalmartReviewsInput(itemIds=["212092810"])) + + assert items[0]["error"] == "reviews_not_found" diff --git a/surfsense_backend/tests/unit/platforms/walmart/test_parsers.py b/surfsense_backend/tests/unit/platforms/walmart/test_parsers.py new file mode 100644 index 000000000..da66064ab --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/walmart/test_parsers.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +from pathlib import Path + +from app.proprietary.platforms.walmart.fetch import is_blocked +from app.proprietary.platforms.walmart.next_data import extract_next_data +from app.proprietary.platforms.walmart.parsers import ( + parse_listing_page, + parse_product, + parse_reviews_page, +) +from app.proprietary.platforms.walmart.url_resolver import extract_item_id, resolve_url + +_FIXTURES = Path(__file__).parent / "fixtures" + + +def _fixture(name: str) -> str: + return (_FIXTURES / name).read_text(encoding="utf-8") + + +def test_product_parser_extracts_core_fields_and_review_sample(): + data = extract_next_data(_fixture("product.html")) + item = parse_product(data, url="https://www.walmart.com/ip/212092810") + + assert item["usItemId"] == "212092810" + assert item["name"].startswith("Midea") + assert item["price"] == {"value": 149.0, "currency": "USD"} + assert item["listPrice"] == {"value": 199.0, "currency": "USD"} + assert item["stars"] == 4.5 + assert item["reviewsCount"] == 6287 + assert item["inStock"] is True + assert item["seller"] == {"id": "F55CT", "name": "Walmart.com", "type": "WALMART"} + assert item["longDescription"] == "

A powerful window unit.

" + assert item["images"] == [ + "https://i5.walmartimages.com/1.jpg", + "https://i5.walmartimages.com/2.jpg", + ] + assert item["reviewsSample"]["totalReviewCount"] == 6287 + assert item["reviewsSample"]["topReviews"][0]["verifiedPurchase"] is True + + +def test_listing_parser_normalizes_cards_and_marketplace_seller(): + data = extract_next_data(_fixture("listing.html")) + cards = parse_listing_page(data) + + assert [c["usItemId"] for c in cards] == ["791595618", "999001"] + first = cards[0] + assert first["url"] == "https://www.walmart.com/ip/Acer-Chromebook/791595618" + assert first["price"] == {"value": 79.99, "currency": "USD"} + assert first["seller"]["type"] == "MARKETPLACE" + assert first["inStock"] is True + # Fallback price + out-of-stock derivation for the second card. + assert cards[1]["price"] == {"value": 499.0} + assert cards[1]["inStock"] is False + + +def test_reviews_parser_extracts_all_records(): + data = extract_next_data(_fixture("reviews.html")) + reviews = parse_reviews_page(data) + + assert len(reviews) == 2 + assert reviews[0]["reviewId"] == "296013686" + assert reviews[0]["author"] == "JohnPaul" + assert reviews[0]["verifiedPurchase"] is True + assert reviews[0]["images"] == ["https://i5.walmartimages.com/r.jpg"] + assert reviews[0]["sellerResponse"] == "Thanks for the review!" + assert reviews[1]["verifiedPurchase"] is False + + +def test_block_detection_handles_status_and_body(): + assert is_blocked(_fixture("blocked.html"), 200) + assert is_blocked("", 412) + assert is_blocked("", 429) + assert not is_blocked(_fixture("product.html"), 200) + + +def test_missing_next_data_yields_none(): + assert extract_next_data("no script here") is None + assert parse_product(extract_next_data("") or {}, url="x") == {} + + +def test_url_resolver_classifies_and_extracts_ids(): + assert resolve_url("https://www.walmart.com/ip/Foo/123456789").kind == "product" + assert ( + extract_item_id("https://www.walmart.com/reviews/product/212092810") + == "212092810" + ) + assert resolve_url("https://www.walmart.com/search?q=tv").kind == "listing" + assert resolve_url("https://www.walmart.com/cp/tvs/3944").kind == "listing" + assert resolve_url("https://example.com/ip/1") is None From f6a8a2a566708ca22b74e30bdd832edeb9898a67 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 19 Jul 2026 08:45:27 +0200 Subject: [PATCH 11/21] feat(walmart): add chat subagent with scrape and reviews tools --- .../subagents/builtins/walmart/__init__.py | 1 + .../subagents/builtins/walmart/agent.py | 43 ++++++++++++ .../subagents/builtins/walmart/description.md | 2 + .../builtins/walmart/system_prompt.md | 68 +++++++++++++++++++ .../builtins/walmart/tools/__init__.py | 0 .../subagents/builtins/walmart/tools/index.py | 28 ++++++++ 6 files changed, 142 insertions(+) create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/__init__.py create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/agent.py create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/description.md create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/system_prompt.md create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/tools/__init__.py create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/tools/index.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/__init__.py new file mode 100644 index 000000000..8126071cf --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/__init__.py @@ -0,0 +1 @@ +"""``walmart`` builtin subagent: structured public Walmart product data and reviews.""" diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/agent.py new file mode 100644 index 000000000..794b050a2 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/agent.py @@ -0,0 +1,43 @@ +"""``walmart`` route: ``SurfSenseSubagentSpec`` builder for deepagents.""" + +from __future__ import annotations + +from typing import Any + +from langchain_core.language_models import BaseChatModel +from langchain_core.tools import BaseTool + +from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( + read_md_file, +) +from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec +from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( + pack_subagent, +) + +from .tools.index import NAME, RULESET, load_tools + + +def build_subagent( + *, + dependencies: dict[str, Any], + model: BaseChatModel | None = None, + middleware_stack: dict[str, Any] | None = None, + mcp_tools: list[BaseTool] | None = None, +) -> SurfSenseSubagentSpec: + tools = [*load_tools(dependencies=dependencies), *(mcp_tools or [])] + description = ( + read_md_file(__package__, "description").strip() + or "Scrapes public Walmart product data and reviews for a URL or search term." + ) + system_prompt = read_md_file(__package__, "system_prompt").strip() + return pack_subagent( + name=NAME, + description=description, + system_prompt=system_prompt, + tools=tools, + ruleset=RULESET, + dependencies=dependencies, + model=model, + middleware_stack=middleware_stack, + ) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/description.md new file mode 100644 index 000000000..88c6b7c28 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/description.md @@ -0,0 +1,2 @@ +Walmart product specialist: scrapes public Walmart listings and returns structured product data — title, item id (usItemId), brand, price and list price, star rating and review count, availability, images, features, seller, and product variants — plus deep paginated customer reviews (rating, text, author, verified-purchase flag, images, and seller responses). Works from a search term (e.g. "air fryer") or from Walmart product (/ip/...), search, category, or browse URLs, and can pull many reviews per product by item id or URL. Only public, anonymous US Walmart data — no login or seller account. +Use it for product research, price tracking, catalog enrichment by item id, and review mining. Triggers include "find X on Walmart", "Walmart price of X", "reviews for this Walmart product", "compare these Walmart products", and "look up this Walmart URL/item id". Not for general web search (use the Google Search specialist), reading an arbitrary non-Walmart page (use the web crawling specialist), or other marketplaces. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/system_prompt.md new file mode 100644 index 000000000..bf30fe2bb --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/system_prompt.md @@ -0,0 +1,68 @@ +You are the SurfSense Walmart sub-agent. +You receive delegated instructions from a supervisor agent and return structured results for supervisor synthesis. + + +Answer the delegated question from public Walmart product data and reviews gathered with your verbs, comparing against earlier results already in this conversation when the task calls for it. + + + +- `walmart_scrape` — product details and search/category listings +- `walmart_reviews` — deep paginated reviews for a product +- `read_run` / `search_run` (free readers for stored scrape output) + + + +- Discovering products: call `walmart_scrape` with `search_terms` (e.g. ["air fryer"]). +- Specific products: pass Walmart product URLs (/ip/...) or search/category/browse URLs in `urls`. +- Faster listings: set `include_details=false` to return card-only results without opening each product page. +- Sampled reviews: `walmart_scrape` returns a small on-page review sample by default (`include_reviews_sample=true`); disable it when reviews are irrelevant. +- Deep review mining: use `walmart_reviews` with product `urls` or numeric `item_ids` (usItemId); raise `max_reviews` and set `sort_by` (most-recent, most-helpful, rating-high, rating-low) as the task needs. Reviews are billed per review, so keep `max_reviews` to what the task actually requires. +- Batch multiple URLs or search terms into one call rather than many single-source calls. + +- Comparison requests: pull the current products, compare against prior values already in this conversation's earlier tool results, and report concrete deltas (price up/down, rating change, stock changes). + + + +- Use only tools in ``. +- Report only results present in the tool output. Never invent titles, item ids, prices, ratings, or reviews. +- `walmart_scrape`: provide at least one of `urls` or `search_terms`. +- `walmart_reviews`: provide at least one of `urls` or `item_ids`. + + + +- Do not perform general web search — that is the Google Search specialist's job. +- Do not read or extract an arbitrary non-Walmart page — return the URL for the web crawling specialist. +- Do not generate deliverables or perform connector mutations; return findings for the supervisor to act on. +- Only public, anonymous Walmart data — never anything behind a login or seller account. + + + +- Report uncertainty explicitly when evidence is incomplete or conflicting. +- Never present unverified claims as facts. + + + +- Underspecified request — no usable search term, URL, or item id — return `status=blocked` with the missing fields. +- Tool failure: return `status=error` with a concise recovery `next_step`. +- No useful evidence: return `status=blocked` with a narrower query or the scope you still need. + + + +Return **only** one JSON object (no markdown/prose): +{ + "status": "success" | "partial" | "blocked" | "error", + "action_summary": string, + "evidence": { + "findings": string[], + "sources": string[], + "confidence": "high" | "medium" | "low" + }, + "next_step": string | null, + "missing_fields": string[] | null, + "assumptions": string[] | null +} + +Route-specific rules: +- `evidence.findings`: max 10 entries, each a single sentence stating one distinct product, review theme, or delta. Do not paste raw payloads. +- `evidence.sources`: max 10 URLs, one per finding when applicable. List each URL once. + diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/tools/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/tools/index.py new file mode 100644 index 000000000..ac3e5ed81 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/tools/index.py @@ -0,0 +1,28 @@ +"""``walmart`` sub-agent tools: the Walmart product scrape and reviews verbs.""" + +from __future__ import annotations + +from typing import Any + +from langchain_core.tools import BaseTool + +from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset +from app.capabilities.core.access.agent import build_capability_tools +from app.capabilities.walmart.reviews.definition import WALMART_REVIEWS +from app.capabilities.walmart.scrape.definition import WALMART_SCRAPE + +NAME = "walmart" + +RULESET = Ruleset(origin=NAME, rules=[]) + +_CI_VERBS = [WALMART_SCRAPE, WALMART_REVIEWS] + + +def load_tools( + *, dependencies: dict[str, Any] | None = None, **kwargs: Any +) -> list[BaseTool]: + d = {**(dependencies or {}), **kwargs} + return build_capability_tools( + workspace_id=d.get("workspace_id"), + capabilities=_CI_VERBS, + ) From cabc5b5c86b0b602bae2a52ef99a1cbfd93ce624 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 19 Jul 2026 08:45:27 +0200 Subject: [PATCH 12/21] feat(walmart): register subagent in roster and connector map --- .../app/agents/chat/multi_agent_chat/constants.py | 1 + .../app/agents/chat/multi_agent_chat/subagents/registry.py | 4 ++++ .../unit/agents/multi_agent_chat/test_subagent_composition.py | 1 + 3 files changed, 6 insertions(+) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py b/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py index 333d4d274..2c44dbefe 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py @@ -39,6 +39,7 @@ SUBAGENT_TO_REQUIRED_CONNECTOR_MAP: dict[str, frozenset[str]] = { "reddit": frozenset(), "instagram": frozenset(), "tiktok": frozenset(), + "walmart": frozenset(), "mcp_discovery": frozenset( { "SLACK_CONNECTOR", diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py index 7b8b8a6f1..48de3e245 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py @@ -42,6 +42,9 @@ from app.agents.chat.multi_agent_chat.subagents.builtins.reddit.agent import ( from app.agents.chat.multi_agent_chat.subagents.builtins.tiktok.agent import ( build_subagent as build_tiktok_subagent, ) +from app.agents.chat.multi_agent_chat.subagents.builtins.walmart.agent import ( + build_subagent as build_walmart_subagent, +) from app.agents.chat.multi_agent_chat.subagents.builtins.web_crawler.agent import ( build_subagent as build_web_crawler_subagent, ) @@ -96,6 +99,7 @@ SUBAGENT_BUILDERS_BY_NAME: dict[str, SubagentBuilder] = { "onedrive": build_onedrive_subagent, "reddit": build_reddit_subagent, "tiktok": build_tiktok_subagent, + "walmart": build_walmart_subagent, "web_crawler": build_web_crawler_subagent, "youtube": build_youtube_subagent, } diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py index f578f3457..bc12f1308 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py @@ -40,6 +40,7 @@ _EXPECTED_SUBAGENTS = frozenset( "onedrive", "reddit", "tiktok", + "walmart", "web_crawler", "youtube", } From 56e8fa13e6ebbb88a9dda9ac62f8ff969ccfee6c Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 19 Jul 2026 08:45:27 +0200 Subject: [PATCH 13/21] feat(walmart): route product-review questions to marketplace specialists --- .../main_agent/system_prompt/prompts/identity/private.md | 2 +- .../main_agent/system_prompt/prompts/identity/team.md | 2 +- .../main_agent/system_prompt/prompts/kb_first.md | 3 ++- .../main_agent/system_prompt/prompts/routing.md | 4 +++- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/private.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/private.md index 82444634a..56e933253 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/private.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/private.md @@ -6,7 +6,7 @@ are changing, and what is being published across the open web — and to put that research to work alongside their own knowledge base. You do this by dispatching **specialist subagents** via the `task` tool: -- **Live web data** — Reddit, YouTube, Instagram, TikTok, Amazon, Google +- **Live web data** — Reddit, YouTube, Instagram, TikTok, Amazon, Walmart, Google Maps, Google Search, and the web crawler return structured, current platform data (posts, comments, transcripts, videos, products, reviews, SERPs, full page content). diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/team.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/team.md index 38cec63dc..a16a8bfd2 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/team.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/team.md @@ -6,7 +6,7 @@ are changing, and what is being published across the open web — and to put that research to work alongside the team's shared knowledge base. You do this by dispatching **specialist subagents** via the `task` tool: -- **Live web data** — Reddit, YouTube, Instagram, TikTok, Amazon, Google +- **Live web data** — Reddit, YouTube, Instagram, TikTok, Amazon, Walmart, Google Maps, Google Search, and the web crawler return structured, current platform data (posts, comments, transcripts, videos, products, reviews, SERPs, full page content). diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md index 3bae85262..60192fb39 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md @@ -2,7 +2,8 @@ CRITICAL — ground factual answers in what you actually receive this turn: - **live platform data** via the market specialists — `task(reddit, ...)`, `task(youtube, ...)`, `task(instagram, ...)`, - `task(tiktok, ...)`, `task(amazon, ...)`, `task(google_maps, ...)`, + `task(tiktok, ...)`, `task(amazon, ...)`, `task(walmart, ...)`, + `task(google_maps, ...)`, `task(google_search, ...)`, `task(web_crawler, ...)`. Anything about competitors, markets, rankings, reviews, or audience sentiment is answered from what these return **this turn**, never from your training data: your diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md index fb818cabc..7e2a0f983 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md @@ -32,7 +32,9 @@ about a brand, product, or topic is answered from the platform where they say it — `task(reddit, …)` for community discussion and threads, `task(youtube, …)` for video content, transcripts, and comment sections, `task(tiktok, …)` for short-form video trends by hashtag or search, -`task(google_maps, …)` for customer reviews of physical businesses. Web +`task(google_maps, …)` for customer reviews of physical businesses, +`task(amazon, …)` / `task(walmart, …)` for product ratings and customer +reviews of retail products (Walmart pages the full review history). Web search only finds articles *about* the conversation; the platform specialists return the conversation itself, structured and current. For competitive questions ("what are people saying about X", "how is Y From d7c30cac2f4a1622059199179a2117b99bed5783 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 19 Jul 2026 08:45:27 +0200 Subject: [PATCH 14/21] refactor(walmart): align reviews executor docstring --- surfsense_backend/app/capabilities/walmart/reviews/executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/surfsense_backend/app/capabilities/walmart/reviews/executor.py b/surfsense_backend/app/capabilities/walmart/reviews/executor.py index 35f88e11a..ba1ee0367 100644 --- a/surfsense_backend/app/capabilities/walmart/reviews/executor.py +++ b/surfsense_backend/app/capabilities/walmart/reviews/executor.py @@ -13,7 +13,7 @@ ReviewsFn = Callable[..., Awaitable[list[dict]]] def build_reviews_executor(scrape_fn: ReviewsFn | None = None) -> Executor: - """Bind the executor to a reviews scraper fn (defaults to the proprietary actor).""" + """Bind the capability input mapping to a replaceable reviews scraper function.""" scrape_fn = scrape_fn or scrape_reviews async def execute(payload: ReviewsInput) -> ReviewsOutput: From b5cddcfb2b251d5a0f88ea929b62da6b4bfe1863 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 19 Jul 2026 08:49:10 +0200 Subject: [PATCH 15/21] docs(walmart): document billing rates in env examples --- docker/.env.example | 4 +++- surfsense_backend/.env.example | 9 +++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/docker/.env.example b/docker/.env.example index cd9789fae..a92939e4a 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -441,7 +441,7 @@ SURFSENSE_ENABLE_DOOM_LOOP=true # WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE=3000 # Debit the credit wallet per *item returned* by the platform-native scrapers -# (Reddit, Google Search, Google Maps, Amazon, YouTube). Default FALSE keeps scraping +# (Reddit, Google Search, Google Maps, Amazon, Walmart, YouTube). Default FALSE keeps scraping # effectively free for self-hosted installs. Each rate is micro-USD per item, # config-driven: = round(USD_per_1000_items * 1_000). Defaults sit # at/above Apify's first-party actor rates (we charge no subscription tiers, @@ -458,6 +458,8 @@ SURFSENSE_ENABLE_DOOM_LOOP=true # TIKTOK_MICROS_PER_VIDEO=3500 # TIKTOK_MICROS_PER_USER=2500 # TIKTOK_MICROS_PER_COMMENT=1500 +# WALMART_MICROS_PER_PRODUCT=3500 +# WALMART_MICROS_PER_REVIEW=1500 # Safety ceiling on per-call premium reservation, in micro-USD ($1.00 default). # QUOTA_MAX_RESERVE_MICROS=1000000 diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index 404bd3b44..3b4a297b9 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -277,8 +277,8 @@ MICROS_PER_PAGE=1000 # WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE=3000 # Debit the credit wallet per *item returned* by the platform-native scrapers -# (Reddit, Google Search, Google Maps, Amazon, YouTube). Default FALSE keeps -# scraping effectively free for self-hosted/OSS installs; hosted set TRUE. +# (Reddit, Google Search, Google Maps, Amazon, Walmart, YouTube). Default FALSE +# keeps scraping effectively free for self-hosted/OSS installs; hosted set TRUE. # Each rate is micro-USD per item, fully config-driven (no hardcoded rate): # = round(USD_per_1000_items * 1_000) # 3500 == $3.50/1000 | 5000 == $5/1000 | 2000 == $2/1000 @@ -301,6 +301,11 @@ MICROS_PER_PAGE=1000 # Browser-listing retries when a feed is empty (profile feed is withheld from # flagged IPs; each retry draws a fresh rotating exit IP). Set to 1 for a static IP. # TIKTOK_LISTING_MAX_ATTEMPTS=3 +# Walmart products (server-rendered JSON behind residential proxies) priced like +# Amazon; reviews are 10/page (many light requests) so priced on the per-review +# market like Google Maps reviews. +# WALMART_MICROS_PER_PRODUCT=3500 +# WALMART_MICROS_PER_REVIEW=1500 # Low-balance warning threshold (micro-USD), surfaced to the UI. Default $0.50. CREDIT_LOW_BALANCE_WARNING_MICROS=500000 From 07721bda40b9086f5cd05ba90f6a8a7d33cb9105 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 19 Jul 2026 08:49:10 +0200 Subject: [PATCH 16/21] test(walmart): cover capability registry, schemas, and executors for both verbs --- .../unit/capabilities/walmart/__init__.py | 0 .../capabilities/walmart/reviews/__init__.py | 0 .../walmart/reviews/test_executor.py | 38 ++++++++++++ .../walmart/reviews/test_schemas.py | 37 ++++++++++++ .../capabilities/walmart/scrape/__init__.py | 0 .../walmart/scrape/test_executor.py | 42 ++++++++++++++ .../walmart/scrape/test_schemas.py | 58 +++++++++++++++++++ .../capabilities/walmart/test_registry.py | 22 +++++++ 8 files changed, 197 insertions(+) create mode 100644 surfsense_backend/tests/unit/capabilities/walmart/__init__.py create mode 100644 surfsense_backend/tests/unit/capabilities/walmart/reviews/__init__.py create mode 100644 surfsense_backend/tests/unit/capabilities/walmart/reviews/test_executor.py create mode 100644 surfsense_backend/tests/unit/capabilities/walmart/reviews/test_schemas.py create mode 100644 surfsense_backend/tests/unit/capabilities/walmart/scrape/__init__.py create mode 100644 surfsense_backend/tests/unit/capabilities/walmart/scrape/test_executor.py create mode 100644 surfsense_backend/tests/unit/capabilities/walmart/scrape/test_schemas.py create mode 100644 surfsense_backend/tests/unit/capabilities/walmart/test_registry.py diff --git a/surfsense_backend/tests/unit/capabilities/walmart/__init__.py b/surfsense_backend/tests/unit/capabilities/walmart/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/capabilities/walmart/reviews/__init__.py b/surfsense_backend/tests/unit/capabilities/walmart/reviews/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/capabilities/walmart/reviews/test_executor.py b/surfsense_backend/tests/unit/capabilities/walmart/reviews/test_executor.py new file mode 100644 index 000000000..d9d325780 --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/walmart/reviews/test_executor.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from app.capabilities.walmart.reviews.executor import build_reviews_executor +from app.capabilities.walmart.reviews.schemas import ReviewsInput +from app.proprietary.platforms.walmart import WalmartReviewsInput + + +class _FakeScraper: + def __init__(self) -> None: + self.calls: list[tuple[WalmartReviewsInput, int | None]] = [] + + async def __call__( + self, input_model: WalmartReviewsInput, *, limit: int | None = None + ) -> list[dict]: + self.calls.append((input_model, limit)) + return [{"reviewId": "r1", "rating": 5}] + + +async def test_executor_maps_agent_input_to_scraper_input(): + scraper = _FakeScraper() + execute = build_reviews_executor(scraper) + + output = await execute( + ReviewsInput( + urls=["https://www.walmart.com/ip/123456"], + item_ids=["222"], + max_reviews=50, + sort_by="most-helpful", + ) + ) + + assert output.items[0].reviewId == "r1" + input_model, limit = scraper.calls[0] + assert input_model.itemIds == ["https://www.walmart.com/ip/123456", "222"] + assert input_model.maxReviews == 50 + assert input_model.sort == "most-helpful" + # limit is the pre-flight worst case: 2 sources * 50 reviews + assert limit == 100 diff --git a/surfsense_backend/tests/unit/capabilities/walmart/reviews/test_schemas.py b/surfsense_backend/tests/unit/capabilities/walmart/reviews/test_schemas.py new file mode 100644 index 000000000..2d35ab191 --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/walmart/reviews/test_schemas.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from app.capabilities.walmart.reviews.schemas import ReviewsInput, ReviewsOutput + + +def test_estimated_units_scale_with_sources_and_max_reviews(): + payload = ReviewsInput( + urls=["https://www.walmart.com/ip/1"], item_ids=["222"], max_reviews=150 + ) + + # 2 sources * 150 reviews each + assert payload.estimated_units == 300 + + +def test_at_least_one_source_is_required(): + with pytest.raises(ValidationError): + ReviewsInput() + + +def test_sources_merge_urls_then_item_ids(): + payload = ReviewsInput(urls=["https://www.walmart.com/ip/1"], item_ids=["222"]) + + assert payload.sources() == ["https://www.walmart.com/ip/1", "222"] + + +def test_error_items_are_not_billable(): + output = ReviewsOutput( + items=[ + {"reviewId": "r1", "rating": 5}, + {"error": "reviews_not_found"}, + ] + ) + + assert output.billable_units == 1 diff --git a/surfsense_backend/tests/unit/capabilities/walmart/scrape/__init__.py b/surfsense_backend/tests/unit/capabilities/walmart/scrape/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/capabilities/walmart/scrape/test_executor.py b/surfsense_backend/tests/unit/capabilities/walmart/scrape/test_executor.py new file mode 100644 index 000000000..e31e4b638 --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/walmart/scrape/test_executor.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from app.capabilities.walmart.scrape.executor import build_scrape_executor +from app.capabilities.walmart.scrape.schemas import MAX_WALMART_RESULTS, ScrapeInput +from app.proprietary.platforms.walmart import WalmartScrapeInput + + +class _FakeScraper: + def __init__(self) -> None: + self.calls: list[tuple[WalmartScrapeInput, int | None]] = [] + + async def __call__( + self, input_model: WalmartScrapeInput, *, limit: int | None = None + ) -> list[dict]: + self.calls.append((input_model, limit)) + return [{"usItemId": "123", "name": "Product"}] + + +async def test_executor_maps_agent_input_to_scraper_input(): + scraper = _FakeScraper() + execute = build_scrape_executor(scraper) + + output = await execute( + ScrapeInput( + search_terms=["air fryer"], + urls=["https://www.walmart.com/ip/123456"], + max_items=5, + include_details=False, + include_reviews_sample=False, + ) + ) + + assert output.items[0].usItemId == "123" + input_model, limit = scraper.calls[0] + assert input_model.startUrls == [ + "https://www.walmart.com/ip/123456", + "https://www.walmart.com/search?q=air+fryer", + ] + assert input_model.maxItemsPerStartUrl == 5 + assert input_model.includeDetails is False + assert input_model.includeReviewsSample is False + assert limit == MAX_WALMART_RESULTS diff --git a/surfsense_backend/tests/unit/capabilities/walmart/scrape/test_schemas.py b/surfsense_backend/tests/unit/capabilities/walmart/scrape/test_schemas.py new file mode 100644 index 000000000..c7ce4ff9e --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/walmart/scrape/test_schemas.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from app.capabilities.walmart.scrape.schemas import ( + MAX_WALMART_RESULTS, + ScrapeInput, + ScrapeOutput, +) + + +def test_estimated_units_cover_search_and_direct_sources(): + payload = ScrapeInput( + search_terms=["air fryer", "blender"], + urls=["https://www.walmart.com/ip/123456"], + max_items=20, + ) + + # (2 search + 1 direct source) * 20 items each + assert payload.estimated_units == 60 + + +def test_estimated_units_respect_hard_run_ceiling(): + payload = ScrapeInput(search_terms=["x"] * 20, max_items=100) + assert payload.estimated_units == MAX_WALMART_RESULTS + + +def test_at_least_one_source_is_required(): + with pytest.raises(ValidationError): + ScrapeInput() + + +def test_combined_sources_are_capped(): + with pytest.raises(ValidationError): + ScrapeInput(urls=["u"] * 11, search_terms=["t"] * 10) + + +def test_start_urls_synthesize_a_search_url_per_term(): + payload = ScrapeInput( + urls=["https://www.walmart.com/ip/1"], search_terms=["air fryer"] + ) + + assert payload.start_urls() == [ + "https://www.walmart.com/ip/1", + "https://www.walmart.com/search?q=air+fryer", + ] + + +def test_error_items_are_not_billable(): + output = ScrapeOutput( + items=[ + {"usItemId": "123", "name": "Product"}, + {"error": "product_not_found", "errorDescription": "Missing"}, + ] + ) + + assert output.billable_units == 1 diff --git a/surfsense_backend/tests/unit/capabilities/walmart/test_registry.py b/surfsense_backend/tests/unit/capabilities/walmart/test_registry.py new file mode 100644 index 000000000..647ad7976 --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/walmart/test_registry.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from app.capabilities.core import BillingUnit +from app.capabilities.core.store import get_capability +from app.capabilities.walmart.reviews.schemas import ReviewsInput, ReviewsOutput +from app.capabilities.walmart.scrape.schemas import ScrapeInput, ScrapeOutput + + +def test_walmart_scrape_is_registered_and_metered(): + capability = get_capability("walmart.scrape") + + assert capability.input_schema is ScrapeInput + assert capability.output_schema is ScrapeOutput + assert capability.billing_unit is BillingUnit.WALMART_PRODUCT + + +def test_walmart_reviews_is_registered_and_metered(): + capability = get_capability("walmart.reviews") + + assert capability.input_schema is ReviewsInput + assert capability.output_schema is ReviewsOutput + assert capability.billing_unit is BillingUnit.WALMART_REVIEW From 1c50a33647ec08549ced4ca43d375d74e2b4b464 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 19 Jul 2026 09:01:04 +0200 Subject: [PATCH 17/21] feat(walmart): add playground catalog entry and icon --- surfsense_web/lib/playground/catalog.ts | 10 ++++++++++ surfsense_web/lib/playground/platform-icons.tsx | 1 + surfsense_web/public/connectors/walmart.svg | 11 +++++++++++ 3 files changed, 22 insertions(+) create mode 100644 surfsense_web/public/connectors/walmart.svg diff --git a/surfsense_web/lib/playground/catalog.ts b/surfsense_web/lib/playground/catalog.ts index d1c1af5cb..c32a24a20 100644 --- a/surfsense_web/lib/playground/catalog.ts +++ b/surfsense_web/lib/playground/catalog.ts @@ -6,6 +6,7 @@ import { InstagramIcon, RedditIcon, TikTokIcon, + WalmartIcon, WebIcon, YouTubeIcon, } from "./platform-icons"; @@ -96,6 +97,15 @@ export const PLAYGROUND_PLATFORMS: PlaygroundPlatform[] = [ icon: AmazonIcon, verbs: [{ name: "amazon.scrape", verb: "scrape", label: "Scrape" }], }, + { + id: "walmart", + label: "Walmart", + icon: WalmartIcon, + verbs: [ + { name: "walmart.scrape", verb: "scrape", label: "Scrape" }, + { name: "walmart.reviews", verb: "reviews", label: "Reviews" }, + ], + }, { id: "web", label: "Web", diff --git a/surfsense_web/lib/playground/platform-icons.tsx b/surfsense_web/lib/playground/platform-icons.tsx index dfa1a828d..c0afc91ef 100644 --- a/surfsense_web/lib/playground/platform-icons.tsx +++ b/surfsense_web/lib/playground/platform-icons.tsx @@ -23,6 +23,7 @@ function brandIcon(src: string, alt: string) { } export const AmazonIcon = brandIcon("/connectors/amazon.svg", "Amazon"); +export const WalmartIcon = brandIcon("/connectors/walmart.svg", "Walmart"); export const RedditIcon = brandIcon("/connectors/reddit.svg", "Reddit"); export const YouTubeIcon = brandIcon("/connectors/youtube.svg", "YouTube"); export const InstagramIcon = brandIcon("/connectors/instagram.svg", "Instagram"); diff --git a/surfsense_web/public/connectors/walmart.svg b/surfsense_web/public/connectors/walmart.svg new file mode 100644 index 000000000..dc334ef22 --- /dev/null +++ b/surfsense_web/public/connectors/walmart.svg @@ -0,0 +1,11 @@ + + Walmart + + + + + + + + + From 2a512e0015ca8ad28f18111fcd4b361e78b75fa0 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 19 Jul 2026 09:01:11 +0200 Subject: [PATCH 18/21] feat(walmart): add connector marketing page --- .../lib/connectors-marketing/index.ts | 2 + .../lib/connectors-marketing/walmart.tsx | 273 ++++++++++++++++++ 2 files changed, 275 insertions(+) create mode 100644 surfsense_web/lib/connectors-marketing/walmart.tsx diff --git a/surfsense_web/lib/connectors-marketing/index.ts b/surfsense_web/lib/connectors-marketing/index.ts index 7d79efd9b..888b9852a 100644 --- a/surfsense_web/lib/connectors-marketing/index.ts +++ b/surfsense_web/lib/connectors-marketing/index.ts @@ -5,6 +5,7 @@ import { instagram } from "./instagram"; import { reddit } from "./reddit"; import { tiktok } from "./tiktok"; import type { ConnectorPageContent } from "./types"; +import { walmart } from "./walmart"; import { webCrawl } from "./web-crawl"; import { youtube } from "./youtube"; @@ -19,6 +20,7 @@ const CONNECTOR_LIST: ConnectorPageContent[] = [ googleMaps, googleSearch, amazon, + walmart, webCrawl, ]; diff --git a/surfsense_web/lib/connectors-marketing/walmart.tsx b/surfsense_web/lib/connectors-marketing/walmart.tsx new file mode 100644 index 000000000..e959e2638 --- /dev/null +++ b/surfsense_web/lib/connectors-marketing/walmart.tsx @@ -0,0 +1,273 @@ +import { IconBrandWalmart } from "@tabler/icons-react"; +import type { ConnectorPageContent } from "./types"; + +export const walmart: ConnectorPageContent = { + slug: "walmart", + name: "Walmart", + cardTitle: "Walmart Product & Review API", + icon: IconBrandWalmart, + + metaTitle: "Walmart Product & Review API | SurfSense", + metaDescription: + "Scrape public Walmart product data and deep customer reviews as structured JSON: prices, ratings, sellers, variants, availability, and full review history. Start free.", + keywords: [ + "walmart product api", + "walmart scraper api", + "walmart review scraper", + "walmart reviews api", + "scrape walmart product data", + "walmart price scraper", + "walmart product data api", + "walmart price tracking api", + "walmart competitor monitoring", + "walmart seller data", + "walmart marketplace api", + "ecommerce product api", + ], + + h1: "Walmart Product and Review API", + heroLede: + "The SurfSense Walmart API scrapes public Walmart.com listings as structured JSON: price and list price, rating and review count, availability, 1P and marketplace sellers, variants, and specifications. A second verb pages the full public review history — ratings, text, authors, verified-purchase flags, images, and seller responses. Point your AI agents at a search term or product URL — no login, only public data.", + + transcript: { + prompt: "Pull the ratings and top complaints for this Walmart air fryer", + toolCall: + 'walmart.reviews({ urls: ["walmart.com/ip/..."],\n max_reviews: 500, sort_by: "most-recent" })', + rows: [ + { + primary: "Ninja 4-Qt Air Fryer — $79.00 (was $99.00)", + secondary: "4.6 stars · 8,204 reviews · In Stock", + tag: "-20%", + }, + { + primary: "512 reviews paged, 71% verified purchase", + secondary: "sorted most-recent · 132 with photos", + tag: "reviews", + }, + { + primary: 'Top complaint: "basket coating flaking after 3 months"', + secondary: "surfaced across 24 recent 1-2 star reviews", + tag: "theme", + }, + ], + resultSummary: "1 product · 512 reviews · surfaced in 6.4s", + }, + + extractIntro: + "Give the scraper Walmart product, search, category, or browse URLs, or plain search terms. The scrape verb returns product cards and full detail with a free on-page review sample; the reviews verb pages the complete public review history for any product. Walmart data is US marketplace (walmart.com), server-rendered as JSON for stability.", + extractFields: [ + { + label: "Product core", + description: + "Name, item id (usItemId), brand, price, list price, availability, in-stock flag, and canonical URL for every product.", + }, + { + label: "Ratings & review sample", + description: + "Average stars and total review count on every product, plus a free sample of on-page reviews when you fetch full detail.", + }, + { + label: "Deep reviews", + description: + "The reviews verb pages the full public review history: rating, title, text, author, date, verified-purchase flag, helpful votes, images, and seller responses.", + }, + { + label: "Sellers", + description: + "The seller behind each product, typed as Walmart first-party (1P) or third-party marketplace (3P), with id and name.", + }, + { + label: "Variants & media", + description: + "Product variants (color, size, count) with their own item ids and prices, plus the gallery and thumbnail images.", + }, + { + label: "Content & taxonomy", + description: + "Short and long descriptions, specifications, category, and breadcrumb trail for catalog enrichment and classification.", + }, + ], + + useCasesHeading: "What teams do with the Walmart API", + useCases: [ + { + title: "Review mining and product research", + description: + "Page the full review history for a product and brief an agent to cluster complaints, track sentiment over time, and separate verified-purchase signal from noise. Walmart exposes far more review depth than most marketplaces — use it to inform your roadmap or listing copy.", + }, + { + title: "Price and buy-box tracking", + description: + "Watch your own and competitors' prices, list prices, and which seller wins the offer (Walmart 1P vs a third-party marketplace seller). Run the same URLs on a schedule and diff the results to alert on undercuts and stockouts.", + }, + { + title: "Seller and marketplace intelligence", + description: + "See which third-party sellers are winning offers on the products that matter to you, and how first-party Walmart pricing compares, to size the marketplace opportunity in your category.", + }, + { + title: "Catalog and assortment enrichment", + description: + "Enrich a catalog by item id with structured attributes, variants, specifications, and images, and discover products from search or category URLs to map an entire assortment.", + }, + ], + + comparison: { + heading: "A Walmart scraper API built for agents", + intro: + "Most Walmart data APIs bill per request, split product and review data across endpoints, and leave the discovery-to-detail logic to you. Here is how SurfSense compares.", + columnLabel: "Typical Walmart data API", + rows: [ + { + feature: "Pricing", + official: "Per-request pricing that climbs fast at scale", + surfsense: "Pay per product or per review returned, with a free tier to start", + }, + { + feature: "Discovery", + official: "Separate search and detail endpoints you stitch together", + surfsense: "One verb takes search terms or product, search, category, and browse URLs", + }, + { + feature: "Reviews", + official: "A separate paid endpoint, often capped shallow", + surfsense: "A dedicated verb pages the full public review history with photos and seller replies", + }, + { + feature: "Data access", + official: "Requires accounts, keys, or approval for many providers", + surfsense: "Public, anonymous data only — no login or seller account", + }, + { + feature: "Agent-ready", + official: "No; you wire the harness yourself", + surfsense: "MCP server exposes walmart.scrape and walmart.reviews as native tools", + }, + ], + }, + + api: { + platform: "walmart", + verb: "scrape", + mcpTool: "walmart.scrape", + requestBody: { + search_terms: ["air fryer"], + max_items: 5, + include_reviews_sample: true, + }, + }, + + schema: { + requestNote: + "walmart.scrape: provide urls or search_terms (at least one), up to 20 combined sources per call. For the full review history, use the walmart.reviews verb with product urls or item_ids.", + request: [ + { + name: "urls", + type: "string[]", + description: + "Walmart product (/ip/), search (/search), category (/cp/), or browse (/browse/) URLs. Provide urls or search_terms.", + }, + { + name: "search_terms", + type: "string[]", + description: + "Search phrases run on walmart.com, e.g. 'air fryer'. Provide search_terms or urls.", + }, + { + name: "max_items", + type: "integer", + defaultValue: "10", + description: "Max products per search term or category/browse URL, 1 to 100.", + }, + { + name: "include_details", + type: "boolean", + defaultValue: "true", + description: "Fetch full product detail pages; false returns faster card-only results.", + }, + { + name: "include_reviews_sample", + type: "boolean", + defaultValue: "true", + description: + "Attach the free on-page review sample from each detail page. For full history use walmart.reviews.", + }, + ], + responseNote: + "The response is { items: [...] } with one item per product. One returned product is one billable unit; error items are never billed.", + response: [ + { + name: "name / usItemId / brand", + type: "string", + description: "Product identity: display name, Walmart item id, and brand.", + }, + { + name: "price / listPrice", + type: "object", + description: "Current price and strike-through list price, each with value and currency.", + }, + { + name: "stars / reviewsCount", + type: "number", + description: "Average star rating and total public review count.", + }, + { + name: "seller", + type: "object", + description: "Offer seller with id, name, and type (WALMART 1P or MARKETPLACE 3P).", + }, + { + name: "availabilityStatus / inStock", + type: "string", + description: "Availability label and a boolean in-stock flag.", + }, + { + name: "variants", + type: "object[]", + description: "Product variants (color, size, count) with their own item ids and prices.", + }, + { + name: "reviewsSample", + type: "object", + description: "Free sample of on-page reviews returned with full detail pages.", + }, + ], + }, + + faq: [ + { + question: "What is the Walmart product API?", + answer: + "It returns a Walmart listing's data as structured JSON instead of raw HTML. The SurfSense Walmart API scrapes public product pages and gives you price, rating, seller, variants, availability, and specifications as clean JSON your agents can read.", + }, + { + question: "Can I get the full Walmart review history?", + answer: + "Yes. The walmart.reviews verb pages the complete public review history for a product — rating, title, text, author, date, verified-purchase flag, helpful votes, images, and seller responses. Pass product URLs or item ids and set max_reviews and sort_by.", + }, + { + question: "Does it only use public data?", + answer: + "It does. The scraper collects public, anonymous Walmart data only — no login or seller account. That covers product details, prices, ratings, public reviews, seller identity, variants, and availability.", + }, + { + question: "How do I look up a product by item id or search term?", + answer: + "Pass a product URL (which contains the item id) or a phrase in search_terms to discover products. You can also pass search, category, and browse URLs. walmart.scrape handles discovery; walmart.reviews takes urls or item_ids directly.", + }, + { + question: "Which Walmart marketplace is supported?", + answer: + "The scraper targets the US marketplace at walmart.com. Data is read from the server-rendered JSON Walmart ships in the page, which keeps extraction stable against the frequent front-end A/B testing on the site.", + }, + ], + + related: [ + { label: "Amazon Product API", href: "/amazon" }, + { label: "Google Maps API", href: "/google-maps" }, + { label: "Google Search API", href: "/google-search" }, + { label: "Reddit API", href: "/reddit" }, + { label: "SurfSense MCP Server", href: "/mcp-server" }, + { label: "Read the docs", href: "/docs" }, + ], +}; From 6c98bd33a83e4656818c245427a1d9c4801b93b0 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 19 Jul 2026 09:01:11 +0200 Subject: [PATCH 19/21] docs(walmart): add connector docs and list in indexes --- .../content/docs/connectors/index.mdx | 2 +- .../content/docs/connectors/native/index.mdx | 7 +- .../content/docs/connectors/native/meta.json | 1 + .../docs/connectors/native/walmart.mdx | 71 +++++++++++++++++++ .../content/docs/how-to/mcp-server.mdx | 2 +- 5 files changed, 80 insertions(+), 3 deletions(-) create mode 100644 surfsense_web/content/docs/connectors/native/walmart.mdx diff --git a/surfsense_web/content/docs/connectors/index.mdx b/surfsense_web/content/docs/connectors/index.mdx index 04241f79f..eb3f28052 100644 --- a/surfsense_web/content/docs/connectors/index.mdx +++ b/surfsense_web/content/docs/connectors/index.mdx @@ -12,7 +12,7 @@ Connectors bring data into SurfSense — either as searchable knowledge or as li } title="Native Connectors" - description="SurfSense's built-in scraper APIs: Reddit, YouTube, Instagram, TikTok, Amazon, Google Maps, Google Search, and Web Crawl — usable in chat, the API Playground, or via REST" + description="SurfSense's built-in scraper APIs: Reddit, YouTube, Instagram, TikTok, Amazon, Walmart, Google Maps, Google Search, and Web Crawl — usable in chat, the API Playground, or via REST" href="/docs/connectors/native" /> + Date: Sun, 19 Jul 2026 09:01:18 +0200 Subject: [PATCH 20/21] feat(walmart): list walmart across marketing surfaces and example prompts --- surfsense_web/app/(home)/mcp-server/page.tsx | 12 +++++++----- surfsense_web/components/homepage/hero-section.tsx | 4 ++-- surfsense_web/components/homepage/persona-paths.tsx | 2 +- surfsense_web/components/seo/json-ld.tsx | 4 ++-- surfsense_web/lib/chat/example-prompts.ts | 1 + 5 files changed, 13 insertions(+), 10 deletions(-) diff --git a/surfsense_web/app/(home)/mcp-server/page.tsx b/surfsense_web/app/(home)/mcp-server/page.tsx index ab7769ee9..439e1ae6f 100644 --- a/surfsense_web/app/(home)/mcp-server/page.tsx +++ b/surfsense_web/app/(home)/mcp-server/page.tsx @@ -16,7 +16,7 @@ import type { FaqItem } from "@/lib/connectors-marketing/types"; const canonicalUrl = "https://www.surfsense.com/mcp-server"; const metaDescription = - "The SurfSense MCP server gives Claude, Cursor, and any MCP client native tools for your workspace: scrape Reddit, YouTube, Instagram, TikTok, Amazon, Google Maps, Google Search, and the web, plus full knowledge base access. One API key."; + "The SurfSense MCP server gives Claude, Cursor, and any MCP client native tools for your workspace: scrape Reddit, YouTube, Instagram, TikTok, Amazon, Walmart, Google Maps, Google Search, and the web, plus full knowledge base access. One API key."; export const metadata: Metadata = { title: "SurfSense MCP Server: Scraper APIs and Knowledge Base as Agent Tools", @@ -103,6 +103,8 @@ const TOOL_GROUPS = [ "surfsense_google_maps_reviews", "surfsense_google_search", "surfsense_amazon_scrape", + "surfsense_walmart_scrape", + "surfsense_walmart_reviews", "surfsense_web_crawl", "surfsense_list_scraper_runs", "surfsense_get_scraper_run", @@ -134,7 +136,7 @@ const FAQ: FaqItem[] = [ { question: "What is the SurfSense MCP server?", answer: - "It is a Model Context Protocol server that exposes your SurfSense workspace to MCP clients like Claude Code, Cursor, and Claude Desktop. Your agents get native tools for every scraper API (Reddit, YouTube, Instagram, TikTok, Amazon, Google Maps, Google Search, web crawl) and for searching, reading, and writing your knowledge base.", + "It is a Model Context Protocol server that exposes your SurfSense workspace to MCP clients like Claude Code, Cursor, and Claude Desktop. Your agents get native tools for every scraper API (Reddit, YouTube, Instagram, TikTok, Amazon, Walmart, Google Maps, Google Search, web crawl) and for searching, reading, and writing your knowledge base.", }, { question: "Which MCP clients does it work with?", @@ -222,9 +224,9 @@ export default function McpServerPage() {

The SurfSense MCP server hands Claude, Cursor, or any MCP client the whole platform: - scrape Reddit, YouTube, Instagram, TikTok, Amazon, Google Maps, Google Search, and - the open web, and search, read, and write your knowledge base. One API key, typed - tools, pay as you go. + scrape Reddit, YouTube, Instagram, TikTok, Amazon, Walmart, Google Maps, Google + Search, and the open web, and search, read, and write your knowledge base. One API + key, typed tools, pay as you go.