Merge pull request #1614 from CREDO23/feature-walmart-scraper

[Feat] Walmart scraper: product + reviews across API, chat, and MCP
This commit is contained in:
Rohan Verma 2026-07-21 15:39:33 -07:00 committed by GitHub
commit 66bf06429e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
69 changed files with 2630 additions and 22 deletions

View file

@ -441,7 +441,7 @@ SURFSENSE_ENABLE_DOOM_LOOP=true
# WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE=3000
# Debit the credit wallet per *item returned* by the platform-native scrapers
# (Reddit, Google Search, Google Maps, Amazon, YouTube). Default FALSE keeps scraping
# (Reddit, Google Search, Google Maps, Amazon, Walmart, YouTube). Default FALSE keeps scraping
# effectively free for self-hosted installs. Each rate is micro-USD per item,
# config-driven: <KEY> = round(USD_per_1000_items * 1_000). Defaults sit
# at/above Apify's first-party actor rates (we charge no subscription tiers,
@ -459,6 +459,8 @@ SURFSENSE_ENABLE_DOOM_LOOP=true
# TIKTOK_MICROS_PER_USER=2500
# TIKTOK_MICROS_PER_COMMENT=1500
# INDEED_SCRAPE_MICROS_PER_JOB=3500
# WALMART_MICROS_PER_PRODUCT=3500
# WALMART_MICROS_PER_REVIEW=1500
# Safety ceiling on per-call premium reservation, in micro-USD ($1.00 default).
# QUOTA_MAX_RESERVE_MICROS=1000000

View file

@ -277,8 +277,8 @@ MICROS_PER_PAGE=1000
# WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE=3000
# Debit the credit wallet per *item returned* by the platform-native scrapers
# (Reddit, Google Search, Google Maps, Amazon, YouTube). Default FALSE keeps
# scraping effectively free for self-hosted/OSS installs; hosted set TRUE.
# (Reddit, Google Search, Google Maps, Amazon, Walmart, YouTube). Default FALSE
# keeps scraping effectively free for self-hosted/OSS installs; hosted set TRUE.
# Each rate is micro-USD per item, fully config-driven (no hardcoded rate):
# <KEY> = round(USD_per_1000_items * 1_000)
# 3500 == $3.50/1000 | 5000 == $5/1000 | 2000 == $2/1000
@ -302,6 +302,11 @@ MICROS_PER_PAGE=1000
# Browser-listing retries when a feed is empty (profile feed is withheld from
# flagged IPs; each retry draws a fresh rotating exit IP). Set to 1 for a static IP.
# TIKTOK_LISTING_MAX_ATTEMPTS=3
# Walmart products (server-rendered JSON behind residential proxies) priced like
# Amazon; reviews are 10/page (many light requests) so priced on the per-review
# market like Google Maps reviews.
# WALMART_MICROS_PER_PRODUCT=3500
# WALMART_MICROS_PER_REVIEW=1500
# Low-balance warning threshold (micro-USD), surfaced to the UI. Default $0.50.
CREDIT_LOW_BALANCE_WARNING_MICROS=500000

View file

@ -40,6 +40,7 @@ SUBAGENT_TO_REQUIRED_CONNECTOR_MAP: dict[str, frozenset[str]] = {
"reddit": frozenset(),
"instagram": frozenset(),
"tiktok": frozenset(),
"walmart": frozenset(),
"mcp_discovery": frozenset(
{
"SLACK_CONNECTOR",

View file

@ -6,7 +6,7 @@ are changing, and what is being published across the open web — and to put
that research to work alongside their own knowledge base.
You do this by dispatching **specialist subagents** via the `task` tool:
- **Live web data** — Reddit, YouTube, Instagram, TikTok, Amazon, Google
- **Live web data** — Reddit, YouTube, Instagram, TikTok, Amazon, Walmart, Google
Maps, Google Search, and the web crawler return structured, current
platform data (posts, comments, transcripts, videos, products, reviews,
SERPs, full page content).

View file

@ -6,7 +6,7 @@ are changing, and what is being published across the open web — and to put
that research to work alongside the team's shared knowledge base.
You do this by dispatching **specialist subagents** via the `task` tool:
- **Live web data** — Reddit, YouTube, Instagram, TikTok, Amazon, Google
- **Live web data** — Reddit, YouTube, Instagram, TikTok, Amazon, Walmart, Google
Maps, Google Search, and the web crawler return structured, current
platform data (posts, comments, transcripts, videos, products, reviews,
SERPs, full page content).

View file

@ -2,7 +2,8 @@
CRITICAL — ground factual answers in what you actually receive this turn:
- **live platform data** via the market specialists —
`task(reddit, ...)`, `task(youtube, ...)`, `task(instagram, ...)`,
`task(tiktok, ...)`, `task(amazon, ...)`, `task(google_maps, ...)`,
`task(tiktok, ...)`, `task(amazon, ...)`, `task(walmart, ...)`,
`task(google_maps, ...)`,
`task(google_search, ...)`, `task(web_crawler, ...)`. Anything about
competitors, markets, rankings, reviews, or audience sentiment is answered
from what these return **this turn**, never from your training data: your

View file

@ -32,7 +32,9 @@ about a brand, product, or topic is answered from the platform where they
say it — `task(reddit, …)` for community discussion and threads,
`task(youtube, …)` for video content, transcripts, and comment sections,
`task(tiktok, …)` for short-form video trends by hashtag or search,
`task(google_maps, …)` for customer reviews of physical businesses. Web
`task(google_maps, …)` for customer reviews of physical businesses,
`task(amazon, …)` / `task(walmart, …)` for product ratings and customer
reviews of retail products (Walmart pages the full review history). Web
search only finds articles *about* the conversation; the platform
specialists return the conversation itself, structured and current. For
competitive questions ("what are people saying about X", "how is Y

View file

@ -0,0 +1 @@
"""``walmart`` builtin subagent: structured public Walmart product data and reviews."""

View file

@ -0,0 +1,43 @@
"""``walmart`` route: ``SurfSenseSubagentSpec`` builder for deepagents."""
from __future__ import annotations
from typing import Any
from langchain_core.language_models import BaseChatModel
from langchain_core.tools import BaseTool
from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import (
read_md_file,
)
from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec
from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import (
pack_subagent,
)
from .tools.index import NAME, RULESET, load_tools
def build_subagent(
*,
dependencies: dict[str, Any],
model: BaseChatModel | None = None,
middleware_stack: dict[str, Any] | None = None,
mcp_tools: list[BaseTool] | None = None,
) -> SurfSenseSubagentSpec:
tools = [*load_tools(dependencies=dependencies), *(mcp_tools or [])]
description = (
read_md_file(__package__, "description").strip()
or "Scrapes public Walmart product data and reviews for a URL or search term."
)
system_prompt = read_md_file(__package__, "system_prompt").strip()
return pack_subagent(
name=NAME,
description=description,
system_prompt=system_prompt,
tools=tools,
ruleset=RULESET,
dependencies=dependencies,
model=model,
middleware_stack=middleware_stack,
)

View file

@ -0,0 +1,2 @@
Walmart product specialist: scrapes public Walmart listings and returns structured product data — title, item id (usItemId), brand, price and list price, star rating and review count, availability, images, features, seller, and product variants — plus deep paginated customer reviews (rating, text, author, verified-purchase flag, images, and seller responses). Works from a search term (e.g. "air fryer") or from Walmart product (/ip/...), search, category, or browse URLs, and can pull many reviews per product by item id or URL. Only public, anonymous US Walmart data — no login or seller account.
Use it for product research, price tracking, catalog enrichment by item id, and review mining. Triggers include "find X on Walmart", "Walmart price of X", "reviews for this Walmart product", "compare these Walmart products", and "look up this Walmart URL/item id". Not for general web search (use the Google Search specialist), reading an arbitrary non-Walmart page (use the web crawling specialist), or other marketplaces.

View file

@ -0,0 +1,68 @@
You are the SurfSense Walmart sub-agent.
You receive delegated instructions from a supervisor agent and return structured results for supervisor synthesis.
<goal>
Answer the delegated question from public Walmart product data and reviews gathered with your verbs, comparing against earlier results already in this conversation when the task calls for it.
</goal>
<available_tools>
- `walmart_scrape` — product details and search/category listings
- `walmart_reviews` — deep paginated reviews for a product
- `read_run` / `search_run` (free readers for stored scrape output)
</available_tools>
<playbook>
- Discovering products: call `walmart_scrape` with `search_terms` (e.g. ["air fryer"]).
- Specific products: pass Walmart product URLs (/ip/...) or search/category/browse URLs in `urls`.
- Faster listings: set `include_details=false` to return card-only results without opening each product page.
- Sampled reviews: `walmart_scrape` returns a small on-page review sample by default (`include_reviews_sample=true`); disable it when reviews are irrelevant.
- Deep review mining: use `walmart_reviews` with product `urls` or numeric `item_ids` (usItemId); raise `max_reviews` and set `sort_by` (most-recent, most-helpful, rating-high, rating-low) as the task needs. Reviews are billed per review, so keep `max_reviews` to what the task actually requires.
- Batch multiple URLs or search terms into one call rather than many single-source calls.
<include snippet="run_reader"/>
- Comparison requests: pull the current products, compare against prior values already in this conversation's earlier tool results, and report concrete deltas (price up/down, rating change, stock changes).
</playbook>
<tool_policy>
- Use only tools in `<available_tools>`.
- Report only results present in the tool output. Never invent titles, item ids, prices, ratings, or reviews.
- `walmart_scrape`: provide at least one of `urls` or `search_terms`.
- `walmart_reviews`: provide at least one of `urls` or `item_ids`.
</tool_policy>
<out_of_scope>
- Do not perform general web search — that is the Google Search specialist's job.
- Do not read or extract an arbitrary non-Walmart page — return the URL for the web crawling specialist.
- Do not generate deliverables or perform connector mutations; return findings for the supervisor to act on.
- Only public, anonymous Walmart data — never anything behind a login or seller account.
</out_of_scope>
<safety>
- Report uncertainty explicitly when evidence is incomplete or conflicting.
- Never present unverified claims as facts.
</safety>
<failure_policy>
- Underspecified request — no usable search term, URL, or item id — return `status=blocked` with the missing fields.
- Tool failure: return `status=error` with a concise recovery `next_step`.
- No useful evidence: return `status=blocked` with a narrower query or the scope you still need.
</failure_policy>
<output_contract>
Return **only** one JSON object (no markdown/prose):
{
"status": "success" | "partial" | "blocked" | "error",
"action_summary": string,
"evidence": {
"findings": string[],
"sources": string[],
"confidence": "high" | "medium" | "low"
},
"next_step": string | null,
"missing_fields": string[] | null,
"assumptions": string[] | null
}
<include snippet="output_contract_base"/>
Route-specific rules:
- `evidence.findings`: max 10 entries, each a single sentence stating one distinct product, review theme, or delta. Do not paste raw payloads.
- `evidence.sources`: max 10 URLs, one per finding when applicable. List each URL once.
</output_contract>

View file

@ -0,0 +1,28 @@
"""``walmart`` sub-agent tools: the Walmart product scrape and reviews verbs."""
from __future__ import annotations
from typing import Any
from langchain_core.tools import BaseTool
from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset
from app.capabilities.core.access.agent import build_capability_tools
from app.capabilities.walmart.reviews.definition import WALMART_REVIEWS
from app.capabilities.walmart.scrape.definition import WALMART_SCRAPE
NAME = "walmart"
RULESET = Ruleset(origin=NAME, rules=[])
_CI_VERBS = [WALMART_SCRAPE, WALMART_REVIEWS]
def load_tools(
*, dependencies: dict[str, Any] | None = None, **kwargs: Any
) -> list[BaseTool]:
d = {**(dependencies or {}), **kwargs}
return build_capability_tools(
workspace_id=d.get("workspace_id"),
capabilities=_CI_VERBS,
)

View file

@ -45,6 +45,9 @@ from app.agents.chat.multi_agent_chat.subagents.builtins.reddit.agent import (
from app.agents.chat.multi_agent_chat.subagents.builtins.tiktok.agent import (
build_subagent as build_tiktok_subagent,
)
from app.agents.chat.multi_agent_chat.subagents.builtins.walmart.agent import (
build_subagent as build_walmart_subagent,
)
from app.agents.chat.multi_agent_chat.subagents.builtins.web_crawler.agent import (
build_subagent as build_web_crawler_subagent,
)
@ -100,6 +103,7 @@ SUBAGENT_BUILDERS_BY_NAME: dict[str, SubagentBuilder] = {
"onedrive": build_onedrive_subagent,
"reddit": build_reddit_subagent,
"tiktok": build_tiktok_subagent,
"walmart": build_walmart_subagent,
"web_crawler": build_web_crawler_subagent,
"youtube": build_youtube_subagent,
}

View file

@ -42,6 +42,8 @@ _PLATFORM_RATE_KEYS: dict[BillingUnit, str] = {
BillingUnit.TIKTOK_USER: "TIKTOK_MICROS_PER_USER",
BillingUnit.TIKTOK_COMMENT: "TIKTOK_MICROS_PER_COMMENT",
BillingUnit.INDEED_JOB: "INDEED_SCRAPE_MICROS_PER_JOB",
BillingUnit.WALMART_PRODUCT: "WALMART_MICROS_PER_PRODUCT",
BillingUnit.WALMART_REVIEW: "WALMART_MICROS_PER_REVIEW",
}
@ -65,6 +67,8 @@ _UNIT_NOUNS: dict[BillingUnit, str] = {
BillingUnit.TIKTOK_USER: "profile",
BillingUnit.TIKTOK_COMMENT: "comment",
BillingUnit.INDEED_JOB: "job",
BillingUnit.WALMART_PRODUCT: "product",
BillingUnit.WALMART_REVIEW: "review",
}

View file

@ -32,6 +32,8 @@ class BillingUnit(StrEnum):
TIKTOK_USER = "tiktok_user"
TIKTOK_COMMENT = "tiktok_comment"
INDEED_JOB = "indeed_job"
WALMART_PRODUCT = "walmart_product"
WALMART_REVIEW = "walmart_review"
class BillableInput(Protocol):

View file

@ -0,0 +1,6 @@
"""``walmart.*`` namespace: platform-native Walmart data verbs."""
from __future__ import annotations
from app.capabilities.walmart.reviews import definition as _reviews # noqa: F401
from app.capabilities.walmart.scrape import definition as _scrape # noqa: F401

View file

@ -0,0 +1,3 @@
"""Walmart deep review scraping capability."""
from __future__ import annotations

View file

@ -0,0 +1,24 @@
"""``walmart.reviews`` capability registration (billed per review; see config
``WALMART_MICROS_PER_REVIEW``)."""
from __future__ import annotations
from app.capabilities.core import BillingUnit, Capability, register_capability
from app.capabilities.walmart.reviews.executor import build_reviews_executor
from app.capabilities.walmart.reviews.schemas import ReviewsInput, ReviewsOutput
WALMART_REVIEWS = Capability(
name="walmart.reviews",
description=(
"Fetch deep paginated public Walmart product reviews with ratings, text, "
"authors, verified-purchase flags, images, and seller responses. Use "
"product urls or item ids."
),
input_schema=ReviewsInput,
output_schema=ReviewsOutput,
executor=build_reviews_executor(),
billing_unit=BillingUnit.WALMART_REVIEW,
docs_url="/docs/connectors/native/walmart",
)
register_capability(WALMART_REVIEWS)

View file

@ -0,0 +1,40 @@
"""``walmart.reviews`` executor: verb input → scraper → review items."""
from __future__ import annotations
from collections.abc import Awaitable, Callable
from app.capabilities.core import Executor
from app.capabilities.core.progress import emit_progress
from app.capabilities.walmart.reviews.schemas import ReviewsInput, ReviewsOutput
from app.proprietary.platforms.walmart import WalmartReviewsInput, scrape_reviews
ReviewsFn = Callable[..., Awaitable[list[dict]]]
def build_reviews_executor(scrape_fn: ReviewsFn | None = None) -> Executor:
"""Bind the capability input mapping to a replaceable reviews scraper function."""
scrape_fn = scrape_fn or scrape_reviews
async def execute(payload: ReviewsInput) -> ReviewsOutput:
input_model = WalmartReviewsInput(
itemIds=payload.sources(),
maxReviews=payload.max_reviews,
sort=payload.sort_by,
)
emit_progress(
"starting",
"Fetching Walmart reviews",
total=payload.estimated_units,
unit="review",
)
items = await scrape_fn(input_model, limit=payload.estimated_units)
emit_progress(
"done",
f"Scraped {sum('error' not in item for item in items)} review(s)",
current=len(items),
unit="review",
)
return ReviewsOutput(items=items)
return execute

View file

@ -0,0 +1,72 @@
"""``walmart.reviews`` I/O contracts.
A lean surface over ``WalmartReviewsInput``; the scraper's ``ReviewItem`` is
reused verbatim as the output element. Accepts product URLs or bare item ids
both resolve to a ``usItemId`` the reviews page is keyed on.
"""
from __future__ import annotations
from typing import Literal
from pydantic import BaseModel, Field, model_validator
from app.proprietary.platforms.walmart import ReviewItem
MAX_WALMART_REVIEW_SOURCES = 20
class ReviewsInput(BaseModel):
urls: list[str] = Field(
default_factory=list,
max_length=MAX_WALMART_REVIEW_SOURCES,
description=(
"Walmart product URLs (/ip/...) or reviews URLs to fetch reviews for. "
"Provide these OR item_ids (at least one is required)."
),
)
item_ids: list[str] = Field(
default_factory=list,
max_length=MAX_WALMART_REVIEW_SOURCES,
description="Walmart numeric item ids (usItemId) to fetch reviews for.",
)
max_reviews: int = Field(
default=200,
ge=1,
le=5000,
description="Max reviews to return per product (10 per page).",
)
sort_by: Literal["most-recent", "most-helpful", "rating-high", "rating-low"] = (
Field(default="most-recent", description="Review ordering.")
)
@model_validator(mode="after")
def _require_a_source(self) -> ReviewsInput:
if not (self.urls or self.item_ids):
raise ValueError("Provide at least one of 'urls' or 'item_ids'.")
return self
def sources(self) -> list[str]:
"""URLs and item ids merged; the scraper resolves each to a usItemId."""
return [*self.urls, *self.item_ids]
@property
def estimated_units(self) -> int:
"""Worst-case billable reviews: up to ``max_reviews`` per source."""
return (len(self.urls) + len(self.item_ids)) * self.max_reviews
class ReviewsOutput(BaseModel):
items: list[ReviewItem] = Field(
default_factory=list,
description="One item per review, in the scraper's emission order.",
)
@property
def billable_units(self) -> int:
"""One returned review = one billable unit; error items are not billed."""
return sum(
1
for item in self.items
if not (item.model_extra and item.model_extra.get("error"))
)

View file

@ -0,0 +1,3 @@
"""Walmart product scraping capability."""
from __future__ import annotations

View file

@ -0,0 +1,22 @@
"""Registration for the ``walmart.scrape`` capability."""
from __future__ import annotations
from app.capabilities.core import BillingUnit, Capability, register_capability
from app.capabilities.walmart.scrape.executor import build_scrape_executor
from app.capabilities.walmart.scrape.schemas import ScrapeInput, ScrapeOutput
WALMART_SCRAPE = Capability(
name="walmart.scrape",
description=(
"Scrape public Walmart product details, search/category listings, "
"prices, sellers, variants, availability, and a sample of on-page reviews."
),
input_schema=ScrapeInput,
output_schema=ScrapeOutput,
executor=build_scrape_executor(),
billing_unit=BillingUnit.WALMART_PRODUCT,
docs_url="/docs/connectors/native/walmart",
)
register_capability(WALMART_SCRAPE)

View file

@ -0,0 +1,46 @@
"""Executor for the ``walmart.scrape`` capability."""
from __future__ import annotations
from collections.abc import Awaitable, Callable
from app.capabilities.core import Executor
from app.capabilities.core.progress import emit_progress
from app.capabilities.walmart.scrape.schemas import (
MAX_WALMART_RESULTS,
ScrapeInput,
ScrapeOutput,
)
from app.proprietary.platforms.walmart import WalmartScrapeInput, scrape_products
ScrapeFn = Callable[..., Awaitable[list[dict]]]
def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor:
"""Bind the capability input mapping to a replaceable scraper function."""
scrape_fn = scrape_fn or scrape_products
async def execute(payload: ScrapeInput) -> ScrapeOutput:
input_model = WalmartScrapeInput(
startUrls=payload.start_urls(),
maxItemsPerStartUrl=payload.max_items,
includeDetails=payload.include_details,
includeReviewsSample=payload.include_reviews_sample,
)
emit_progress(
"starting",
"Scraping Walmart products",
total=payload.estimated_units,
unit="product",
)
items = await scrape_fn(input_model, limit=MAX_WALMART_RESULTS)
emit_progress(
"done",
f"Scraped {sum('error' not in item for item in items)} product(s)",
current=len(items),
total=payload.estimated_units,
unit="product",
)
return ScrapeOutput(items=items)
return execute

View file

@ -0,0 +1,64 @@
"""Input and output contracts for ``walmart.scrape``."""
from __future__ import annotations
from urllib.parse import quote_plus
from pydantic import BaseModel, Field, model_validator
from app.proprietary.platforms.walmart import ProductItem
MAX_WALMART_SOURCES = 20
MAX_WALMART_RESULTS = 1000
class ScrapeInput(BaseModel):
"""Agent-facing controls for public Walmart product discovery and enrichment."""
urls: list[str] = Field(default_factory=list, max_length=MAX_WALMART_SOURCES)
search_terms: list[str] = Field(
default_factory=list, max_length=MAX_WALMART_SOURCES
)
max_items: int = Field(default=10, ge=1, le=100)
include_details: bool = True
include_reviews_sample: bool = True
@model_validator(mode="after")
def _require_source(self) -> ScrapeInput:
if not (self.urls or self.search_terms):
raise ValueError("Provide at least one URL or search term.")
if len(self.urls) + len(self.search_terms) > MAX_WALMART_SOURCES:
raise ValueError(
f"Provide no more than {MAX_WALMART_SOURCES} combined sources."
)
return self
def start_urls(self) -> list[str]:
"""Direct URLs plus a search URL synthesized per search term."""
searches = [
f"https://www.walmart.com/search?q={quote_plus(term)}"
for term in self.search_terms
]
return [*self.urls, *searches]
@property
def estimated_units(self) -> int:
"""Worst-case returned products within the hard per-run ceiling."""
search_products = len(self.search_terms) * self.max_items
direct_products = len(self.urls) * self.max_items
return min(search_products + direct_products, MAX_WALMART_RESULTS)
class ScrapeOutput(BaseModel):
"""Products and structured per-input errors in emission order."""
items: list[ProductItem] = Field(default_factory=list)
@property
def billable_units(self) -> int:
"""Count successful products; error items are never billed."""
return sum(
1
for item in self.items
if not (item.model_extra and item.model_extra.get("error"))
)

View file

@ -741,6 +741,12 @@ class Config:
INDEED_SCRAPE_MICROS_PER_JOB = int(
os.getenv("INDEED_SCRAPE_MICROS_PER_JOB", "3500")
)
# Walmart products come from server-rendered JSON behind residential proxies,
# priced alongside Amazon's per-product meter.
WALMART_MICROS_PER_PRODUCT = int(os.getenv("WALMART_MICROS_PER_PRODUCT", "3500"))
# Reviews are 10 per page (many light requests per product), priced on the
# cheaper per-review market like the Google Maps review meter.
WALMART_MICROS_PER_REVIEW = int(os.getenv("WALMART_MICROS_PER_REVIEW", "1500"))
# Retry an empty listing draw on a fresh rotating IP. Set to 1 for a static
# proxy, where every retry re-hits the same exit.
TIKTOK_LISTING_MAX_ATTEMPTS = int(os.getenv("TIKTOK_LISTING_MAX_ATTEMPTS", "3"))

View file

@ -0,0 +1,52 @@
# Walmart Scraper
Two verbs read Walmart's public, anonymous pages: `walmart.scrape` (products +
search/category/browse listings, per-product billing) and `walmart.reviews`
(deep paginated reviews, per-review billing).
## Scope
The scraper reads public pages available to anonymous visitors — no login, no
account cookies. Data is extracted from the Next.js `__NEXT_DATA__` JSON blob
embedded in each page (with an `__APP_DATA__` fallback), not from the rendered
DOM, because Walmart obfuscates CSS classes and A/B-tests layout constantly.
`walmart.scrape` returns a free sample of on-page reviews (rating distribution,
aspects, top reviews) under `reviewsSample`. `walmart.reviews` fetches the full
review history from the public `/reviews/product/{usItemId}` page, which
robots.txt permits (unlike `/search`).
## Architecture
- `schemas.py` defines the stable input, product, review, and error models.
- `url_resolver.py` classifies product (`/ip/`) vs listing (`/search`, `/cp/`,
`/browse/`) URLs and extracts the numeric `usItemId`.
- `next_data.py` extracts and navigates the hidden Next.js JSON state.
- `fetch.py` owns proxy-aware HTTP access (US-pinned), block detection, and
retries.
- `parsers.py` contains pure, defensive JSON parsers.
- `scraper.py` coordinates discovery, enrichment, pagination, concurrency,
limits, and in-stream error items.
## Anti-bot
Walmart runs Akamai (edge/TLS) + PerimeterX/HUMAN (behavioral JS). Requests go
through US residential proxies with TLS-impersonated headers; blocked responses
(body markers, `412`/`429`/`503`, or the `200`-OK CAPTCHA body) rotate to a
fresh proxy exit.
Known ceilings and upgrade paths (see `fetch.py` / `scraper.py` `ponytail:`
notes): reviews page at 10/page; search capped at Walmart's 25-page limit;
session warming (seed `_px3`/`_pxhd` on a sticky exit) is the next lever if
block rates on the SSR pages climb, and the `/orchestra/*` GraphQL API is
deliberately avoided (rotating persisted-query hashes make it brittle).
## Verification
Offline fixtures cover the parsers and both flows:
uv run pytest tests/unit/platforms/walmart/
A manual live check (requires network + residential proxy):
uv run python scripts/e2e_walmart_scraper.py

View file

@ -0,0 +1,27 @@
"""Platform-native Walmart scraper (products, listings, reviews)."""
from .schemas import (
ErrorItem,
ProductItem,
ReviewItem,
WalmartReviewsInput,
WalmartScrapeInput,
)
from .scraper import (
iter_products,
iter_reviews,
scrape_products,
scrape_reviews,
)
__all__ = [
"ErrorItem",
"ProductItem",
"ReviewItem",
"WalmartReviewsInput",
"WalmartScrapeInput",
"iter_products",
"iter_reviews",
"scrape_products",
"scrape_reviews",
]

View file

@ -0,0 +1,171 @@
"""Network access for public Walmart pages.
Mirrors the Amazon fetch layer: every request goes through the configured
residential proxy (pinned to the US, since Walmart geo-locks inventory), and any
response that looks like an anti-bot interstitial is retried on a fresh proxy
exit. Ordinary HTTP failures are returned to the caller for domain-specific
handling.
Walmart runs Akamai (edge/TLS) + PerimeterX/HUMAN (behavioral JS). Two Walmart
specifics differ from Amazon:
* Walmart serves CAPTCHA with an HTTP ``200`` body ("Robot or human?"), so block
detection scans the body, never the status alone.
* ``412`` is PerimeterX's rejection code and is treated as blocked → rotate.
``ponytail:`` MVP hits only the server-rendered ``__NEXT_DATA__`` pages, which
TLS impersonation + residential proxies clear without seeding PerimeterX cookies.
If block rates on those pages climb, the upgrade path is a warmed sticky session
(seed ``_px3``/``_pxhd``/``ACID`` from a homepage fetch, reuse exit + cookies)
the same shape as Amazon's ``get_location_session``.
"""
from __future__ import annotations
import asyncio
import logging
import time
from collections.abc import Awaitable, Callable
from dataclasses import dataclass, field
from typing import Any
from scrapling.fetchers import AsyncFetcher
from app.utils.proxy import get_geo_proxy_url, get_sticky_proxy_url
logger = logging.getLogger(__name__)
_MAX_IP_ATTEMPTS = 6
_REQUEST_TIMEOUT_S = 30
_HEADERS = {
"Accept-Language": "en-US,en;q=0.9",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
}
_BLOCK_MARKERS = (
"robot or human",
"px-captcha",
"/blocked",
"verify you are a human",
"access to this page has been denied",
)
@dataclass(frozen=True)
class FetchResult:
"""The response details needed by scraper flows."""
status: int
html: str
url: str
cookies: dict[str, str]
headers: dict[str, str] = field(default_factory=dict)
async def gather_bounded[T](
factories: list[Callable[[], Awaitable[T]]], *, concurrency: int
) -> list[T]:
"""Run async factories concurrently while preserving input order."""
if not factories:
return []
semaphore = asyncio.Semaphore(max(1, concurrency))
async def run(factory: Callable[[], Awaitable[T]]) -> T:
async with semaphore:
return await factory()
return await asyncio.gather(*(run(factory) for factory in factories))
def is_blocked(
html: str | None, status: int, headers: dict[str, str] | None = None
) -> bool:
"""Return whether a response is a Walmart anti-bot interstitial.
``412`` is PerimeterX's rejection; ``429``/``503`` are throttles. Walmart also
serves CAPTCHA with a ``200`` body, so the body is scanned regardless of
status.
"""
if status in {412, 429, 503}:
return True
text = (html or "")[:200_000].lower()
return any(marker in text for marker in _BLOCK_MARKERS)
def _response_url(page: Any, fallback: str) -> str:
value = getattr(page, "url", None)
return str(value) if value else fallback
def _response_cookies(page: Any) -> dict[str, str]:
cookies = getattr(page, "cookies", None)
return dict(cookies) if isinstance(cookies, dict) else {}
def _response_headers(page: Any) -> dict[str, str]:
headers = getattr(page, "headers", None)
return dict(headers) if isinstance(headers, dict) else {}
def _selected_proxy(
proxy: str | None, country: str, attempt: int, url: str
) -> str | None:
if proxy is not None:
return proxy
if attempt > 1:
session_id = f"walmart-{attempt}-{abs(hash((url, time.time_ns()))):x}"
return get_sticky_proxy_url(session_id, country)
return get_geo_proxy_url(country)
async def fetch_page(
url: str,
*,
cookies: dict[str, str] | None = None,
proxy: str | None = None,
country: str = "us",
rotate_on_block: bool = True,
) -> FetchResult | None:
"""Fetch a page and retry blocked responses with fresh proxy exits."""
attempts = _MAX_IP_ATTEMPTS if rotate_on_block else 1
for attempt in range(1, attempts + 1):
selected_proxy = _selected_proxy(proxy, country, attempt, url)
started = time.perf_counter()
try:
page = await AsyncFetcher.get(
url,
headers={**_HEADERS},
cookies=cookies or {},
proxy=selected_proxy,
stealthy_headers=True,
timeout=_REQUEST_TIMEOUT_S,
)
except Exception as exc:
logger.warning("Walmart request failed for %s: %s", url, exc)
if proxy is not None:
return None
continue
status = int(getattr(page, "status", 0) or 0)
html = getattr(page, "html_content", None) or ""
response_headers = _response_headers(page)
logger.info(
"[walmart][perf] status=%s attempt=%s fetch_ms=%.1f url=%s",
status,
attempt,
(time.perf_counter() - started) * 1000,
url,
)
if rotate_on_block and is_blocked(html, status, response_headers):
logger.info(
"Walmart blocked proxy attempt %s/%s for %s", attempt, attempts, url
)
continue
return FetchResult(
status=status,
html=html,
url=_response_url(page, url),
cookies=_response_cookies(page),
headers=response_headers,
)
logger.warning("Walmart exhausted %s proxy attempts for %s", attempts, url)
return None

View file

@ -0,0 +1,69 @@
"""Extract Walmart's hidden Next.js JSON state from page HTML.
Every serious Walmart scraper (Scrapfly, Oxylabs, ScrapeOps, Apify) reads the
``<script id="__NEXT_DATA__">`` JSON blob rather than parsing the rendered DOM,
because Walmart obfuscates CSS classes and A/B-tests layout constantly. The JSON
shape is far more stable across redesigns.
``__APP_DATA__`` is the documented fallback anchor Walmart occasionally ships
instead during layout experiments.
"""
from __future__ import annotations
import json
import logging
import re
from typing import Any
logger = logging.getLogger(__name__)
_NEXT_DATA_RE = re.compile(r'<script id="__NEXT_DATA__"[^>]*>(.*?)</script>', re.DOTALL)
_APP_DATA_RE = re.compile(r'<script id="__APP_DATA__"[^>]*>(.*?)</script>', re.DOTALL)
def extract_next_data(html: str | None) -> dict[str, Any] | None:
"""Return the parsed Next.js state object, or ``None`` when absent/invalid.
Tries ``__NEXT_DATA__`` first, then the ``__APP_DATA__`` fallback so a single
Walmart layout experiment does not blank the whole extractor.
"""
if not html:
return None
for pattern in (_NEXT_DATA_RE, _APP_DATA_RE):
match = pattern.search(html)
if not match:
continue
try:
data = json.loads(match.group(1))
except (ValueError, TypeError):
logger.warning("Walmart hidden JSON present but did not parse")
continue
if isinstance(data, dict):
return data
return None
def dig(obj: Any, *keys: str | int) -> Any:
"""Walk nested dict/list keys, returning ``None`` on any miss.
Tolerates the layout drift between Walmart's ``initialData`` variants without
a cascade of ``if key in ...`` guards at every call site.
"""
current = obj
for key in keys:
if isinstance(key, int):
if not isinstance(current, list) or not -len(current) <= key < len(current):
return None
current = current[key]
else:
if not isinstance(current, dict) or key not in current:
return None
current = current[key]
return current
def initial_data(next_data: dict[str, Any]) -> dict[str, Any] | None:
"""The ``props.pageProps.initialData`` node shared by every page type."""
node = dig(next_data, "props", "pageProps", "initialData")
return node if isinstance(node, dict) else None

View file

@ -0,0 +1,246 @@
"""Pure parsers for Walmart's hidden ``__NEXT_DATA__`` JSON.
Product, listing, and review data all live in the same Next.js state tree; these
functions navigate it defensively (via :func:`.next_data.dig`) and normalize the
fields into the stable output shape. Missing sections yield ``None``/``[]`` so an
isolated schema change never discards an otherwise usable record.
"""
from __future__ import annotations
from typing import Any
from urllib.parse import urljoin
from .next_data import dig, initial_data
_WALMART_ORIGIN = "https://www.walmart.com"
# --------------------------------------------------------------------------- #
# Shared helpers. #
# --------------------------------------------------------------------------- #
def _price(node: Any) -> dict[str, Any] | None:
"""Normalize a Walmart price node into ``{value, currency}``."""
if not isinstance(node, dict):
return None
value = node.get("price")
currency = node.get("currencyUnit")
if value is None and currency is None:
return None
return {"value": value, "currency": currency}
def _seller(raw: dict[str, Any]) -> dict[str, Any] | None:
name = raw.get("sellerName") or raw.get("sellerDisplayName")
seller_id = raw.get("sellerId")
if not name and not seller_id:
return None
is_walmart = bool(name) and "walmart" in name.lower()
return {
"id": seller_id,
"name": name,
"type": "WALMART" if is_walmart else "MARKETPLACE",
}
def _absolute(url: str | None) -> str | None:
return urljoin(_WALMART_ORIGIN, url) if url else None
# --------------------------------------------------------------------------- #
# Listing / search cards. #
# --------------------------------------------------------------------------- #
def _listing_card(raw: dict[str, Any]) -> dict[str, Any] | None:
"""Normalize one search/category result item into a product card."""
item_id = raw.get("usItemId") or raw.get("id")
name = raw.get("name")
if not item_id or not name:
return None
price_info = raw.get("priceInfo") or {}
availability = raw.get("availabilityStatusV2") or {}
image = raw.get("imageInfo") or {}
return {
"usItemId": str(item_id),
"name": name,
"brand": raw.get("brand"),
"url": _absolute(raw.get("canonicalUrl")),
"price": _price(price_info.get("currentPrice"))
or ({"value": raw.get("price")} if raw.get("price") is not None else None),
"listPrice": _price(price_info.get("wasPrice")),
"stars": raw.get("averageRating"),
"reviewsCount": raw.get("numberOfReviews"),
"seller": _seller(raw),
"availabilityStatus": availability.get("value")
or ("OUT_OF_STOCK" if raw.get("isOutOfStock") else None),
"inStock": (availability.get("value") == "IN_STOCK")
if availability.get("value")
else (raw.get("isOutOfStock") is False if "isOutOfStock" in raw else None),
"thumbnailImage": image.get("thumbnailUrl") or raw.get("image"),
"sponsored": raw.get("isSponsoredFlag"),
}
def parse_listing_page(next_data: dict[str, Any]) -> list[dict[str, Any]]:
"""Extract normalized product cards from a search/category/browse page."""
data = initial_data(next_data)
if data is None:
return []
stacks = dig(data, "searchResult", "itemStacks")
if not isinstance(stacks, list):
return []
cards: list[dict[str, Any]] = []
seen: set[str] = set()
for stack in stacks:
for raw in (stack or {}).get("items") or []:
if not isinstance(raw, dict) or raw.get("__typename") not in (
None,
"Product",
):
continue
card = _listing_card(raw)
if card and card["usItemId"] not in seen:
seen.add(card["usItemId"])
cards.append(card)
return cards
# --------------------------------------------------------------------------- #
# Product detail. #
# --------------------------------------------------------------------------- #
def _images(image_info: dict[str, Any]) -> list[str]:
images: list[str] = []
for entry in image_info.get("allImages") or []:
url = entry.get("url") if isinstance(entry, dict) else None
if url and url not in images:
images.append(url)
return images
def _breadcrumbs(product: dict[str, Any]) -> list[str]:
"""Category breadcrumb names, root→leaf.
Walmart ships ``category.path`` as a list of ``{name, url}`` nodes, e.g.
``[{"name": "Home Improvement"}, ..., {"name": "Air Conditioners"}]`` not
a string.
"""
path = dig(product, "category", "path")
if not isinstance(path, list):
return []
return [node["name"] for node in path if isinstance(node, dict) and node.get("name")]
def _reviews_sample(
reviews: dict[str, Any] | None, limit: int = 10
) -> dict[str, Any] | None:
if not isinstance(reviews, dict):
return None
customer = reviews.get("customerReviews") or []
return {
"averageOverallRating": reviews.get("averageOverallRating"),
"totalReviewCount": reviews.get("totalReviewCount"),
"aspects": reviews.get("aspects") or [],
"topReviews": [
normalize_review(r) for r in customer[:limit] if isinstance(r, dict)
],
}
def parse_product(
next_data: dict[str, Any], *, url: str, include_reviews_sample: bool = True
) -> dict[str, Any]:
"""Extract normalized product fields from a product detail page."""
data = initial_data(next_data)
product = dig(data, "data", "product") if data else None
if not isinstance(product, dict):
return {}
idml = dig(data, "data", "idml") if data else None
reviews = dig(data, "data", "reviews") if data else None
price_info = product.get("priceInfo") or {}
image_info = product.get("imageInfo") or {}
availability = product.get("availabilityStatus")
breadcrumbs = _breadcrumbs(product)
fields: dict[str, Any] = {
"usItemId": str(product.get("usItemId") or product.get("id") or "") or None,
"name": product.get("name"),
"brand": product.get("brand"),
"url": url,
"price": _price(price_info.get("currentPrice")),
"listPrice": _price(price_info.get("wasPrice")),
"currency": dig(price_info, "currentPrice", "currencyUnit"),
"availabilityStatus": availability,
"inStock": availability == "IN_STOCK" if availability else None,
"stars": product.get("averageRating"),
"reviewsCount": product.get("numberOfReviews"),
"seller": _seller(product),
"manufacturerName": product.get("manufacturerName"),
"shortDescription": product.get("shortDescription"),
"longDescription": (idml or {}).get("longDescription")
if isinstance(idml, dict)
else None,
"thumbnailImage": image_info.get("thumbnailUrl"),
"images": _images(image_info),
"category": breadcrumbs[-1] if breadcrumbs else None,
"breadCrumbs": breadcrumbs,
"variants": product.get("variantCriteria") or [],
}
if include_reviews_sample:
fields["reviewsSample"] = _reviews_sample(reviews)
return {key: value for key, value in fields.items() if value is not None}
# --------------------------------------------------------------------------- #
# Reviews (deep pagination). #
# --------------------------------------------------------------------------- #
def normalize_review(raw: dict[str, Any]) -> dict[str, Any]:
"""Normalize one Walmart ``customerReviews`` record into a ``ReviewItem`` dict."""
badges = raw.get("badges") or []
verified = any(
isinstance(b, dict) and b.get("id") == "VerifiedPurchaser" for b in badges
)
photos = raw.get("photos") or raw.get("media") or []
images: list[str] = []
for photo in photos:
if not isinstance(photo, dict):
continue
url = photo.get("normalUrl") or dig(photo, "sizes", "normal", "url")
if url and url not in images:
images.append(url)
responses = raw.get("clientResponses") or []
seller_response = None
if responses and isinstance(responses[0], dict):
seller_response = responses[0].get("response") or responses[0].get(
"responseText"
)
return {
"reviewId": raw.get("reviewId"),
"rating": raw.get("rating"),
"title": raw.get("reviewTitle"),
"text": raw.get("reviewText"),
"submissionTime": raw.get("reviewSubmissionTime"),
"author": raw.get("userNickname"),
"verifiedPurchase": verified,
"positiveFeedback": raw.get("positiveFeedback"),
"negativeFeedback": raw.get("negativeFeedback"),
"images": images,
"syndicated": bool(raw.get("syndicationSource")),
"sellerResponse": seller_response,
}
def parse_reviews_page(next_data: dict[str, Any]) -> list[dict[str, Any]]:
"""Extract normalized reviews from one ``/reviews/product/{id}`` page."""
data = initial_data(next_data)
reviews = dig(data, "data", "reviews") if data else None
customer = reviews.get("customerReviews") if isinstance(reviews, dict) else None
if not isinstance(customer, list):
return []
return [normalize_review(r) for r in customer if isinstance(r, dict)]

View file

@ -0,0 +1,188 @@
# ruff: noqa: N815
"""Input/output models for the public Walmart scraper.
Two verbs share this module: ``walmart.scrape`` (products + listings) and
``walmart.reviews`` (deep paginated reviews). Walmart is a Next.js app that
ships its data as JSON in a ``<script id="__NEXT_DATA__">`` tag, so the parsers
read structured JSON rather than the rendered DOM more robust against the
constant CSS/layout A/B testing on walmart.com.
Outputs use ``extra="allow"`` on purpose (same as the Amazon scraper): the wire
shape can grow without breaking existing consumers. Only the small, stable,
always-populated nested objects are typed; volatile sections stay loose.
Public, anonymous data only: there is no auth/login field. Deep review
pagination uses the public ``/reviews/product/{id}`` page, which robots.txt
permits (unlike ``/search``).
"""
from __future__ import annotations
from typing import Any, Literal
from pydantic import BaseModel, ConfigDict, Field
# --------------------------------------------------------------------------- #
# Typed nested objects (the stable, always-populated ones). #
# --------------------------------------------------------------------------- #
class Price(BaseModel):
"""A monetary amount as Walmart renders it (``price``, ``listPrice``)."""
model_config = ConfigDict(extra="allow")
value: float | None = None
currency: str | None = None
class Seller(BaseModel):
"""The offer seller. ``type`` distinguishes Walmart 1P from 3P marketplace."""
model_config = ConfigDict(extra="allow")
id: str | None = None
name: str | None = None
type: Literal["WALMART", "MARKETPLACE"] | None = None
# --------------------------------------------------------------------------- #
# Input surfaces. #
# --------------------------------------------------------------------------- #
class WalmartScrapeInput(BaseModel):
"""Full input surface for ``walmart.scrape``.
``startUrls`` mixes product (``/ip/``), search (``/search``), category
(``/cp/``), and browse (``/browse/``) URLs; each is classified and dispatched
by :mod:`.url_resolver`. ``extra="allow"`` keeps a verbatim payload valid.
"""
model_config = ConfigDict(extra="allow")
startUrls: list[str] = Field(min_length=1)
maxItemsPerStartUrl: int | None = Field(default=None, ge=0)
maxSearchPagesPerStartUrl: int = Field(default=25, ge=1)
includeDetails: bool = True
includeReviewsSample: bool = True
country: str = "us"
class WalmartReviewsInput(BaseModel):
"""Full input surface for ``walmart.reviews`` (deep paginated reviews)."""
model_config = ConfigDict(extra="allow")
itemIds: list[str] = Field(min_length=1)
maxReviews: int = Field(default=200, ge=1)
sort: Literal["most-recent", "most-helpful", "rating-high", "rating-low"] = (
"most-recent"
)
country: str = "us"
# --------------------------------------------------------------------------- #
# Output items. #
# --------------------------------------------------------------------------- #
class ProductItem(BaseModel):
"""``walmart.scrape`` output item (one per product / listing card).
Unsourced fields default to ``None``/``[]``; ``extra="allow"`` keeps the
contract open so later milestones add fields without breaking consumers.
"""
model_config = ConfigDict(extra="allow", populate_by_name=True)
# Identity / core
usItemId: str | None = None
name: str | None = None
brand: str | None = None
url: str | None = None
price: Price | None = None
listPrice: Price | None = None
currency: str | None = None
availabilityStatus: str | None = None
inStock: bool | None = None
# Ratings / social
stars: float | None = None
reviewsCount: int | None = None
# Commerce
seller: Seller | None = None
manufacturerName: str | None = None
sponsored: bool | None = None
# Content (detail pages only)
shortDescription: str | None = None
longDescription: str | None = None
thumbnailImage: str | None = None
images: list[str] = Field(default_factory=list)
breadCrumbs: list[str] = Field(default_factory=list)
category: str | None = None
specifications: dict | None = None
variants: list[dict] = Field(default_factory=list)
fulfillment: dict | None = None
# Embedded free review sample (detail pages only)
reviewsSample: dict | None = None
# Provenance / meta
input: str | None = None
def to_output(self) -> dict[str, Any]:
"""Serialize to the flat dict output shape (keeps extras)."""
return self.model_dump(by_alias=True, exclude_none=False)
class ReviewItem(BaseModel):
"""``walmart.reviews`` output item (one per customer review)."""
model_config = ConfigDict(extra="allow")
reviewId: str | None = None
rating: float | None = None
title: str | None = None
text: str | None = None
submissionTime: str | None = None
author: str | None = None
verifiedPurchase: bool | None = None
positiveFeedback: int | None = None
negativeFeedback: int | None = None
images: list[str] = Field(default_factory=list)
syndicated: bool | None = None
sellerResponse: str | None = None
usItemId: str | None = None
input: str | None = None
def to_output(self) -> dict[str, Any]:
return self.model_dump(exclude_none=False)
# --------------------------------------------------------------------------- #
# Error item (the failure model — pushed into the stream, not raised). #
# --------------------------------------------------------------------------- #
ErrorCode = Literal[
"invalid_url",
"product_not_found",
"no_results_found",
"reviews_not_found",
]
class ErrorItem(BaseModel):
"""A per-input failure emitted into the dataset instead of a normal item.
Consumers tell error items apart from products/reviews by the ``error`` key.
"""
model_config = ConfigDict(extra="allow")
error: ErrorCode
errorDescription: str | None = None
input: str | None = None
url: str | None = None

View file

@ -0,0 +1,288 @@
"""Orchestrate public Walmart product discovery, enrichment, and reviews.
Two streaming cores: :func:`iter_products` dispatches each start URL to a product
or listing flow, and :func:`iter_reviews` paginates the public reviews page per
item id. Network and parsing concerns stay isolated in their own modules so
markup changes and retry policy can be tested independently.
The failure model is in-stream error items (dicts with an ``error`` key), never
exceptions identical to the Amazon scraper.
"""
from __future__ import annotations
from collections.abc import AsyncIterator
from typing import Any
from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
from .fetch import fetch_page, gather_bounded
from .next_data import extract_next_data
from .parsers import parse_listing_page, parse_product, parse_reviews_page
from .schemas import (
ErrorItem,
ProductItem,
ReviewItem,
WalmartReviewsInput,
WalmartScrapeInput,
)
from .url_resolver import ResolvedUrl, extract_item_id, resolve_url
__all__ = ["iter_products", "iter_reviews", "scrape_products", "scrape_reviews"]
_DETAIL_CONCURRENCY = 6
_SEARCH_PAGE_LIMIT = 25 # Walmart caps result pagination at 25 pages.
_REVIEWS_PAGE_LIMIT = 500 # 10 reviews/page → 5000-review safety ceiling.
_DEFAULT_ITEMS_PER_START_URL = 40
# Friendly sort → Walmart's ``sort`` query value on the reviews page.
_REVIEW_SORT = {
"most-recent": "submission-desc",
"most-helpful": "helpful",
"rating-high": "rating-desc",
"rating-low": "rating-asc",
}
def _error(
code: str, description: str, *, input_url: str | None, url: str | None = None
) -> dict[str, Any]:
return ErrorItem(
error=code,
errorDescription=description,
input=input_url,
url=url or input_url,
).model_dump()
def _page_url(url: str, page: int) -> str:
parsed = urlparse(url)
query = dict(parse_qsl(parsed.query, keep_blank_values=True))
query["page"] = str(page)
return urlunparse(parsed._replace(query=urlencode(query)))
def _product_url(item_id: str) -> str:
return f"https://www.walmart.com/ip/{item_id}"
def _reviews_url(item_id: str, page: int, sort: str) -> str:
query = urlencode(
{"page": page, "sort": sort, "entryPoint": "viewAllReviewsBottom"}
)
return f"https://www.walmart.com/reviews/product/{item_id}?{query}"
# --------------------------------------------------------------------------- #
# Product / listing flows. #
# --------------------------------------------------------------------------- #
async def _product_flow(
resolved: ResolvedUrl, input_model: WalmartScrapeInput
) -> AsyncIterator[dict[str, Any]]:
"""Fetch and parse one product detail page."""
url = _product_url(resolved.item_id) if resolved.item_id else resolved.url
response = await fetch_page(url, country=input_model.country)
if response is None:
yield _error(
"product_not_found",
"The product page could not be loaded after retrying proxy exits.",
input_url=resolved.url,
)
return
if response.status in {404, 410}:
yield _error(
"product_not_found",
"The product page was not found.",
input_url=resolved.url,
)
return
next_data = extract_next_data(response.html)
fields = (
parse_product(
next_data,
url=response.url,
include_reviews_sample=input_model.includeReviewsSample,
)
if next_data
else {}
)
if not fields.get("name"):
yield _error(
"product_not_found",
"The response did not contain a recognizable product.",
input_url=resolved.url,
url=response.url,
)
return
fields["input"] = resolved.url
yield ProductItem(**fields).to_output()
async def _listing_flow(
resolved: ResolvedUrl, input_model: WalmartScrapeInput
) -> AsyncIterator[dict[str, Any]]:
"""Page through a search/category/browse listing and optionally enrich."""
cap = (
input_model.maxItemsPerStartUrl
if input_model.maxItemsPerStartUrl is not None
else _DEFAULT_ITEMS_PER_START_URL
)
max_pages = min(input_model.maxSearchPagesPerStartUrl, _SEARCH_PAGE_LIMIT)
seen: set[str] = set()
emitted = 0
for page in range(1, max_pages + 1):
response = await fetch_page(
_page_url(resolved.url, page), country=input_model.country
)
next_data = extract_next_data(response.html) if response else None
cards = parse_listing_page(next_data) if next_data else []
cards = [c for c in cards if c["usItemId"] not in seen]
for card in cards:
seen.add(card["usItemId"])
if not cards:
if page == 1:
yield _error(
"no_results_found",
"The listing page did not contain any products.",
input_url=resolved.url,
)
return
cards = cards[: max(0, cap - emitted)]
if not input_model.includeDetails:
for card in cards:
card["input"] = resolved.url
yield ProductItem(**card).to_output()
emitted += 1
else:
async def load_card(card: dict[str, Any]) -> list[dict[str, Any]]:
product = resolve_url(card["url"]) if card.get("url") else None
if product is None or product.item_id is None:
card["input"] = resolved.url
return [ProductItem(**card).to_output()]
return [item async for item in _product_flow(product, input_model)]
for batch in await gather_bounded(
[lambda card=card: load_card(card) for card in cards],
concurrency=_DETAIL_CONCURRENCY,
):
for item in batch:
if "error" not in item and emitted >= cap:
return
yield item
if "error" not in item:
emitted += 1
if emitted >= cap:
return
_FLOWS = {"product": _product_flow, "listing": _listing_flow}
async def iter_products(
input_model: WalmartScrapeInput,
) -> AsyncIterator[dict[str, Any]]:
"""Yield product items for every start URL.
Each URL is classified and dispatched to its per-kind flow. An unrecognized
URL yields an ``invalid_url`` error item.
"""
for url in input_model.startUrls:
resolved = resolve_url(url)
if resolved is None:
yield _error(
"invalid_url",
"Start URL was malformed or not a recognized Walmart URL.",
input_url=url,
)
continue
async for item in _FLOWS[resolved.kind](resolved, input_model):
yield item
async def scrape_products(
input_model: WalmartScrapeInput, *, limit: int | None = None
) -> list[dict[str, Any]]:
"""Collect :func:`iter_products` into a list, honoring an optional ``limit``."""
from app.capabilities.core.progress import emit_progress
results: list[dict[str, Any]] = []
async for item in iter_products(input_model):
results.append(item)
emit_progress("scraping", current=len(results), total=limit, unit="product")
if limit is not None and len(results) >= limit:
break
return results
# --------------------------------------------------------------------------- #
# Reviews flow (deep pagination). #
# --------------------------------------------------------------------------- #
async def _reviews_flow(
item_id: str, input_model: WalmartReviewsInput
) -> AsyncIterator[dict[str, Any]]:
"""Paginate the public reviews page until empty or ``maxReviews`` reached.
Walmart does not expose a reliable total up front, so the loop stops on the
first empty page rather than trusting a page count.
"""
sort = _REVIEW_SORT[input_model.sort]
emitted = 0
for page in range(1, _REVIEWS_PAGE_LIMIT + 1):
response = await fetch_page(
_reviews_url(item_id, page, sort), country=input_model.country
)
next_data = extract_next_data(response.html) if response else None
reviews = parse_reviews_page(next_data) if next_data else []
if not reviews:
if page == 1 and emitted == 0:
yield _error(
"reviews_not_found",
"The product has no public reviews or could not be loaded.",
input_url=item_id,
)
return
for raw in reviews:
raw["usItemId"] = item_id
raw["input"] = item_id
yield ReviewItem(**raw).to_output()
emitted += 1
if emitted >= input_model.maxReviews:
return
async def iter_reviews(
input_model: WalmartReviewsInput,
) -> AsyncIterator[dict[str, Any]]:
"""Yield review items for every requested item id (or resolvable URL)."""
for raw_id in input_model.itemIds:
item_id = extract_item_id(raw_id)
if item_id is None:
yield _error(
"invalid_url",
"Could not extract a Walmart item id from the input.",
input_url=raw_id,
)
continue
async for item in _reviews_flow(item_id, input_model):
yield item
async def scrape_reviews(
input_model: WalmartReviewsInput, *, limit: int | None = None
) -> list[dict[str, Any]]:
"""Collect :func:`iter_reviews` into a list, honoring an optional ``limit``."""
from app.capabilities.core.progress import emit_progress
results: list[dict[str, Any]] = []
async for item in iter_reviews(input_model):
results.append(item)
emit_progress("scraping", current=len(results), total=limit, unit="review")
if limit is not None and len(results) >= limit:
break
return results

View file

@ -0,0 +1,75 @@
"""Classify a Walmart start URL and extract its identifiers.
Covers the shapes accepted on input: product pages (``/ip/{slug}/{id}`` or
``/ip/{id}``), search (``/search?q=``), category (``/cp/{slug}/{id}``), and
browse (``/browse/...``) pages. Category and browse render the same
``searchResult`` JSON as search, so they share the ``listing`` kind and parser.
Pure, no I/O. ``item_id`` is Walmart's ``usItemId`` — the numeric id the review
page and detail page are both keyed on.
"""
from __future__ import annotations
import re
from dataclasses import dataclass
from typing import Literal
from urllib.parse import parse_qs, urlparse
ResolvedKind = Literal["product", "listing"]
# usItemId: the trailing numeric segment of an /ip/ or /reviews/product/ path,
# or a bare numeric id passed directly.
_ITEM_ID_PATH_RE = re.compile(r"/(?:ip|reviews/product)/(?:[^/?#]+/)*(\d{4,})")
_BARE_ID_RE = re.compile(r"^\d{4,}$")
_LISTING_QUERY_KEYS = ("q", "cat_id", "browse")
@dataclass(frozen=True)
class ResolvedUrl:
kind: ResolvedKind
url: str
item_id: str | None = None
domain: str | None = None
def extract_item_id(url: str) -> str | None:
"""Pull the numeric ``usItemId`` out of a product/review URL or bare id."""
candidate = url.strip()
if _BARE_ID_RE.match(candidate):
return candidate
match = _ITEM_ID_PATH_RE.search(candidate)
return match.group(1) if match else None
def resolve_url(url: str) -> ResolvedUrl | None:
"""Classify a Walmart start URL into a scrape job, or ``None`` if unrecognized.
A bare numeric id resolves to a product (its ``usItemId``) so callers can
pass item ids directly without constructing a full URL.
"""
if _BARE_ID_RE.match(url.strip()):
return ResolvedUrl("product", url.strip(), item_id=url.strip())
parsed = urlparse(url)
host = (parsed.hostname or "").lower()
path = parsed.path or ""
if "walmart." not in host:
return None
if path.startswith("/ip/"):
return ResolvedUrl("product", url, item_id=extract_item_id(path), domain=host)
query = parse_qs(parsed.query)
is_listing = (
path.startswith("/search")
or path.startswith("/cp/")
or path.startswith("/browse/")
or any(key in query for key in _LISTING_QUERY_KEYS)
)
if is_listing:
return ResolvedUrl("listing", url, domain=host)
return None

View file

@ -8,6 +8,7 @@ import app.capabilities.indeed
import app.capabilities.instagram
import app.capabilities.reddit
import app.capabilities.tiktok
import app.capabilities.walmart
import app.capabilities.web
import app.capabilities.youtube # noqa: F401
from app.automations.api import router as automations_router

View file

@ -0,0 +1,80 @@
"""Manual end-to-end check for the public Walmart scraper.
Run from the backend directory:
uv run python scripts/e2e_walmart_scraper.py
The script requires live network access and the configured residential proxy
(US exit). It is intentionally excluded from pytest.
"""
from __future__ import annotations
import asyncio
import json
import sys
from pathlib import Path
from dotenv import load_dotenv
_BACKEND_ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(_BACKEND_ROOT))
for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"):
if _candidate.exists():
load_dotenv(_candidate)
break
from app.proprietary.platforms.walmart import ( # noqa: E402
WalmartReviewsInput,
WalmartScrapeInput,
scrape_products,
scrape_reviews,
)
_PRODUCT_URL = "https://www.walmart.com/ip/212092810"
_SEARCH_URL = "https://www.walmart.com/search?q=air+fryer"
def _check(label: str, passed: bool) -> bool:
print(f"[{'PASS' if passed else 'FAIL'}] {label}")
return passed
async def main() -> int:
ok = True
product_items = await scrape_products(
WalmartScrapeInput(startUrls=[_PRODUCT_URL]), limit=1
)
product = product_items[0] if product_items else {}
print(json.dumps(product, indent=2, ensure_ascii=False)[:2500])
ok &= _check(
"product detail has id and name",
bool(product.get("usItemId") and product.get("name")),
)
search_items = await scrape_products(
WalmartScrapeInput(
startUrls=[_SEARCH_URL], maxItemsPerStartUrl=3, includeDetails=False
),
limit=3,
)
ok &= _check(
"search returns product cards",
bool(search_items) and any("name" in item for item in search_items),
)
item_id = product.get("usItemId") or "212092810"
reviews = await scrape_reviews(
WalmartReviewsInput(itemIds=[item_id], maxReviews=15), limit=15
)
ok &= _check(
"reviews returned with rating and text",
bool(reviews) and any(r.get("rating") and r.get("text") for r in reviews),
)
return 0 if ok else 1
if __name__ == "__main__":
raise SystemExit(asyncio.run(main()))

View file

@ -41,6 +41,7 @@ _EXPECTED_SUBAGENTS = frozenset(
"onedrive",
"reddit",
"tiktok",
"walmart",
"web_crawler",
"youtube",
}

View file

@ -0,0 +1,38 @@
from __future__ import annotations
from app.capabilities.walmart.reviews.executor import build_reviews_executor
from app.capabilities.walmart.reviews.schemas import ReviewsInput
from app.proprietary.platforms.walmart import WalmartReviewsInput
class _FakeScraper:
def __init__(self) -> None:
self.calls: list[tuple[WalmartReviewsInput, int | None]] = []
async def __call__(
self, input_model: WalmartReviewsInput, *, limit: int | None = None
) -> list[dict]:
self.calls.append((input_model, limit))
return [{"reviewId": "r1", "rating": 5}]
async def test_executor_maps_agent_input_to_scraper_input():
scraper = _FakeScraper()
execute = build_reviews_executor(scraper)
output = await execute(
ReviewsInput(
urls=["https://www.walmart.com/ip/123456"],
item_ids=["222"],
max_reviews=50,
sort_by="most-helpful",
)
)
assert output.items[0].reviewId == "r1"
input_model, limit = scraper.calls[0]
assert input_model.itemIds == ["https://www.walmart.com/ip/123456", "222"]
assert input_model.maxReviews == 50
assert input_model.sort == "most-helpful"
# limit is the pre-flight worst case: 2 sources * 50 reviews
assert limit == 100

View file

@ -0,0 +1,37 @@
from __future__ import annotations
import pytest
from pydantic import ValidationError
from app.capabilities.walmart.reviews.schemas import ReviewsInput, ReviewsOutput
def test_estimated_units_scale_with_sources_and_max_reviews():
payload = ReviewsInput(
urls=["https://www.walmart.com/ip/1"], item_ids=["222"], max_reviews=150
)
# 2 sources * 150 reviews each
assert payload.estimated_units == 300
def test_at_least_one_source_is_required():
with pytest.raises(ValidationError):
ReviewsInput()
def test_sources_merge_urls_then_item_ids():
payload = ReviewsInput(urls=["https://www.walmart.com/ip/1"], item_ids=["222"])
assert payload.sources() == ["https://www.walmart.com/ip/1", "222"]
def test_error_items_are_not_billable():
output = ReviewsOutput(
items=[
{"reviewId": "r1", "rating": 5},
{"error": "reviews_not_found"},
]
)
assert output.billable_units == 1

View file

@ -0,0 +1,42 @@
from __future__ import annotations
from app.capabilities.walmart.scrape.executor import build_scrape_executor
from app.capabilities.walmart.scrape.schemas import MAX_WALMART_RESULTS, ScrapeInput
from app.proprietary.platforms.walmart import WalmartScrapeInput
class _FakeScraper:
def __init__(self) -> None:
self.calls: list[tuple[WalmartScrapeInput, int | None]] = []
async def __call__(
self, input_model: WalmartScrapeInput, *, limit: int | None = None
) -> list[dict]:
self.calls.append((input_model, limit))
return [{"usItemId": "123", "name": "Product"}]
async def test_executor_maps_agent_input_to_scraper_input():
scraper = _FakeScraper()
execute = build_scrape_executor(scraper)
output = await execute(
ScrapeInput(
search_terms=["air fryer"],
urls=["https://www.walmart.com/ip/123456"],
max_items=5,
include_details=False,
include_reviews_sample=False,
)
)
assert output.items[0].usItemId == "123"
input_model, limit = scraper.calls[0]
assert input_model.startUrls == [
"https://www.walmart.com/ip/123456",
"https://www.walmart.com/search?q=air+fryer",
]
assert input_model.maxItemsPerStartUrl == 5
assert input_model.includeDetails is False
assert input_model.includeReviewsSample is False
assert limit == MAX_WALMART_RESULTS

View file

@ -0,0 +1,58 @@
from __future__ import annotations
import pytest
from pydantic import ValidationError
from app.capabilities.walmart.scrape.schemas import (
MAX_WALMART_RESULTS,
ScrapeInput,
ScrapeOutput,
)
def test_estimated_units_cover_search_and_direct_sources():
payload = ScrapeInput(
search_terms=["air fryer", "blender"],
urls=["https://www.walmart.com/ip/123456"],
max_items=20,
)
# (2 search + 1 direct source) * 20 items each
assert payload.estimated_units == 60
def test_estimated_units_respect_hard_run_ceiling():
payload = ScrapeInput(search_terms=["x"] * 20, max_items=100)
assert payload.estimated_units == MAX_WALMART_RESULTS
def test_at_least_one_source_is_required():
with pytest.raises(ValidationError):
ScrapeInput()
def test_combined_sources_are_capped():
with pytest.raises(ValidationError):
ScrapeInput(urls=["u"] * 11, search_terms=["t"] * 10)
def test_start_urls_synthesize_a_search_url_per_term():
payload = ScrapeInput(
urls=["https://www.walmart.com/ip/1"], search_terms=["air fryer"]
)
assert payload.start_urls() == [
"https://www.walmart.com/ip/1",
"https://www.walmart.com/search?q=air+fryer",
]
def test_error_items_are_not_billable():
output = ScrapeOutput(
items=[
{"usItemId": "123", "name": "Product"},
{"error": "product_not_found", "errorDescription": "Missing"},
]
)
assert output.billable_units == 1

View file

@ -0,0 +1,22 @@
from __future__ import annotations
from app.capabilities.core import BillingUnit
from app.capabilities.core.store import get_capability
from app.capabilities.walmart.reviews.schemas import ReviewsInput, ReviewsOutput
from app.capabilities.walmart.scrape.schemas import ScrapeInput, ScrapeOutput
def test_walmart_scrape_is_registered_and_metered():
capability = get_capability("walmart.scrape")
assert capability.input_schema is ScrapeInput
assert capability.output_schema is ScrapeOutput
assert capability.billing_unit is BillingUnit.WALMART_PRODUCT
def test_walmart_reviews_is_registered_and_metered():
capability = get_capability("walmart.reviews")
assert capability.input_schema is ReviewsInput
assert capability.output_schema is ReviewsOutput
assert capability.billing_unit is BillingUnit.WALMART_REVIEW

View file

@ -0,0 +1,4 @@
<!doctype html><html><head><title>Robot or human?</title></head><body>
<div id="px-captcha">Activate and hold the button to confirm that you're human.</div>
<p>Robot or human?</p>
</body></html>

View file

@ -0,0 +1,3 @@
<!doctype html><html><head><title>laptop - Walmart.com</title></head><body>
<script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"initialData":{"searchResult":{"itemStacks":[{"count":2,"items":[{"__typename":"Product","usItemId":"791595618","name":"Restored Acer Chromebook","brand":null,"canonicalUrl":"/ip/Acer-Chromebook/791595618","averageRating":3.5,"numberOfReviews":46,"sellerId":"AB4FE","sellerName":"Ellison ProTech","availabilityStatusV2":{"value":"IN_STOCK"},"imageInfo":{"thumbnailUrl":"https://i5.walmartimages.com/a.jpg"},"priceInfo":{"currentPrice":{"price":79.99,"currencyUnit":"USD"}},"isSponsoredFlag":false},{"__typename":"Product","usItemId":"999001","name":"HP Laptop 15","canonicalUrl":"/ip/HP-Laptop/999001","isOutOfStock":true,"price":499.0}]}]}}}}}</script>
</body></html>

View file

@ -0,0 +1,3 @@
<!doctype html><html><head><title>Midea AC</title></head><body>
<script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"initialData":{"data":{"product":{"usItemId":"212092810","id":"ABC123","name":"Midea 5,000 BTU Window Air Conditioner","brand":"Midea","availabilityStatus":"IN_STOCK","averageRating":4.5,"numberOfReviews":6287,"manufacturerName":"Midea","shortDescription":"Cools a small room fast.","sellerName":"Walmart.com","sellerId":"F55CT","priceInfo":{"currentPrice":{"price":149.0,"currencyUnit":"USD"},"wasPrice":{"price":199.0,"currencyUnit":"USD"}},"imageInfo":{"thumbnailUrl":"https://i5.walmartimages.com/t.jpg","allImages":[{"url":"https://i5.walmartimages.com/1.jpg"},{"url":"https://i5.walmartimages.com/2.jpg"}]},"category":{"categoryPathId":"0:1072864:133026","path":[{"name":"Home","url":"/cp/home/1072864"},{"name":"Air Conditioners","url":"/cp/air-conditioners/133026"}]},"variantCriteria":[{"name":"capacity"}]},"idml":{"longDescription":"<p>A powerful window unit.</p>"},"reviews":{"averageOverallRating":4.5,"totalReviewCount":6287,"aspects":[{"name":"Value","score":90}],"customerReviews":[{"reviewId":"r1","rating":5,"reviewTitle":"Great AC","reviewText":"Very cold air.","reviewSubmissionTime":"2024-04-28","userNickname":"Anne","positiveFeedback":3,"negativeFeedback":0,"badges":[{"id":"VerifiedPurchaser"}]}]}}}}}}</script>
</body></html>

View file

@ -0,0 +1,3 @@
<!doctype html><html><head><title>Reviews</title></head><body>
<script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"initialData":{"data":{"reviews":{"customerReviews":[{"reviewId":"296013686","rating":5,"reviewTitle":"Awesome","reviewText":"The PS5 is awesome!","reviewSubmissionTime":"12/20/2022","userNickname":"JohnPaul","positiveFeedback":10,"negativeFeedback":3,"badges":[{"id":"VerifiedPurchaser"}],"photos":[{"normalUrl":"https://i5.walmartimages.com/r.jpg"}],"clientResponses":[{"response":"Thanks for the review!"}],"syndicationSource":null},{"reviewId":"296013687","rating":4,"reviewTitle":"Good","reviewText":"Solid console.","reviewSubmissionTime":"12/21/2022","userNickname":"Sam","positiveFeedback":1,"negativeFeedback":0,"badges":[]}]}}}}}}</script>
</body></html>

View file

@ -0,0 +1,138 @@
from __future__ import annotations
from pathlib import Path
from app.proprietary.platforms.walmart import (
WalmartReviewsInput,
WalmartScrapeInput,
scrape_products,
scrape_reviews,
scraper,
)
from app.proprietary.platforms.walmart.fetch import FetchResult
_FIXTURES = Path(__file__).parent / "fixtures"
def _fixture(name: str) -> str:
return (_FIXTURES / name).read_text(encoding="utf-8")
def _response(url: str, html: str, status: int = 200) -> FetchResult:
return FetchResult(status=status, html=html, url=url, cookies={})
async def test_product_flow_emits_parsed_item(monkeypatch):
async def fetch_page(url: str, **_kwargs):
return _response(url, _fixture("product.html"))
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
items = await scrape_products(
WalmartScrapeInput(startUrls=["https://www.walmart.com/ip/212092810"])
)
assert len(items) == 1
assert items[0]["usItemId"] == "212092810"
assert items[0]["name"].startswith("Midea")
async def test_product_flow_maps_not_found_to_error_item(monkeypatch):
async def fetch_page(url: str, **_kwargs):
return _response(url, "", status=404)
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
items = await scrape_products(
WalmartScrapeInput(startUrls=["https://www.walmart.com/ip/212092810"])
)
assert items[0]["error"] == "product_not_found"
async def test_listing_flow_card_only_honors_cap(monkeypatch):
calls: list[str] = []
async def fetch_page(url: str, **_kwargs):
calls.append(url)
html = _fixture("listing.html") if "page=1" in url else "<html></html>"
return _response(url, html)
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
items = await scrape_products(
WalmartScrapeInput(
startUrls=["https://www.walmart.com/search?q=laptop"],
maxItemsPerStartUrl=1,
includeDetails=False,
)
)
assert len(items) == 1
assert items[0]["usItemId"] == "791595618"
assert len(calls) == 1
async def test_listing_flow_enriches_with_detail_pages(monkeypatch):
async def fetch_page(url: str, **_kwargs):
if "/ip/" in url:
return _response(url, _fixture("product.html"))
html = _fixture("listing.html") if "page=1" in url else "<html></html>"
return _response(url, html)
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
items = await scrape_products(
WalmartScrapeInput(
startUrls=["https://www.walmart.com/search?q=laptop"],
maxItemsPerStartUrl=2,
includeDetails=True,
)
)
# Both cards enrich to the same product fixture (detail fetch wins).
assert all(item["longDescription"] for item in items)
async def test_invalid_url_yields_error_item(monkeypatch):
items = await scrape_products(
WalmartScrapeInput(startUrls=["https://example.com/not-walmart"])
)
assert items[0]["error"] == "invalid_url"
async def test_reviews_flow_paginates_until_empty(monkeypatch):
calls: list[str] = []
async def fetch_page(url: str, **_kwargs):
calls.append(url)
html = _fixture("reviews.html") if "page=1" in url else "<html></html>"
return _response(url, html)
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
items = await scrape_reviews(
WalmartReviewsInput(itemIds=["212092810"], maxReviews=100)
)
assert len(items) == 2
assert items[0]["reviewId"] == "296013686"
# page=1 returned records, page=2 empty → stop. Two fetches total.
assert len(calls) == 2
async def test_reviews_flow_honors_max_reviews(monkeypatch):
async def fetch_page(url: str, **_kwargs):
return _response(url, _fixture("reviews.html"))
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
items = await scrape_reviews(
WalmartReviewsInput(itemIds=["212092810"], maxReviews=1)
)
assert len(items) == 1
async def test_reviews_flow_maps_empty_to_error_item(monkeypatch):
async def fetch_page(url: str, **_kwargs):
return _response(url, "<html></html>")
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
items = await scrape_reviews(WalmartReviewsInput(itemIds=["212092810"]))
assert items[0]["error"] == "reviews_not_found"

View file

@ -0,0 +1,93 @@
from __future__ import annotations
from pathlib import Path
from app.proprietary.platforms.walmart.fetch import is_blocked
from app.proprietary.platforms.walmart.next_data import extract_next_data
from app.proprietary.platforms.walmart.parsers import (
parse_listing_page,
parse_product,
parse_reviews_page,
)
from app.proprietary.platforms.walmart.url_resolver import extract_item_id, resolve_url
_FIXTURES = Path(__file__).parent / "fixtures"
def _fixture(name: str) -> str:
return (_FIXTURES / name).read_text(encoding="utf-8")
def test_product_parser_extracts_core_fields_and_review_sample():
data = extract_next_data(_fixture("product.html"))
item = parse_product(data, url="https://www.walmart.com/ip/212092810")
assert item["usItemId"] == "212092810"
assert item["name"].startswith("Midea")
assert item["price"] == {"value": 149.0, "currency": "USD"}
assert item["listPrice"] == {"value": 199.0, "currency": "USD"}
assert item["stars"] == 4.5
assert item["reviewsCount"] == 6287
assert item["inStock"] is True
assert item["seller"] == {"id": "F55CT", "name": "Walmart.com", "type": "WALMART"}
assert item["longDescription"] == "<p>A powerful window unit.</p>"
assert item["images"] == [
"https://i5.walmartimages.com/1.jpg",
"https://i5.walmartimages.com/2.jpg",
]
# Walmart ships category.path as breadcrumb objects, not a string.
assert item["breadCrumbs"] == ["Home", "Air Conditioners"]
assert item["category"] == "Air Conditioners"
assert item["reviewsSample"]["totalReviewCount"] == 6287
assert item["reviewsSample"]["topReviews"][0]["verifiedPurchase"] is True
def test_listing_parser_normalizes_cards_and_marketplace_seller():
data = extract_next_data(_fixture("listing.html"))
cards = parse_listing_page(data)
assert [c["usItemId"] for c in cards] == ["791595618", "999001"]
first = cards[0]
assert first["url"] == "https://www.walmart.com/ip/Acer-Chromebook/791595618"
assert first["price"] == {"value": 79.99, "currency": "USD"}
assert first["seller"]["type"] == "MARKETPLACE"
assert first["inStock"] is True
# Fallback price + out-of-stock derivation for the second card.
assert cards[1]["price"] == {"value": 499.0}
assert cards[1]["inStock"] is False
def test_reviews_parser_extracts_all_records():
data = extract_next_data(_fixture("reviews.html"))
reviews = parse_reviews_page(data)
assert len(reviews) == 2
assert reviews[0]["reviewId"] == "296013686"
assert reviews[0]["author"] == "JohnPaul"
assert reviews[0]["verifiedPurchase"] is True
assert reviews[0]["images"] == ["https://i5.walmartimages.com/r.jpg"]
assert reviews[0]["sellerResponse"] == "Thanks for the review!"
assert reviews[1]["verifiedPurchase"] is False
def test_block_detection_handles_status_and_body():
assert is_blocked(_fixture("blocked.html"), 200)
assert is_blocked("", 412)
assert is_blocked("", 429)
assert not is_blocked(_fixture("product.html"), 200)
def test_missing_next_data_yields_none():
assert extract_next_data("<html><body>no script here</body></html>") is None
assert parse_product(extract_next_data("") or {}, url="x") == {}
def test_url_resolver_classifies_and_extracts_ids():
assert resolve_url("https://www.walmart.com/ip/Foo/123456789").kind == "product"
assert (
extract_item_id("https://www.walmart.com/reviews/product/212092810")
== "212092810"
)
assert resolve_url("https://www.walmart.com/search?q=tv").kind == "listing"
assert resolve_url("https://www.walmart.com/cp/tvs/3944").kind == "listing"
assert resolve_url("https://example.com/ip/1") is None

View file

@ -1,7 +1,8 @@
"""Scraper tools: one MCP surface per SurfSense platform capability.
Web crawl, Google Search, Reddit, YouTube, Google Maps, Indeed, and Amazon each
get a tool that maps a natural-language request to the workspace's scraper. Two run-history tools
Web crawl, Google Search, Reddit, YouTube, Google Maps, Amazon, Indeed, and
Walmart each get a tool that maps a natural-language request to the workspace's
scraper. Two run-history tools
list and fetch past runs, so a large result truncated inline can be retrieved in
full later. Each platform lives in its own module under platforms/.
"""
@ -21,6 +22,7 @@ from .platforms import (
instagram,
reddit,
tiktok,
walmart,
web,
youtube,
)
@ -35,6 +37,7 @@ _REGISTRARS = (
google_maps,
indeed,
amazon,
walmart,
run_history,
)

View file

@ -0,0 +1,140 @@
"""Walmart scraper tools: products/listings and deep reviews."""
from __future__ import annotations
from typing import Annotated, Literal
from mcp.server.fastmcp import FastMCP
from pydantic import Field
from ....core.client import SurfSenseClient
from ....core.rendering import ResponseFormatParam
from ....core.workspace_context import WorkspaceContext, WorkspaceParam
from ..annotations import SCRAPE
from ..capability import run_scraper
ReviewSort = Literal["most-recent", "most-helpful", "rating-high", "rating-low"]
def register(mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext) -> None:
"""Register the Walmart product and review tools."""
@mcp.tool(
name="surfsense_walmart_scrape",
title="Scrape Walmart products",
annotations=SCRAPE,
structured_output=False,
)
async def walmart_scrape(
urls: Annotated[
list[str] | None,
Field(
description="Walmart product (/ip/), search (/search), category "
"(/cp/), or browse (/browse/) URLs. Provide urls OR search_terms."
),
] = None,
search_terms: Annotated[
list[str] | None,
Field(
description="Search phrases run on walmart.com, e.g. ['air fryer']. "
"Provide search_terms OR urls."
),
] = None,
max_items: Annotated[
int,
Field(ge=1, le=100, description="Max products per search term or listing URL."),
] = 10,
include_details: Annotated[
bool,
Field(
description="Fetch full product detail pages. False returns faster "
"card-only results from listings."
),
] = True,
include_reviews_sample: Annotated[
bool,
Field(
description="Include the free on-page review sample (rating "
"distribution, aspects, top reviews) on detail pages."
),
] = True,
workspace: WorkspaceParam = None,
response_format: ResponseFormatParam = "markdown",
) -> str:
"""Scrape public Walmart product data by URL or search term.
Use this for product research: title, price, list price, rating and
review count, availability, seller (Walmart 1P vs marketplace), images,
description, variants, and a sample of on-page reviews. Only public,
anonymous data no login. For a product's full review history use
surfsense_walmart_reviews instead.
Example: search_terms=['air fryer'], max_items=5.
"""
return await run_scraper(
client,
context,
platform="walmart",
verb="scrape",
payload={
"urls": urls,
"search_terms": search_terms,
"max_items": max_items,
"include_details": include_details,
"include_reviews_sample": include_reviews_sample,
},
workspace=workspace,
response_format=response_format,
)
@mcp.tool(
name="surfsense_walmart_reviews",
title="Fetch Walmart reviews",
annotations=SCRAPE,
structured_output=False,
)
async def walmart_reviews(
urls: Annotated[
list[str] | None,
Field(
description="Walmart product URLs (/ip/...). Provide urls OR item_ids."
),
] = None,
item_ids: Annotated[
list[str] | None,
Field(
description="Walmart numeric item ids (usItemId) from "
"surfsense_walmart_scrape."
),
] = None,
max_reviews: Annotated[
int,
Field(ge=1, le=5000, description="Max reviews per product (10 per page)."),
] = 200,
sort_by: Annotated[
ReviewSort, Field(description="Review ordering.")
] = "most-recent",
workspace: WorkspaceParam = None,
response_format: ResponseFormatParam = "markdown",
) -> str:
"""Fetch deep paginated customer reviews for Walmart products.
Use this to read the full review history on specific products; get urls
or item_ids from surfsense_walmart_scrape first if you only have a name.
Returns rating, title, text, author, verified-purchase flag, images, and
seller response per review.
Example: item_ids=['212092810'], sort_by='most-helpful', max_reviews=100.
"""
return await run_scraper(
client,
context,
platform="walmart",
verb="reviews",
payload={
"urls": urls,
"item_ids": item_ids,
"max_reviews": max_reviews,
"sort_by": sort_by,
},
workspace=workspace,
response_format=response_format,
)

View file

@ -16,7 +16,7 @@ import type { FaqItem } from "@/lib/connectors-marketing/types";
const canonicalUrl = "https://www.surfsense.com/mcp-server";
const metaDescription =
"The SurfSense MCP server gives Claude, Cursor, and any MCP client native tools for your workspace: scrape Reddit, YouTube, Instagram, TikTok, Amazon, Google Maps, Google Search, and the web, plus full knowledge base access. One API key.";
"The SurfSense MCP server gives Claude, Cursor, and any MCP client native tools for your workspace: scrape Reddit, YouTube, Instagram, TikTok, Amazon, Walmart, Google Maps, Google Search, and the web, plus full knowledge base access. One API key.";
export const metadata: Metadata = {
title: "SurfSense MCP Server: Scraper APIs and Knowledge Base as Agent Tools",
@ -103,6 +103,8 @@ const TOOL_GROUPS = [
"surfsense_google_maps_reviews",
"surfsense_google_search",
"surfsense_amazon_scrape",
"surfsense_walmart_scrape",
"surfsense_walmart_reviews",
"surfsense_web_crawl",
"surfsense_list_scraper_runs",
"surfsense_get_scraper_run",
@ -134,7 +136,7 @@ const FAQ: FaqItem[] = [
{
question: "What is the SurfSense MCP server?",
answer:
"It is a Model Context Protocol server that exposes your SurfSense workspace to MCP clients like Claude Code, Cursor, and Claude Desktop. Your agents get native tools for every scraper API (Reddit, YouTube, Instagram, TikTok, Amazon, Google Maps, Google Search, web crawl) and for searching, reading, and writing your knowledge base.",
"It is a Model Context Protocol server that exposes your SurfSense workspace to MCP clients like Claude Code, Cursor, and Claude Desktop. Your agents get native tools for every scraper API (Reddit, YouTube, Instagram, TikTok, Amazon, Walmart, Google Maps, Google Search, web crawl) and for searching, reading, and writing your knowledge base.",
},
{
question: "Which MCP clients does it work with?",
@ -222,9 +224,9 @@ export default function McpServerPage() {
</h1>
<p className="mt-5 max-w-xl text-base leading-relaxed text-muted-foreground sm:text-lg">
The SurfSense MCP server hands Claude, Cursor, or any MCP client the whole platform:
scrape Reddit, YouTube, Instagram, TikTok, Amazon, Google Maps, Google Search, and
the open web, and search, read, and write your knowledge base. One API key, typed
tools, pay as you go.
scrape Reddit, YouTube, Instagram, TikTok, Amazon, Walmart, Google Maps, Google
Search, and the open web, and search, read, and write your knowledge base. One API
key, typed tools, pay as you go.
</p>
<div className="mt-8 flex flex-wrap items-center gap-3">
<Button asChild size="lg">

View file

@ -658,8 +658,8 @@ export function HeroSection() {
>
SurfSense is the open-source NotebookLM alternative for AI agents, an open web
research platform with live data connectors. Your AI agents research the live web with
structured data from Reddit, YouTube, Instagram, TikTok, Amazon, Google Maps, Google
Search, and any page on the open web.
structured data from Reddit, YouTube, Instagram, TikTok, Amazon, Walmart, Google
Maps, Google Search, and any page on the open web.
</p>
<div className="relative mb-4 flex w-full flex-col justify-center gap-y-2 sm:flex-row sm:justify-start sm:space-y-0 sm:space-x-4">

View file

@ -35,7 +35,7 @@ const PATHS: {
eyebrow: "For developers & agents",
title: "The whole platform is programmable",
description:
"Everything SurfSense agents can do is a typed REST API: scrape Reddit, YouTube, TikTok, Amazon, Google Maps, Google Search, and the open web, search the knowledge base, run automations. One key, JSON in and out, $5 free credit, pay as you go. Already running agents in Claude, Cursor, or your own harness? The SurfSense MCP server hands them the same tools natively.",
"Everything SurfSense agents can do is a typed REST API: scrape Reddit, YouTube, TikTok, Amazon, Walmart, Google Maps, Google Search, and the open web, search the knowledge base, run automations. One key, JSON in and out, $5 free credit, pay as you go. Already running agents in Claude, Cursor, or your own harness? The SurfSense MCP server hands them the same tools natively.",
links: [
{ label: "Read the docs", href: "/docs" },
{ label: "SurfSense MCP server", href: "/mcp-server" },

View file

@ -76,11 +76,11 @@ export function SoftwareApplicationJsonLd() {
"Free self-hosted from the open-source repo; cloud starts with $5 of free credit, then pay as you go",
},
description:
"SurfSense is an open-source NotebookLM alternative for AI agents. It researches the live web with platform-native connectors for Reddit, YouTube, TikTok, Amazon, Google Maps, Google Search, and any page on the open web, through one API or MCP server.",
"SurfSense is an open-source NotebookLM alternative for AI agents. It researches the live web with platform-native connectors for Reddit, YouTube, TikTok, Amazon, Walmart, Google Maps, Google Search, and any page on the open web, through one API or MCP server.",
url: "https://www.surfsense.com",
downloadUrl: "https://github.com/MODSetter/SurfSense/releases",
featureList: [
"Platform-native connectors: Reddit, YouTube, TikTok, Amazon, Google Maps, Google Search, Web Crawl",
"Platform-native connectors: Reddit, YouTube, TikTok, Amazon, Walmart, Google Maps, Google Search, Web Crawl",
"MCP server that exposes every connector as a native agent tool",
"Agent harness with retries, structured output, and credit metering",
"Live web research with cited briefs and alerts",

View file

@ -12,7 +12,7 @@ Connectors bring data into SurfSense — either as searchable knowledge or as li
<Card
icon={<Zap />}
title="Native Connectors"
description="SurfSense's built-in scraper APIs: Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed, Amazon, and Web Crawl — usable in chat, the API Playground, or via REST"
description="SurfSense's built-in scraper APIs: Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed, Amazon, Walmart, and Web Crawl — usable in chat, the API Playground, or via REST"
href="/docs/connectors/native"
/>
<Card

View file

@ -1,6 +1,6 @@
---
title: Native Connectors
description: SurfSense's built-in scraper APIs for Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed, Amazon, and the web
description: SurfSense's built-in scraper APIs for Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed, Amazon, Walmart, and the web
---
import { Card, Cards } from 'fumadocs-ui/components/card';
@ -48,6 +48,11 @@ Native connectors are SurfSense's own scraper APIs — built into the platform,
description="Public product data: prices, ratings, offers, sellers, and best-seller ranks"
href="/docs/connectors/native/amazon"
/>
<Card
title="Walmart"
description="Public product data and deep reviews: prices, ratings, sellers, and variants"
href="/docs/connectors/native/walmart"
/>
<Card
title="Web Crawl"
description="Scrape any page or spider a whole site into clean markdown"

View file

@ -9,6 +9,7 @@
"google-search",
"indeed",
"amazon",
"walmart",
"web-crawl"
],
"defaultOpen": false

View file

@ -0,0 +1,71 @@
---
title: Walmart
description: Scrape public Walmart product data and deep reviews as structured JSON
---
The Walmart scraper returns public product data as structured JSON: price and list price, rating and review count, availability, 1P and marketplace sellers, variants, specifications, and a free on-page review sample. A second verb pages the full public review history for a product. It only uses public, anonymous data — no login or seller account. Give it search terms or Walmart product, search, category, or browse URLs.
Walmart data is the US marketplace (`walmart.com`). The scraper reads the JSON Walmart server-renders into each page rather than the rendered DOM, which keeps extraction stable against the frequent front-end A/B testing on the site.
## Scrape products
```bash
POST /api/v1/workspaces/{workspace_id}/scrapers/walmart/scrape
```
### Inputs
| Field | Default | Description |
|-------|---------|-------------|
| `urls` | — | Walmart product (`/ip/`), search (`/search`), category (`/cp/`), or browse (`/browse/`) URLs. Provide `urls` or `search_terms` |
| `search_terms` | — | Search phrases run on `walmart.com`, e.g. `air fryer`. Provide `search_terms` or `urls` |
| `max_items` | `10` | Max products per search term or category/browse URL (1100) |
| `include_details` | `true` | Fetch full product detail pages; `false` returns faster card-only results |
| `include_reviews_sample` | `true` | Attach the free on-page review sample from each detail page. For full history use the reviews verb |
At least one of `urls` or `search_terms` is required, with up to 20 combined sources per call.
```bash
curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/walmart/scrape" \
-H "Authorization: Bearer $SURFSENSE_API_KEY" \
-H "Content-Type: application/json" \
-d '{
"search_terms": ["air fryer"],
"max_items": 5,
"include_reviews_sample": true
}'
```
The response is `{ "items": [...] }` — one item per product, plus structured error items for any input that could not be resolved. Billing is per returned product; error items are never billed.
## Fetch reviews
```bash
POST /api/v1/workspaces/{workspace_id}/scrapers/walmart/reviews
```
### Inputs
| Field | Default | Description |
|-------|---------|-------------|
| `urls` | — | Walmart product (`/ip/`) or reviews URLs to fetch reviews for. Provide `urls` or `item_ids` |
| `item_ids` | — | Walmart numeric item ids (`usItemId`) to fetch reviews for. Provide `item_ids` or `urls` |
| `max_reviews` | `200` | Max reviews per product (15000); reviews are paged 10 at a time |
| `sort_by` | `most-recent` | Review ordering: `most-recent`, `most-helpful`, `rating-high`, or `rating-low` |
At least one of `urls` or `item_ids` is required, with up to 20 combined sources per call.
```bash
curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/walmart/reviews" \
-H "Authorization: Bearer $SURFSENSE_API_KEY" \
-H "Content-Type: application/json" \
-d '{
"urls": ["https://www.walmart.com/ip/1234567890"],
"max_reviews": 500,
"sort_by": "most-recent"
}'
```
The response is `{ "items": [...] }` — one item per review with rating, title, text, author, date, verified-purchase flag, helpful votes, images, and any seller response. Billing is per returned review; error items are never billed.
For the full input and output JSON schemas and generated code snippets in your language, open **API Playground → Walmart** in your workspace.

View file

@ -7,7 +7,7 @@ import { Tab, Tabs } from 'fumadocs-ui/components/tabs';
# SurfSense MCP Server
The SurfSense MCP server exposes your workspace to any [Model Context Protocol](https://modelcontextprotocol.io/) client. Your agent gets 26 native, typed tools: every scraper (Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed, Amazon, web crawl), full knowledge-base access (search, read, add, upload, update, delete), and a workspace selector.
The SurfSense MCP server exposes your workspace to any [Model Context Protocol](https://modelcontextprotocol.io/) client. Your agent gets 28 native, typed tools: every scraper (Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed, Amazon, Walmart, web crawl), full knowledge-base access (search, read, add, upload, update, delete), and a workspace selector.
Connect it two ways: the **hosted** server at `https://mcp.surfsense.com/mcp` (nothing to install — just an API key), or run it yourself over **stdio** against any SurfSense backend, cloud or self-hosted.

View file

@ -37,6 +37,7 @@ export const CHAT_EXAMPLE_CATEGORIES: ChatExampleCategory[] = [
"Analyze the comments on [channel]'s last 10 videos and cluster the complaints",
"What is Reddit saying about [topic] this week?",
"Pull the top TikTok videos for [hashtag] and summarize the trend",
"Summarize the top complaints in the Walmart reviews for [product]",
],
},
{

View file

@ -6,6 +6,7 @@ import { instagram } from "./instagram";
import { reddit } from "./reddit";
import { tiktok } from "./tiktok";
import type { ConnectorPageContent } from "./types";
import { walmart } from "./walmart";
import { webCrawl } from "./web-crawl";
import { youtube } from "./youtube";
@ -21,6 +22,7 @@ const CONNECTOR_LIST: ConnectorPageContent[] = [
googleSearch,
indeed,
amazon,
walmart,
webCrawl,
];

View file

@ -0,0 +1,273 @@
import { IconBrandWalmart } from "@tabler/icons-react";
import type { ConnectorPageContent } from "./types";
export const walmart: ConnectorPageContent = {
slug: "walmart",
name: "Walmart",
cardTitle: "Walmart Product & Review API",
icon: IconBrandWalmart,
metaTitle: "Walmart Product & Review API | SurfSense",
metaDescription:
"Scrape public Walmart product data and deep customer reviews as structured JSON: prices, ratings, sellers, variants, availability, and full review history. Start free.",
keywords: [
"walmart product api",
"walmart scraper api",
"walmart review scraper",
"walmart reviews api",
"scrape walmart product data",
"walmart price scraper",
"walmart product data api",
"walmart price tracking api",
"walmart competitor monitoring",
"walmart seller data",
"walmart marketplace api",
"ecommerce product api",
],
h1: "Walmart Product and Review API",
heroLede:
"The SurfSense Walmart API scrapes public Walmart.com listings as structured JSON: price and list price, rating and review count, availability, 1P and marketplace sellers, variants, and specifications. A second verb pages the full public review history — ratings, text, authors, verified-purchase flags, images, and seller responses. Point your AI agents at a search term or product URL — no login, only public data.",
transcript: {
prompt: "Pull the ratings and top complaints for this Walmart air fryer",
toolCall:
'walmart.reviews({ urls: ["walmart.com/ip/..."],\n max_reviews: 500, sort_by: "most-recent" })',
rows: [
{
primary: "Ninja 4-Qt Air Fryer — $79.00 (was $99.00)",
secondary: "4.6 stars · 8,204 reviews · In Stock",
tag: "-20%",
},
{
primary: "512 reviews paged, 71% verified purchase",
secondary: "sorted most-recent · 132 with photos",
tag: "reviews",
},
{
primary: 'Top complaint: "basket coating flaking after 3 months"',
secondary: "surfaced across 24 recent 1-2 star reviews",
tag: "theme",
},
],
resultSummary: "1 product · 512 reviews · surfaced in 6.4s",
},
extractIntro:
"Give the scraper Walmart product, search, category, or browse URLs, or plain search terms. The scrape verb returns product cards and full detail with a free on-page review sample; the reviews verb pages the complete public review history for any product. Walmart data is US marketplace (walmart.com), server-rendered as JSON for stability.",
extractFields: [
{
label: "Product core",
description:
"Name, item id (usItemId), brand, price, list price, availability, in-stock flag, and canonical URL for every product.",
},
{
label: "Ratings & review sample",
description:
"Average stars and total review count on every product, plus a free sample of on-page reviews when you fetch full detail.",
},
{
label: "Deep reviews",
description:
"The reviews verb pages the full public review history: rating, title, text, author, date, verified-purchase flag, helpful votes, images, and seller responses.",
},
{
label: "Sellers",
description:
"The seller behind each product, typed as Walmart first-party (1P) or third-party marketplace (3P), with id and name.",
},
{
label: "Variants & media",
description:
"Product variants (color, size, count) with their own item ids and prices, plus the gallery and thumbnail images.",
},
{
label: "Content & taxonomy",
description:
"Short and long descriptions, specifications, category, and breadcrumb trail for catalog enrichment and classification.",
},
],
useCasesHeading: "What teams do with the Walmart API",
useCases: [
{
title: "Review mining and product research",
description:
"Page the full review history for a product and brief an agent to cluster complaints, track sentiment over time, and separate verified-purchase signal from noise. Walmart exposes far more review depth than most marketplaces — use it to inform your roadmap or listing copy.",
},
{
title: "Price and buy-box tracking",
description:
"Watch your own and competitors' prices, list prices, and which seller wins the offer (Walmart 1P vs a third-party marketplace seller). Run the same URLs on a schedule and diff the results to alert on undercuts and stockouts.",
},
{
title: "Seller and marketplace intelligence",
description:
"See which third-party sellers are winning offers on the products that matter to you, and how first-party Walmart pricing compares, to size the marketplace opportunity in your category.",
},
{
title: "Catalog and assortment enrichment",
description:
"Enrich a catalog by item id with structured attributes, variants, specifications, and images, and discover products from search or category URLs to map an entire assortment.",
},
],
comparison: {
heading: "A Walmart scraper API built for agents",
intro:
"Most Walmart data APIs bill per request, split product and review data across endpoints, and leave the discovery-to-detail logic to you. Here is how SurfSense compares.",
columnLabel: "Typical Walmart data API",
rows: [
{
feature: "Pricing",
official: "Per-request pricing that climbs fast at scale",
surfsense: "Pay per product or per review returned, with a free tier to start",
},
{
feature: "Discovery",
official: "Separate search and detail endpoints you stitch together",
surfsense: "One verb takes search terms or product, search, category, and browse URLs",
},
{
feature: "Reviews",
official: "A separate paid endpoint, often capped shallow",
surfsense: "A dedicated verb pages the full public review history with photos and seller replies",
},
{
feature: "Data access",
official: "Requires accounts, keys, or approval for many providers",
surfsense: "Public, anonymous data only — no login or seller account",
},
{
feature: "Agent-ready",
official: "No; you wire the harness yourself",
surfsense: "MCP server exposes walmart.scrape and walmart.reviews as native tools",
},
],
},
api: {
platform: "walmart",
verb: "scrape",
mcpTool: "walmart.scrape",
requestBody: {
search_terms: ["air fryer"],
max_items: 5,
include_reviews_sample: true,
},
},
schema: {
requestNote:
"walmart.scrape: provide urls or search_terms (at least one), up to 20 combined sources per call. For the full review history, use the walmart.reviews verb with product urls or item_ids.",
request: [
{
name: "urls",
type: "string[]",
description:
"Walmart product (/ip/), search (/search), category (/cp/), or browse (/browse/) URLs. Provide urls or search_terms.",
},
{
name: "search_terms",
type: "string[]",
description:
"Search phrases run on walmart.com, e.g. 'air fryer'. Provide search_terms or urls.",
},
{
name: "max_items",
type: "integer",
defaultValue: "10",
description: "Max products per search term or category/browse URL, 1 to 100.",
},
{
name: "include_details",
type: "boolean",
defaultValue: "true",
description: "Fetch full product detail pages; false returns faster card-only results.",
},
{
name: "include_reviews_sample",
type: "boolean",
defaultValue: "true",
description:
"Attach the free on-page review sample from each detail page. For full history use walmart.reviews.",
},
],
responseNote:
"The response is { items: [...] } with one item per product. One returned product is one billable unit; error items are never billed.",
response: [
{
name: "name / usItemId / brand",
type: "string",
description: "Product identity: display name, Walmart item id, and brand.",
},
{
name: "price / listPrice",
type: "object",
description: "Current price and strike-through list price, each with value and currency.",
},
{
name: "stars / reviewsCount",
type: "number",
description: "Average star rating and total public review count.",
},
{
name: "seller",
type: "object",
description: "Offer seller with id, name, and type (WALMART 1P or MARKETPLACE 3P).",
},
{
name: "availabilityStatus / inStock",
type: "string",
description: "Availability label and a boolean in-stock flag.",
},
{
name: "variants",
type: "object[]",
description: "Product variants (color, size, count) with their own item ids and prices.",
},
{
name: "reviewsSample",
type: "object",
description: "Free sample of on-page reviews returned with full detail pages.",
},
],
},
faq: [
{
question: "What is the Walmart product API?",
answer:
"It returns a Walmart listing's data as structured JSON instead of raw HTML. The SurfSense Walmart API scrapes public product pages and gives you price, rating, seller, variants, availability, and specifications as clean JSON your agents can read.",
},
{
question: "Can I get the full Walmart review history?",
answer:
"Yes. The walmart.reviews verb pages the complete public review history for a product — rating, title, text, author, date, verified-purchase flag, helpful votes, images, and seller responses. Pass product URLs or item ids and set max_reviews and sort_by.",
},
{
question: "Does it only use public data?",
answer:
"It does. The scraper collects public, anonymous Walmart data only — no login or seller account. That covers product details, prices, ratings, public reviews, seller identity, variants, and availability.",
},
{
question: "How do I look up a product by item id or search term?",
answer:
"Pass a product URL (which contains the item id) or a phrase in search_terms to discover products. You can also pass search, category, and browse URLs. walmart.scrape handles discovery; walmart.reviews takes urls or item_ids directly.",
},
{
question: "Which Walmart marketplace is supported?",
answer:
"The scraper targets the US marketplace at walmart.com. Data is read from the server-rendered JSON Walmart ships in the page, which keeps extraction stable against the frequent front-end A/B testing on the site.",
},
],
related: [
{ label: "Amazon Product API", href: "/amazon" },
{ label: "Google Maps API", href: "/google-maps" },
{ label: "Google Search API", href: "/google-search" },
{ label: "Reddit API", href: "/reddit" },
{ label: "SurfSense MCP Server", href: "/mcp-server" },
{ label: "Read the docs", href: "/docs" },
],
};

View file

@ -7,6 +7,7 @@ import {
InstagramIcon,
RedditIcon,
TikTokIcon,
WalmartIcon,
WebIcon,
YouTubeIcon,
} from "./platform-icons";
@ -103,6 +104,15 @@ export const PLAYGROUND_PLATFORMS: PlaygroundPlatform[] = [
icon: AmazonIcon,
verbs: [{ name: "amazon.scrape", verb: "scrape", label: "Scrape" }],
},
{
id: "walmart",
label: "Walmart",
icon: WalmartIcon,
verbs: [
{ name: "walmart.scrape", verb: "scrape", label: "Scrape" },
{ name: "walmart.reviews", verb: "reviews", label: "Reviews" },
],
},
{
id: "web",
label: "Web",

View file

@ -23,6 +23,7 @@ function brandIcon(src: string, alt: string) {
}
export const AmazonIcon = brandIcon("/connectors/amazon.svg", "Amazon");
export const WalmartIcon = brandIcon("/connectors/walmart.svg", "Walmart");
export const RedditIcon = brandIcon("/connectors/reddit.svg", "Reddit");
export const YouTubeIcon = brandIcon("/connectors/youtube.svg", "YouTube");
export const InstagramIcon = brandIcon("/connectors/instagram.svg", "Instagram");

View file

@ -0,0 +1,11 @@
<svg width="48" height="48" viewBox="0 0 48 48" fill="none" xmlns="http://www.w3.org/2000/svg">
<title>Walmart</title>
<g fill="#ffc220">
<rect x="21.4" y="3.5" width="5.2" height="14" rx="2.6" transform="rotate(0 24 24)" />
<rect x="21.4" y="3.5" width="5.2" height="14" rx="2.6" transform="rotate(60 24 24)" />
<rect x="21.4" y="3.5" width="5.2" height="14" rx="2.6" transform="rotate(120 24 24)" />
<rect x="21.4" y="3.5" width="5.2" height="14" rx="2.6" transform="rotate(180 24 24)" />
<rect x="21.4" y="3.5" width="5.2" height="14" rx="2.6" transform="rotate(240 24 24)" />
<rect x="21.4" y="3.5" width="5.2" height="14" rx="2.6" transform="rotate(300 24 24)" />
</g>
</svg>

After

Width:  |  Height:  |  Size: 711 B