mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-22 23:31:12 +02:00
Merge pull request #1614 from CREDO23/feature-walmart-scraper
[Feat] Walmart scraper: product + reviews across API, chat, and MCP
This commit is contained in:
commit
66bf06429e
69 changed files with 2630 additions and 22 deletions
|
|
@ -441,7 +441,7 @@ SURFSENSE_ENABLE_DOOM_LOOP=true
|
|||
# WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE=3000
|
||||
|
||||
# Debit the credit wallet per *item returned* by the platform-native scrapers
|
||||
# (Reddit, Google Search, Google Maps, Amazon, YouTube). Default FALSE keeps scraping
|
||||
# (Reddit, Google Search, Google Maps, Amazon, Walmart, YouTube). Default FALSE keeps scraping
|
||||
# effectively free for self-hosted installs. Each rate is micro-USD per item,
|
||||
# config-driven: <KEY> = round(USD_per_1000_items * 1_000). Defaults sit
|
||||
# at/above Apify's first-party actor rates (we charge no subscription tiers,
|
||||
|
|
@ -459,6 +459,8 @@ SURFSENSE_ENABLE_DOOM_LOOP=true
|
|||
# TIKTOK_MICROS_PER_USER=2500
|
||||
# TIKTOK_MICROS_PER_COMMENT=1500
|
||||
# INDEED_SCRAPE_MICROS_PER_JOB=3500
|
||||
# WALMART_MICROS_PER_PRODUCT=3500
|
||||
# WALMART_MICROS_PER_REVIEW=1500
|
||||
|
||||
# Safety ceiling on per-call premium reservation, in micro-USD ($1.00 default).
|
||||
# QUOTA_MAX_RESERVE_MICROS=1000000
|
||||
|
|
|
|||
|
|
@ -277,8 +277,8 @@ MICROS_PER_PAGE=1000
|
|||
# WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE=3000
|
||||
|
||||
# Debit the credit wallet per *item returned* by the platform-native scrapers
|
||||
# (Reddit, Google Search, Google Maps, Amazon, YouTube). Default FALSE keeps
|
||||
# scraping effectively free for self-hosted/OSS installs; hosted set TRUE.
|
||||
# (Reddit, Google Search, Google Maps, Amazon, Walmart, YouTube). Default FALSE
|
||||
# keeps scraping effectively free for self-hosted/OSS installs; hosted set TRUE.
|
||||
# Each rate is micro-USD per item, fully config-driven (no hardcoded rate):
|
||||
# <KEY> = round(USD_per_1000_items * 1_000)
|
||||
# 3500 == $3.50/1000 | 5000 == $5/1000 | 2000 == $2/1000
|
||||
|
|
@ -302,6 +302,11 @@ MICROS_PER_PAGE=1000
|
|||
# Browser-listing retries when a feed is empty (profile feed is withheld from
|
||||
# flagged IPs; each retry draws a fresh rotating exit IP). Set to 1 for a static IP.
|
||||
# TIKTOK_LISTING_MAX_ATTEMPTS=3
|
||||
# Walmart products (server-rendered JSON behind residential proxies) priced like
|
||||
# Amazon; reviews are 10/page (many light requests) so priced on the per-review
|
||||
# market like Google Maps reviews.
|
||||
# WALMART_MICROS_PER_PRODUCT=3500
|
||||
# WALMART_MICROS_PER_REVIEW=1500
|
||||
|
||||
# Low-balance warning threshold (micro-USD), surfaced to the UI. Default $0.50.
|
||||
CREDIT_LOW_BALANCE_WARNING_MICROS=500000
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ SUBAGENT_TO_REQUIRED_CONNECTOR_MAP: dict[str, frozenset[str]] = {
|
|||
"reddit": frozenset(),
|
||||
"instagram": frozenset(),
|
||||
"tiktok": frozenset(),
|
||||
"walmart": frozenset(),
|
||||
"mcp_discovery": frozenset(
|
||||
{
|
||||
"SLACK_CONNECTOR",
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ are changing, and what is being published across the open web — and to put
|
|||
that research to work alongside their own knowledge base.
|
||||
|
||||
You do this by dispatching **specialist subagents** via the `task` tool:
|
||||
- **Live web data** — Reddit, YouTube, Instagram, TikTok, Amazon, Google
|
||||
- **Live web data** — Reddit, YouTube, Instagram, TikTok, Amazon, Walmart, Google
|
||||
Maps, Google Search, and the web crawler return structured, current
|
||||
platform data (posts, comments, transcripts, videos, products, reviews,
|
||||
SERPs, full page content).
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ are changing, and what is being published across the open web — and to put
|
|||
that research to work alongside the team's shared knowledge base.
|
||||
|
||||
You do this by dispatching **specialist subagents** via the `task` tool:
|
||||
- **Live web data** — Reddit, YouTube, Instagram, TikTok, Amazon, Google
|
||||
- **Live web data** — Reddit, YouTube, Instagram, TikTok, Amazon, Walmart, Google
|
||||
Maps, Google Search, and the web crawler return structured, current
|
||||
platform data (posts, comments, transcripts, videos, products, reviews,
|
||||
SERPs, full page content).
|
||||
|
|
|
|||
|
|
@ -2,7 +2,8 @@
|
|||
CRITICAL — ground factual answers in what you actually receive this turn:
|
||||
- **live platform data** via the market specialists —
|
||||
`task(reddit, ...)`, `task(youtube, ...)`, `task(instagram, ...)`,
|
||||
`task(tiktok, ...)`, `task(amazon, ...)`, `task(google_maps, ...)`,
|
||||
`task(tiktok, ...)`, `task(amazon, ...)`, `task(walmart, ...)`,
|
||||
`task(google_maps, ...)`,
|
||||
`task(google_search, ...)`, `task(web_crawler, ...)`. Anything about
|
||||
competitors, markets, rankings, reviews, or audience sentiment is answered
|
||||
from what these return **this turn**, never from your training data: your
|
||||
|
|
|
|||
|
|
@ -32,7 +32,9 @@ about a brand, product, or topic is answered from the platform where they
|
|||
say it — `task(reddit, …)` for community discussion and threads,
|
||||
`task(youtube, …)` for video content, transcripts, and comment sections,
|
||||
`task(tiktok, …)` for short-form video trends by hashtag or search,
|
||||
`task(google_maps, …)` for customer reviews of physical businesses. Web
|
||||
`task(google_maps, …)` for customer reviews of physical businesses,
|
||||
`task(amazon, …)` / `task(walmart, …)` for product ratings and customer
|
||||
reviews of retail products (Walmart pages the full review history). Web
|
||||
search only finds articles *about* the conversation; the platform
|
||||
specialists return the conversation itself, structured and current. For
|
||||
competitive questions ("what are people saying about X", "how is Y
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
"""``walmart`` builtin subagent: structured public Walmart product data and reviews."""
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
"""``walmart`` route: ``SurfSenseSubagentSpec`` builder for deepagents."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from langchain_core.language_models import BaseChatModel
|
||||
from langchain_core.tools import BaseTool
|
||||
|
||||
from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import (
|
||||
read_md_file,
|
||||
)
|
||||
from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec
|
||||
from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import (
|
||||
pack_subagent,
|
||||
)
|
||||
|
||||
from .tools.index import NAME, RULESET, load_tools
|
||||
|
||||
|
||||
def build_subagent(
|
||||
*,
|
||||
dependencies: dict[str, Any],
|
||||
model: BaseChatModel | None = None,
|
||||
middleware_stack: dict[str, Any] | None = None,
|
||||
mcp_tools: list[BaseTool] | None = None,
|
||||
) -> SurfSenseSubagentSpec:
|
||||
tools = [*load_tools(dependencies=dependencies), *(mcp_tools or [])]
|
||||
description = (
|
||||
read_md_file(__package__, "description").strip()
|
||||
or "Scrapes public Walmart product data and reviews for a URL or search term."
|
||||
)
|
||||
system_prompt = read_md_file(__package__, "system_prompt").strip()
|
||||
return pack_subagent(
|
||||
name=NAME,
|
||||
description=description,
|
||||
system_prompt=system_prompt,
|
||||
tools=tools,
|
||||
ruleset=RULESET,
|
||||
dependencies=dependencies,
|
||||
model=model,
|
||||
middleware_stack=middleware_stack,
|
||||
)
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
Walmart product specialist: scrapes public Walmart listings and returns structured product data — title, item id (usItemId), brand, price and list price, star rating and review count, availability, images, features, seller, and product variants — plus deep paginated customer reviews (rating, text, author, verified-purchase flag, images, and seller responses). Works from a search term (e.g. "air fryer") or from Walmart product (/ip/...), search, category, or browse URLs, and can pull many reviews per product by item id or URL. Only public, anonymous US Walmart data — no login or seller account.
|
||||
Use it for product research, price tracking, catalog enrichment by item id, and review mining. Triggers include "find X on Walmart", "Walmart price of X", "reviews for this Walmart product", "compare these Walmart products", and "look up this Walmart URL/item id". Not for general web search (use the Google Search specialist), reading an arbitrary non-Walmart page (use the web crawling specialist), or other marketplaces.
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
You are the SurfSense Walmart sub-agent.
|
||||
You receive delegated instructions from a supervisor agent and return structured results for supervisor synthesis.
|
||||
|
||||
<goal>
|
||||
Answer the delegated question from public Walmart product data and reviews gathered with your verbs, comparing against earlier results already in this conversation when the task calls for it.
|
||||
</goal>
|
||||
|
||||
<available_tools>
|
||||
- `walmart_scrape` — product details and search/category listings
|
||||
- `walmart_reviews` — deep paginated reviews for a product
|
||||
- `read_run` / `search_run` (free readers for stored scrape output)
|
||||
</available_tools>
|
||||
|
||||
<playbook>
|
||||
- Discovering products: call `walmart_scrape` with `search_terms` (e.g. ["air fryer"]).
|
||||
- Specific products: pass Walmart product URLs (/ip/...) or search/category/browse URLs in `urls`.
|
||||
- Faster listings: set `include_details=false` to return card-only results without opening each product page.
|
||||
- Sampled reviews: `walmart_scrape` returns a small on-page review sample by default (`include_reviews_sample=true`); disable it when reviews are irrelevant.
|
||||
- Deep review mining: use `walmart_reviews` with product `urls` or numeric `item_ids` (usItemId); raise `max_reviews` and set `sort_by` (most-recent, most-helpful, rating-high, rating-low) as the task needs. Reviews are billed per review, so keep `max_reviews` to what the task actually requires.
|
||||
- Batch multiple URLs or search terms into one call rather than many single-source calls.
|
||||
<include snippet="run_reader"/>
|
||||
- Comparison requests: pull the current products, compare against prior values already in this conversation's earlier tool results, and report concrete deltas (price up/down, rating change, stock changes).
|
||||
</playbook>
|
||||
|
||||
<tool_policy>
|
||||
- Use only tools in `<available_tools>`.
|
||||
- Report only results present in the tool output. Never invent titles, item ids, prices, ratings, or reviews.
|
||||
- `walmart_scrape`: provide at least one of `urls` or `search_terms`.
|
||||
- `walmart_reviews`: provide at least one of `urls` or `item_ids`.
|
||||
</tool_policy>
|
||||
|
||||
<out_of_scope>
|
||||
- Do not perform general web search — that is the Google Search specialist's job.
|
||||
- Do not read or extract an arbitrary non-Walmart page — return the URL for the web crawling specialist.
|
||||
- Do not generate deliverables or perform connector mutations; return findings for the supervisor to act on.
|
||||
- Only public, anonymous Walmart data — never anything behind a login or seller account.
|
||||
</out_of_scope>
|
||||
|
||||
<safety>
|
||||
- Report uncertainty explicitly when evidence is incomplete or conflicting.
|
||||
- Never present unverified claims as facts.
|
||||
</safety>
|
||||
|
||||
<failure_policy>
|
||||
- Underspecified request — no usable search term, URL, or item id — return `status=blocked` with the missing fields.
|
||||
- Tool failure: return `status=error` with a concise recovery `next_step`.
|
||||
- No useful evidence: return `status=blocked` with a narrower query or the scope you still need.
|
||||
</failure_policy>
|
||||
|
||||
<output_contract>
|
||||
Return **only** one JSON object (no markdown/prose):
|
||||
{
|
||||
"status": "success" | "partial" | "blocked" | "error",
|
||||
"action_summary": string,
|
||||
"evidence": {
|
||||
"findings": string[],
|
||||
"sources": string[],
|
||||
"confidence": "high" | "medium" | "low"
|
||||
},
|
||||
"next_step": string | null,
|
||||
"missing_fields": string[] | null,
|
||||
"assumptions": string[] | null
|
||||
}
|
||||
<include snippet="output_contract_base"/>
|
||||
Route-specific rules:
|
||||
- `evidence.findings`: max 10 entries, each a single sentence stating one distinct product, review theme, or delta. Do not paste raw payloads.
|
||||
- `evidence.sources`: max 10 URLs, one per finding when applicable. List each URL once.
|
||||
</output_contract>
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
"""``walmart`` sub-agent tools: the Walmart product scrape and reviews verbs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from langchain_core.tools import BaseTool
|
||||
|
||||
from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset
|
||||
from app.capabilities.core.access.agent import build_capability_tools
|
||||
from app.capabilities.walmart.reviews.definition import WALMART_REVIEWS
|
||||
from app.capabilities.walmart.scrape.definition import WALMART_SCRAPE
|
||||
|
||||
NAME = "walmart"
|
||||
|
||||
RULESET = Ruleset(origin=NAME, rules=[])
|
||||
|
||||
_CI_VERBS = [WALMART_SCRAPE, WALMART_REVIEWS]
|
||||
|
||||
|
||||
def load_tools(
|
||||
*, dependencies: dict[str, Any] | None = None, **kwargs: Any
|
||||
) -> list[BaseTool]:
|
||||
d = {**(dependencies or {}), **kwargs}
|
||||
return build_capability_tools(
|
||||
workspace_id=d.get("workspace_id"),
|
||||
capabilities=_CI_VERBS,
|
||||
)
|
||||
|
|
@ -45,6 +45,9 @@ from app.agents.chat.multi_agent_chat.subagents.builtins.reddit.agent import (
|
|||
from app.agents.chat.multi_agent_chat.subagents.builtins.tiktok.agent import (
|
||||
build_subagent as build_tiktok_subagent,
|
||||
)
|
||||
from app.agents.chat.multi_agent_chat.subagents.builtins.walmart.agent import (
|
||||
build_subagent as build_walmart_subagent,
|
||||
)
|
||||
from app.agents.chat.multi_agent_chat.subagents.builtins.web_crawler.agent import (
|
||||
build_subagent as build_web_crawler_subagent,
|
||||
)
|
||||
|
|
@ -100,6 +103,7 @@ SUBAGENT_BUILDERS_BY_NAME: dict[str, SubagentBuilder] = {
|
|||
"onedrive": build_onedrive_subagent,
|
||||
"reddit": build_reddit_subagent,
|
||||
"tiktok": build_tiktok_subagent,
|
||||
"walmart": build_walmart_subagent,
|
||||
"web_crawler": build_web_crawler_subagent,
|
||||
"youtube": build_youtube_subagent,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -42,6 +42,8 @@ _PLATFORM_RATE_KEYS: dict[BillingUnit, str] = {
|
|||
BillingUnit.TIKTOK_USER: "TIKTOK_MICROS_PER_USER",
|
||||
BillingUnit.TIKTOK_COMMENT: "TIKTOK_MICROS_PER_COMMENT",
|
||||
BillingUnit.INDEED_JOB: "INDEED_SCRAPE_MICROS_PER_JOB",
|
||||
BillingUnit.WALMART_PRODUCT: "WALMART_MICROS_PER_PRODUCT",
|
||||
BillingUnit.WALMART_REVIEW: "WALMART_MICROS_PER_REVIEW",
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -65,6 +67,8 @@ _UNIT_NOUNS: dict[BillingUnit, str] = {
|
|||
BillingUnit.TIKTOK_USER: "profile",
|
||||
BillingUnit.TIKTOK_COMMENT: "comment",
|
||||
BillingUnit.INDEED_JOB: "job",
|
||||
BillingUnit.WALMART_PRODUCT: "product",
|
||||
BillingUnit.WALMART_REVIEW: "review",
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -32,6 +32,8 @@ class BillingUnit(StrEnum):
|
|||
TIKTOK_USER = "tiktok_user"
|
||||
TIKTOK_COMMENT = "tiktok_comment"
|
||||
INDEED_JOB = "indeed_job"
|
||||
WALMART_PRODUCT = "walmart_product"
|
||||
WALMART_REVIEW = "walmart_review"
|
||||
|
||||
|
||||
class BillableInput(Protocol):
|
||||
|
|
|
|||
6
surfsense_backend/app/capabilities/walmart/__init__.py
Normal file
6
surfsense_backend/app/capabilities/walmart/__init__.py
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
"""``walmart.*`` namespace: platform-native Walmart data verbs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.capabilities.walmart.reviews import definition as _reviews # noqa: F401
|
||||
from app.capabilities.walmart.scrape import definition as _scrape # noqa: F401
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
"""Walmart deep review scraping capability."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
"""``walmart.reviews`` capability registration (billed per review; see config
|
||||
``WALMART_MICROS_PER_REVIEW``)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.capabilities.core import BillingUnit, Capability, register_capability
|
||||
from app.capabilities.walmart.reviews.executor import build_reviews_executor
|
||||
from app.capabilities.walmart.reviews.schemas import ReviewsInput, ReviewsOutput
|
||||
|
||||
WALMART_REVIEWS = Capability(
|
||||
name="walmart.reviews",
|
||||
description=(
|
||||
"Fetch deep paginated public Walmart product reviews with ratings, text, "
|
||||
"authors, verified-purchase flags, images, and seller responses. Use "
|
||||
"product urls or item ids."
|
||||
),
|
||||
input_schema=ReviewsInput,
|
||||
output_schema=ReviewsOutput,
|
||||
executor=build_reviews_executor(),
|
||||
billing_unit=BillingUnit.WALMART_REVIEW,
|
||||
docs_url="/docs/connectors/native/walmart",
|
||||
)
|
||||
|
||||
register_capability(WALMART_REVIEWS)
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
"""``walmart.reviews`` executor: verb input → scraper → review items."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Awaitable, Callable
|
||||
|
||||
from app.capabilities.core import Executor
|
||||
from app.capabilities.core.progress import emit_progress
|
||||
from app.capabilities.walmart.reviews.schemas import ReviewsInput, ReviewsOutput
|
||||
from app.proprietary.platforms.walmart import WalmartReviewsInput, scrape_reviews
|
||||
|
||||
ReviewsFn = Callable[..., Awaitable[list[dict]]]
|
||||
|
||||
|
||||
def build_reviews_executor(scrape_fn: ReviewsFn | None = None) -> Executor:
|
||||
"""Bind the capability input mapping to a replaceable reviews scraper function."""
|
||||
scrape_fn = scrape_fn or scrape_reviews
|
||||
|
||||
async def execute(payload: ReviewsInput) -> ReviewsOutput:
|
||||
input_model = WalmartReviewsInput(
|
||||
itemIds=payload.sources(),
|
||||
maxReviews=payload.max_reviews,
|
||||
sort=payload.sort_by,
|
||||
)
|
||||
emit_progress(
|
||||
"starting",
|
||||
"Fetching Walmart reviews",
|
||||
total=payload.estimated_units,
|
||||
unit="review",
|
||||
)
|
||||
items = await scrape_fn(input_model, limit=payload.estimated_units)
|
||||
emit_progress(
|
||||
"done",
|
||||
f"Scraped {sum('error' not in item for item in items)} review(s)",
|
||||
current=len(items),
|
||||
unit="review",
|
||||
)
|
||||
return ReviewsOutput(items=items)
|
||||
|
||||
return execute
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
"""``walmart.reviews`` I/O contracts.
|
||||
|
||||
A lean surface over ``WalmartReviewsInput``; the scraper's ``ReviewItem`` is
|
||||
reused verbatim as the output element. Accepts product URLs or bare item ids —
|
||||
both resolve to a ``usItemId`` the reviews page is keyed on.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, Field, model_validator
|
||||
|
||||
from app.proprietary.platforms.walmart import ReviewItem
|
||||
|
||||
MAX_WALMART_REVIEW_SOURCES = 20
|
||||
|
||||
|
||||
class ReviewsInput(BaseModel):
|
||||
urls: list[str] = Field(
|
||||
default_factory=list,
|
||||
max_length=MAX_WALMART_REVIEW_SOURCES,
|
||||
description=(
|
||||
"Walmart product URLs (/ip/...) or reviews URLs to fetch reviews for. "
|
||||
"Provide these OR item_ids (at least one is required)."
|
||||
),
|
||||
)
|
||||
item_ids: list[str] = Field(
|
||||
default_factory=list,
|
||||
max_length=MAX_WALMART_REVIEW_SOURCES,
|
||||
description="Walmart numeric item ids (usItemId) to fetch reviews for.",
|
||||
)
|
||||
max_reviews: int = Field(
|
||||
default=200,
|
||||
ge=1,
|
||||
le=5000,
|
||||
description="Max reviews to return per product (10 per page).",
|
||||
)
|
||||
sort_by: Literal["most-recent", "most-helpful", "rating-high", "rating-low"] = (
|
||||
Field(default="most-recent", description="Review ordering.")
|
||||
)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _require_a_source(self) -> ReviewsInput:
|
||||
if not (self.urls or self.item_ids):
|
||||
raise ValueError("Provide at least one of 'urls' or 'item_ids'.")
|
||||
return self
|
||||
|
||||
def sources(self) -> list[str]:
|
||||
"""URLs and item ids merged; the scraper resolves each to a usItemId."""
|
||||
return [*self.urls, *self.item_ids]
|
||||
|
||||
@property
|
||||
def estimated_units(self) -> int:
|
||||
"""Worst-case billable reviews: up to ``max_reviews`` per source."""
|
||||
return (len(self.urls) + len(self.item_ids)) * self.max_reviews
|
||||
|
||||
|
||||
class ReviewsOutput(BaseModel):
|
||||
items: list[ReviewItem] = Field(
|
||||
default_factory=list,
|
||||
description="One item per review, in the scraper's emission order.",
|
||||
)
|
||||
|
||||
@property
|
||||
def billable_units(self) -> int:
|
||||
"""One returned review = one billable unit; error items are not billed."""
|
||||
return sum(
|
||||
1
|
||||
for item in self.items
|
||||
if not (item.model_extra and item.model_extra.get("error"))
|
||||
)
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
"""Walmart product scraping capability."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
"""Registration for the ``walmart.scrape`` capability."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.capabilities.core import BillingUnit, Capability, register_capability
|
||||
from app.capabilities.walmart.scrape.executor import build_scrape_executor
|
||||
from app.capabilities.walmart.scrape.schemas import ScrapeInput, ScrapeOutput
|
||||
|
||||
WALMART_SCRAPE = Capability(
|
||||
name="walmart.scrape",
|
||||
description=(
|
||||
"Scrape public Walmart product details, search/category listings, "
|
||||
"prices, sellers, variants, availability, and a sample of on-page reviews."
|
||||
),
|
||||
input_schema=ScrapeInput,
|
||||
output_schema=ScrapeOutput,
|
||||
executor=build_scrape_executor(),
|
||||
billing_unit=BillingUnit.WALMART_PRODUCT,
|
||||
docs_url="/docs/connectors/native/walmart",
|
||||
)
|
||||
|
||||
register_capability(WALMART_SCRAPE)
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
"""Executor for the ``walmart.scrape`` capability."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Awaitable, Callable
|
||||
|
||||
from app.capabilities.core import Executor
|
||||
from app.capabilities.core.progress import emit_progress
|
||||
from app.capabilities.walmart.scrape.schemas import (
|
||||
MAX_WALMART_RESULTS,
|
||||
ScrapeInput,
|
||||
ScrapeOutput,
|
||||
)
|
||||
from app.proprietary.platforms.walmart import WalmartScrapeInput, scrape_products
|
||||
|
||||
ScrapeFn = Callable[..., Awaitable[list[dict]]]
|
||||
|
||||
|
||||
def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor:
|
||||
"""Bind the capability input mapping to a replaceable scraper function."""
|
||||
scrape_fn = scrape_fn or scrape_products
|
||||
|
||||
async def execute(payload: ScrapeInput) -> ScrapeOutput:
|
||||
input_model = WalmartScrapeInput(
|
||||
startUrls=payload.start_urls(),
|
||||
maxItemsPerStartUrl=payload.max_items,
|
||||
includeDetails=payload.include_details,
|
||||
includeReviewsSample=payload.include_reviews_sample,
|
||||
)
|
||||
emit_progress(
|
||||
"starting",
|
||||
"Scraping Walmart products",
|
||||
total=payload.estimated_units,
|
||||
unit="product",
|
||||
)
|
||||
items = await scrape_fn(input_model, limit=MAX_WALMART_RESULTS)
|
||||
emit_progress(
|
||||
"done",
|
||||
f"Scraped {sum('error' not in item for item in items)} product(s)",
|
||||
current=len(items),
|
||||
total=payload.estimated_units,
|
||||
unit="product",
|
||||
)
|
||||
return ScrapeOutput(items=items)
|
||||
|
||||
return execute
|
||||
64
surfsense_backend/app/capabilities/walmart/scrape/schemas.py
Normal file
64
surfsense_backend/app/capabilities/walmart/scrape/schemas.py
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
"""Input and output contracts for ``walmart.scrape``."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
from pydantic import BaseModel, Field, model_validator
|
||||
|
||||
from app.proprietary.platforms.walmart import ProductItem
|
||||
|
||||
MAX_WALMART_SOURCES = 20
|
||||
MAX_WALMART_RESULTS = 1000
|
||||
|
||||
|
||||
class ScrapeInput(BaseModel):
|
||||
"""Agent-facing controls for public Walmart product discovery and enrichment."""
|
||||
|
||||
urls: list[str] = Field(default_factory=list, max_length=MAX_WALMART_SOURCES)
|
||||
search_terms: list[str] = Field(
|
||||
default_factory=list, max_length=MAX_WALMART_SOURCES
|
||||
)
|
||||
max_items: int = Field(default=10, ge=1, le=100)
|
||||
include_details: bool = True
|
||||
include_reviews_sample: bool = True
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _require_source(self) -> ScrapeInput:
|
||||
if not (self.urls or self.search_terms):
|
||||
raise ValueError("Provide at least one URL or search term.")
|
||||
if len(self.urls) + len(self.search_terms) > MAX_WALMART_SOURCES:
|
||||
raise ValueError(
|
||||
f"Provide no more than {MAX_WALMART_SOURCES} combined sources."
|
||||
)
|
||||
return self
|
||||
|
||||
def start_urls(self) -> list[str]:
|
||||
"""Direct URLs plus a search URL synthesized per search term."""
|
||||
searches = [
|
||||
f"https://www.walmart.com/search?q={quote_plus(term)}"
|
||||
for term in self.search_terms
|
||||
]
|
||||
return [*self.urls, *searches]
|
||||
|
||||
@property
|
||||
def estimated_units(self) -> int:
|
||||
"""Worst-case returned products within the hard per-run ceiling."""
|
||||
search_products = len(self.search_terms) * self.max_items
|
||||
direct_products = len(self.urls) * self.max_items
|
||||
return min(search_products + direct_products, MAX_WALMART_RESULTS)
|
||||
|
||||
|
||||
class ScrapeOutput(BaseModel):
|
||||
"""Products and structured per-input errors in emission order."""
|
||||
|
||||
items: list[ProductItem] = Field(default_factory=list)
|
||||
|
||||
@property
|
||||
def billable_units(self) -> int:
|
||||
"""Count successful products; error items are never billed."""
|
||||
return sum(
|
||||
1
|
||||
for item in self.items
|
||||
if not (item.model_extra and item.model_extra.get("error"))
|
||||
)
|
||||
|
|
@ -741,6 +741,12 @@ class Config:
|
|||
INDEED_SCRAPE_MICROS_PER_JOB = int(
|
||||
os.getenv("INDEED_SCRAPE_MICROS_PER_JOB", "3500")
|
||||
)
|
||||
# Walmart products come from server-rendered JSON behind residential proxies,
|
||||
# priced alongside Amazon's per-product meter.
|
||||
WALMART_MICROS_PER_PRODUCT = int(os.getenv("WALMART_MICROS_PER_PRODUCT", "3500"))
|
||||
# Reviews are 10 per page (many light requests per product), priced on the
|
||||
# cheaper per-review market like the Google Maps review meter.
|
||||
WALMART_MICROS_PER_REVIEW = int(os.getenv("WALMART_MICROS_PER_REVIEW", "1500"))
|
||||
# Retry an empty listing draw on a fresh rotating IP. Set to 1 for a static
|
||||
# proxy, where every retry re-hits the same exit.
|
||||
TIKTOK_LISTING_MAX_ATTEMPTS = int(os.getenv("TIKTOK_LISTING_MAX_ATTEMPTS", "3"))
|
||||
|
|
|
|||
|
|
@ -0,0 +1,52 @@
|
|||
# Walmart Scraper
|
||||
|
||||
Two verbs read Walmart's public, anonymous pages: `walmart.scrape` (products +
|
||||
search/category/browse listings, per-product billing) and `walmart.reviews`
|
||||
(deep paginated reviews, per-review billing).
|
||||
|
||||
## Scope
|
||||
|
||||
The scraper reads public pages available to anonymous visitors — no login, no
|
||||
account cookies. Data is extracted from the Next.js `__NEXT_DATA__` JSON blob
|
||||
embedded in each page (with an `__APP_DATA__` fallback), not from the rendered
|
||||
DOM, because Walmart obfuscates CSS classes and A/B-tests layout constantly.
|
||||
|
||||
`walmart.scrape` returns a free sample of on-page reviews (rating distribution,
|
||||
aspects, top reviews) under `reviewsSample`. `walmart.reviews` fetches the full
|
||||
review history from the public `/reviews/product/{usItemId}` page, which
|
||||
robots.txt permits (unlike `/search`).
|
||||
|
||||
## Architecture
|
||||
|
||||
- `schemas.py` defines the stable input, product, review, and error models.
|
||||
- `url_resolver.py` classifies product (`/ip/`) vs listing (`/search`, `/cp/`,
|
||||
`/browse/`) URLs and extracts the numeric `usItemId`.
|
||||
- `next_data.py` extracts and navigates the hidden Next.js JSON state.
|
||||
- `fetch.py` owns proxy-aware HTTP access (US-pinned), block detection, and
|
||||
retries.
|
||||
- `parsers.py` contains pure, defensive JSON parsers.
|
||||
- `scraper.py` coordinates discovery, enrichment, pagination, concurrency,
|
||||
limits, and in-stream error items.
|
||||
|
||||
## Anti-bot
|
||||
|
||||
Walmart runs Akamai (edge/TLS) + PerimeterX/HUMAN (behavioral JS). Requests go
|
||||
through US residential proxies with TLS-impersonated headers; blocked responses
|
||||
(body markers, `412`/`429`/`503`, or the `200`-OK CAPTCHA body) rotate to a
|
||||
fresh proxy exit.
|
||||
|
||||
Known ceilings and upgrade paths (see `fetch.py` / `scraper.py` `ponytail:`
|
||||
notes): reviews page at 10/page; search capped at Walmart's 25-page limit;
|
||||
session warming (seed `_px3`/`_pxhd` on a sticky exit) is the next lever if
|
||||
block rates on the SSR pages climb, and the `/orchestra/*` GraphQL API is
|
||||
deliberately avoided (rotating persisted-query hashes make it brittle).
|
||||
|
||||
## Verification
|
||||
|
||||
Offline fixtures cover the parsers and both flows:
|
||||
|
||||
uv run pytest tests/unit/platforms/walmart/
|
||||
|
||||
A manual live check (requires network + residential proxy):
|
||||
|
||||
uv run python scripts/e2e_walmart_scraper.py
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
"""Platform-native Walmart scraper (products, listings, reviews)."""
|
||||
|
||||
from .schemas import (
|
||||
ErrorItem,
|
||||
ProductItem,
|
||||
ReviewItem,
|
||||
WalmartReviewsInput,
|
||||
WalmartScrapeInput,
|
||||
)
|
||||
from .scraper import (
|
||||
iter_products,
|
||||
iter_reviews,
|
||||
scrape_products,
|
||||
scrape_reviews,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"ErrorItem",
|
||||
"ProductItem",
|
||||
"ReviewItem",
|
||||
"WalmartReviewsInput",
|
||||
"WalmartScrapeInput",
|
||||
"iter_products",
|
||||
"iter_reviews",
|
||||
"scrape_products",
|
||||
"scrape_reviews",
|
||||
]
|
||||
171
surfsense_backend/app/proprietary/platforms/walmart/fetch.py
Normal file
171
surfsense_backend/app/proprietary/platforms/walmart/fetch.py
Normal file
|
|
@ -0,0 +1,171 @@
|
|||
"""Network access for public Walmart pages.
|
||||
|
||||
Mirrors the Amazon fetch layer: every request goes through the configured
|
||||
residential proxy (pinned to the US, since Walmart geo-locks inventory), and any
|
||||
response that looks like an anti-bot interstitial is retried on a fresh proxy
|
||||
exit. Ordinary HTTP failures are returned to the caller for domain-specific
|
||||
handling.
|
||||
|
||||
Walmart runs Akamai (edge/TLS) + PerimeterX/HUMAN (behavioral JS). Two Walmart
|
||||
specifics differ from Amazon:
|
||||
|
||||
* Walmart serves CAPTCHA with an HTTP ``200`` body ("Robot or human?"), so block
|
||||
detection scans the body, never the status alone.
|
||||
* ``412`` is PerimeterX's rejection code and is treated as blocked → rotate.
|
||||
|
||||
``ponytail:`` MVP hits only the server-rendered ``__NEXT_DATA__`` pages, which
|
||||
TLS impersonation + residential proxies clear without seeding PerimeterX cookies.
|
||||
If block rates on those pages climb, the upgrade path is a warmed sticky session
|
||||
(seed ``_px3``/``_pxhd``/``ACID`` from a homepage fetch, reuse exit + cookies) —
|
||||
the same shape as Amazon's ``get_location_session``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from scrapling.fetchers import AsyncFetcher
|
||||
|
||||
from app.utils.proxy import get_geo_proxy_url, get_sticky_proxy_url
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_MAX_IP_ATTEMPTS = 6
|
||||
_REQUEST_TIMEOUT_S = 30
|
||||
_HEADERS = {
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
}
|
||||
_BLOCK_MARKERS = (
|
||||
"robot or human",
|
||||
"px-captcha",
|
||||
"/blocked",
|
||||
"verify you are a human",
|
||||
"access to this page has been denied",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FetchResult:
|
||||
"""The response details needed by scraper flows."""
|
||||
|
||||
status: int
|
||||
html: str
|
||||
url: str
|
||||
cookies: dict[str, str]
|
||||
headers: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
|
||||
async def gather_bounded[T](
|
||||
factories: list[Callable[[], Awaitable[T]]], *, concurrency: int
|
||||
) -> list[T]:
|
||||
"""Run async factories concurrently while preserving input order."""
|
||||
if not factories:
|
||||
return []
|
||||
semaphore = asyncio.Semaphore(max(1, concurrency))
|
||||
|
||||
async def run(factory: Callable[[], Awaitable[T]]) -> T:
|
||||
async with semaphore:
|
||||
return await factory()
|
||||
|
||||
return await asyncio.gather(*(run(factory) for factory in factories))
|
||||
|
||||
|
||||
def is_blocked(
|
||||
html: str | None, status: int, headers: dict[str, str] | None = None
|
||||
) -> bool:
|
||||
"""Return whether a response is a Walmart anti-bot interstitial.
|
||||
|
||||
``412`` is PerimeterX's rejection; ``429``/``503`` are throttles. Walmart also
|
||||
serves CAPTCHA with a ``200`` body, so the body is scanned regardless of
|
||||
status.
|
||||
"""
|
||||
if status in {412, 429, 503}:
|
||||
return True
|
||||
text = (html or "")[:200_000].lower()
|
||||
return any(marker in text for marker in _BLOCK_MARKERS)
|
||||
|
||||
|
||||
def _response_url(page: Any, fallback: str) -> str:
|
||||
value = getattr(page, "url", None)
|
||||
return str(value) if value else fallback
|
||||
|
||||
|
||||
def _response_cookies(page: Any) -> dict[str, str]:
|
||||
cookies = getattr(page, "cookies", None)
|
||||
return dict(cookies) if isinstance(cookies, dict) else {}
|
||||
|
||||
|
||||
def _response_headers(page: Any) -> dict[str, str]:
|
||||
headers = getattr(page, "headers", None)
|
||||
return dict(headers) if isinstance(headers, dict) else {}
|
||||
|
||||
|
||||
def _selected_proxy(
|
||||
proxy: str | None, country: str, attempt: int, url: str
|
||||
) -> str | None:
|
||||
if proxy is not None:
|
||||
return proxy
|
||||
if attempt > 1:
|
||||
session_id = f"walmart-{attempt}-{abs(hash((url, time.time_ns()))):x}"
|
||||
return get_sticky_proxy_url(session_id, country)
|
||||
return get_geo_proxy_url(country)
|
||||
|
||||
|
||||
async def fetch_page(
|
||||
url: str,
|
||||
*,
|
||||
cookies: dict[str, str] | None = None,
|
||||
proxy: str | None = None,
|
||||
country: str = "us",
|
||||
rotate_on_block: bool = True,
|
||||
) -> FetchResult | None:
|
||||
"""Fetch a page and retry blocked responses with fresh proxy exits."""
|
||||
attempts = _MAX_IP_ATTEMPTS if rotate_on_block else 1
|
||||
for attempt in range(1, attempts + 1):
|
||||
selected_proxy = _selected_proxy(proxy, country, attempt, url)
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
page = await AsyncFetcher.get(
|
||||
url,
|
||||
headers={**_HEADERS},
|
||||
cookies=cookies or {},
|
||||
proxy=selected_proxy,
|
||||
stealthy_headers=True,
|
||||
timeout=_REQUEST_TIMEOUT_S,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("Walmart request failed for %s: %s", url, exc)
|
||||
if proxy is not None:
|
||||
return None
|
||||
continue
|
||||
|
||||
status = int(getattr(page, "status", 0) or 0)
|
||||
html = getattr(page, "html_content", None) or ""
|
||||
response_headers = _response_headers(page)
|
||||
logger.info(
|
||||
"[walmart][perf] status=%s attempt=%s fetch_ms=%.1f url=%s",
|
||||
status,
|
||||
attempt,
|
||||
(time.perf_counter() - started) * 1000,
|
||||
url,
|
||||
)
|
||||
if rotate_on_block and is_blocked(html, status, response_headers):
|
||||
logger.info(
|
||||
"Walmart blocked proxy attempt %s/%s for %s", attempt, attempts, url
|
||||
)
|
||||
continue
|
||||
return FetchResult(
|
||||
status=status,
|
||||
html=html,
|
||||
url=_response_url(page, url),
|
||||
cookies=_response_cookies(page),
|
||||
headers=response_headers,
|
||||
)
|
||||
logger.warning("Walmart exhausted %s proxy attempts for %s", attempts, url)
|
||||
return None
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
"""Extract Walmart's hidden Next.js JSON state from page HTML.
|
||||
|
||||
Every serious Walmart scraper (Scrapfly, Oxylabs, ScrapeOps, Apify) reads the
|
||||
``<script id="__NEXT_DATA__">`` JSON blob rather than parsing the rendered DOM,
|
||||
because Walmart obfuscates CSS classes and A/B-tests layout constantly. The JSON
|
||||
shape is far more stable across redesigns.
|
||||
|
||||
``__APP_DATA__`` is the documented fallback anchor Walmart occasionally ships
|
||||
instead during layout experiments.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_NEXT_DATA_RE = re.compile(r'<script id="__NEXT_DATA__"[^>]*>(.*?)</script>', re.DOTALL)
|
||||
_APP_DATA_RE = re.compile(r'<script id="__APP_DATA__"[^>]*>(.*?)</script>', re.DOTALL)
|
||||
|
||||
|
||||
def extract_next_data(html: str | None) -> dict[str, Any] | None:
|
||||
"""Return the parsed Next.js state object, or ``None`` when absent/invalid.
|
||||
|
||||
Tries ``__NEXT_DATA__`` first, then the ``__APP_DATA__`` fallback so a single
|
||||
Walmart layout experiment does not blank the whole extractor.
|
||||
"""
|
||||
if not html:
|
||||
return None
|
||||
for pattern in (_NEXT_DATA_RE, _APP_DATA_RE):
|
||||
match = pattern.search(html)
|
||||
if not match:
|
||||
continue
|
||||
try:
|
||||
data = json.loads(match.group(1))
|
||||
except (ValueError, TypeError):
|
||||
logger.warning("Walmart hidden JSON present but did not parse")
|
||||
continue
|
||||
if isinstance(data, dict):
|
||||
return data
|
||||
return None
|
||||
|
||||
|
||||
def dig(obj: Any, *keys: str | int) -> Any:
|
||||
"""Walk nested dict/list keys, returning ``None`` on any miss.
|
||||
|
||||
Tolerates the layout drift between Walmart's ``initialData`` variants without
|
||||
a cascade of ``if key in ...`` guards at every call site.
|
||||
"""
|
||||
current = obj
|
||||
for key in keys:
|
||||
if isinstance(key, int):
|
||||
if not isinstance(current, list) or not -len(current) <= key < len(current):
|
||||
return None
|
||||
current = current[key]
|
||||
else:
|
||||
if not isinstance(current, dict) or key not in current:
|
||||
return None
|
||||
current = current[key]
|
||||
return current
|
||||
|
||||
|
||||
def initial_data(next_data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""The ``props.pageProps.initialData`` node shared by every page type."""
|
||||
node = dig(next_data, "props", "pageProps", "initialData")
|
||||
return node if isinstance(node, dict) else None
|
||||
246
surfsense_backend/app/proprietary/platforms/walmart/parsers.py
Normal file
246
surfsense_backend/app/proprietary/platforms/walmart/parsers.py
Normal file
|
|
@ -0,0 +1,246 @@
|
|||
"""Pure parsers for Walmart's hidden ``__NEXT_DATA__`` JSON.
|
||||
|
||||
Product, listing, and review data all live in the same Next.js state tree; these
|
||||
functions navigate it defensively (via :func:`.next_data.dig`) and normalize the
|
||||
fields into the stable output shape. Missing sections yield ``None``/``[]`` so an
|
||||
isolated schema change never discards an otherwise usable record.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from .next_data import dig, initial_data
|
||||
|
||||
_WALMART_ORIGIN = "https://www.walmart.com"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Shared helpers. #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _price(node: Any) -> dict[str, Any] | None:
|
||||
"""Normalize a Walmart price node into ``{value, currency}``."""
|
||||
if not isinstance(node, dict):
|
||||
return None
|
||||
value = node.get("price")
|
||||
currency = node.get("currencyUnit")
|
||||
if value is None and currency is None:
|
||||
return None
|
||||
return {"value": value, "currency": currency}
|
||||
|
||||
|
||||
def _seller(raw: dict[str, Any]) -> dict[str, Any] | None:
|
||||
name = raw.get("sellerName") or raw.get("sellerDisplayName")
|
||||
seller_id = raw.get("sellerId")
|
||||
if not name and not seller_id:
|
||||
return None
|
||||
is_walmart = bool(name) and "walmart" in name.lower()
|
||||
return {
|
||||
"id": seller_id,
|
||||
"name": name,
|
||||
"type": "WALMART" if is_walmart else "MARKETPLACE",
|
||||
}
|
||||
|
||||
|
||||
def _absolute(url: str | None) -> str | None:
|
||||
return urljoin(_WALMART_ORIGIN, url) if url else None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Listing / search cards. #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _listing_card(raw: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Normalize one search/category result item into a product card."""
|
||||
item_id = raw.get("usItemId") or raw.get("id")
|
||||
name = raw.get("name")
|
||||
if not item_id or not name:
|
||||
return None
|
||||
price_info = raw.get("priceInfo") or {}
|
||||
availability = raw.get("availabilityStatusV2") or {}
|
||||
image = raw.get("imageInfo") or {}
|
||||
return {
|
||||
"usItemId": str(item_id),
|
||||
"name": name,
|
||||
"brand": raw.get("brand"),
|
||||
"url": _absolute(raw.get("canonicalUrl")),
|
||||
"price": _price(price_info.get("currentPrice"))
|
||||
or ({"value": raw.get("price")} if raw.get("price") is not None else None),
|
||||
"listPrice": _price(price_info.get("wasPrice")),
|
||||
"stars": raw.get("averageRating"),
|
||||
"reviewsCount": raw.get("numberOfReviews"),
|
||||
"seller": _seller(raw),
|
||||
"availabilityStatus": availability.get("value")
|
||||
or ("OUT_OF_STOCK" if raw.get("isOutOfStock") else None),
|
||||
"inStock": (availability.get("value") == "IN_STOCK")
|
||||
if availability.get("value")
|
||||
else (raw.get("isOutOfStock") is False if "isOutOfStock" in raw else None),
|
||||
"thumbnailImage": image.get("thumbnailUrl") or raw.get("image"),
|
||||
"sponsored": raw.get("isSponsoredFlag"),
|
||||
}
|
||||
|
||||
|
||||
def parse_listing_page(next_data: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
"""Extract normalized product cards from a search/category/browse page."""
|
||||
data = initial_data(next_data)
|
||||
if data is None:
|
||||
return []
|
||||
stacks = dig(data, "searchResult", "itemStacks")
|
||||
if not isinstance(stacks, list):
|
||||
return []
|
||||
cards: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
for stack in stacks:
|
||||
for raw in (stack or {}).get("items") or []:
|
||||
if not isinstance(raw, dict) or raw.get("__typename") not in (
|
||||
None,
|
||||
"Product",
|
||||
):
|
||||
continue
|
||||
card = _listing_card(raw)
|
||||
if card and card["usItemId"] not in seen:
|
||||
seen.add(card["usItemId"])
|
||||
cards.append(card)
|
||||
return cards
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Product detail. #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _images(image_info: dict[str, Any]) -> list[str]:
|
||||
images: list[str] = []
|
||||
for entry in image_info.get("allImages") or []:
|
||||
url = entry.get("url") if isinstance(entry, dict) else None
|
||||
if url and url not in images:
|
||||
images.append(url)
|
||||
return images
|
||||
|
||||
|
||||
def _breadcrumbs(product: dict[str, Any]) -> list[str]:
|
||||
"""Category breadcrumb names, root→leaf.
|
||||
|
||||
Walmart ships ``category.path`` as a list of ``{name, url}`` nodes, e.g.
|
||||
``[{"name": "Home Improvement"}, ..., {"name": "Air Conditioners"}]`` — not
|
||||
a string.
|
||||
"""
|
||||
path = dig(product, "category", "path")
|
||||
if not isinstance(path, list):
|
||||
return []
|
||||
return [node["name"] for node in path if isinstance(node, dict) and node.get("name")]
|
||||
|
||||
|
||||
def _reviews_sample(
|
||||
reviews: dict[str, Any] | None, limit: int = 10
|
||||
) -> dict[str, Any] | None:
|
||||
if not isinstance(reviews, dict):
|
||||
return None
|
||||
customer = reviews.get("customerReviews") or []
|
||||
return {
|
||||
"averageOverallRating": reviews.get("averageOverallRating"),
|
||||
"totalReviewCount": reviews.get("totalReviewCount"),
|
||||
"aspects": reviews.get("aspects") or [],
|
||||
"topReviews": [
|
||||
normalize_review(r) for r in customer[:limit] if isinstance(r, dict)
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def parse_product(
|
||||
next_data: dict[str, Any], *, url: str, include_reviews_sample: bool = True
|
||||
) -> dict[str, Any]:
|
||||
"""Extract normalized product fields from a product detail page."""
|
||||
data = initial_data(next_data)
|
||||
product = dig(data, "data", "product") if data else None
|
||||
if not isinstance(product, dict):
|
||||
return {}
|
||||
idml = dig(data, "data", "idml") if data else None
|
||||
reviews = dig(data, "data", "reviews") if data else None
|
||||
price_info = product.get("priceInfo") or {}
|
||||
image_info = product.get("imageInfo") or {}
|
||||
availability = product.get("availabilityStatus")
|
||||
breadcrumbs = _breadcrumbs(product)
|
||||
|
||||
fields: dict[str, Any] = {
|
||||
"usItemId": str(product.get("usItemId") or product.get("id") or "") or None,
|
||||
"name": product.get("name"),
|
||||
"brand": product.get("brand"),
|
||||
"url": url,
|
||||
"price": _price(price_info.get("currentPrice")),
|
||||
"listPrice": _price(price_info.get("wasPrice")),
|
||||
"currency": dig(price_info, "currentPrice", "currencyUnit"),
|
||||
"availabilityStatus": availability,
|
||||
"inStock": availability == "IN_STOCK" if availability else None,
|
||||
"stars": product.get("averageRating"),
|
||||
"reviewsCount": product.get("numberOfReviews"),
|
||||
"seller": _seller(product),
|
||||
"manufacturerName": product.get("manufacturerName"),
|
||||
"shortDescription": product.get("shortDescription"),
|
||||
"longDescription": (idml or {}).get("longDescription")
|
||||
if isinstance(idml, dict)
|
||||
else None,
|
||||
"thumbnailImage": image_info.get("thumbnailUrl"),
|
||||
"images": _images(image_info),
|
||||
"category": breadcrumbs[-1] if breadcrumbs else None,
|
||||
"breadCrumbs": breadcrumbs,
|
||||
"variants": product.get("variantCriteria") or [],
|
||||
}
|
||||
if include_reviews_sample:
|
||||
fields["reviewsSample"] = _reviews_sample(reviews)
|
||||
return {key: value for key, value in fields.items() if value is not None}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Reviews (deep pagination). #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def normalize_review(raw: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Normalize one Walmart ``customerReviews`` record into a ``ReviewItem`` dict."""
|
||||
badges = raw.get("badges") or []
|
||||
verified = any(
|
||||
isinstance(b, dict) and b.get("id") == "VerifiedPurchaser" for b in badges
|
||||
)
|
||||
photos = raw.get("photos") or raw.get("media") or []
|
||||
images: list[str] = []
|
||||
for photo in photos:
|
||||
if not isinstance(photo, dict):
|
||||
continue
|
||||
url = photo.get("normalUrl") or dig(photo, "sizes", "normal", "url")
|
||||
if url and url not in images:
|
||||
images.append(url)
|
||||
responses = raw.get("clientResponses") or []
|
||||
seller_response = None
|
||||
if responses and isinstance(responses[0], dict):
|
||||
seller_response = responses[0].get("response") or responses[0].get(
|
||||
"responseText"
|
||||
)
|
||||
return {
|
||||
"reviewId": raw.get("reviewId"),
|
||||
"rating": raw.get("rating"),
|
||||
"title": raw.get("reviewTitle"),
|
||||
"text": raw.get("reviewText"),
|
||||
"submissionTime": raw.get("reviewSubmissionTime"),
|
||||
"author": raw.get("userNickname"),
|
||||
"verifiedPurchase": verified,
|
||||
"positiveFeedback": raw.get("positiveFeedback"),
|
||||
"negativeFeedback": raw.get("negativeFeedback"),
|
||||
"images": images,
|
||||
"syndicated": bool(raw.get("syndicationSource")),
|
||||
"sellerResponse": seller_response,
|
||||
}
|
||||
|
||||
|
||||
def parse_reviews_page(next_data: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
"""Extract normalized reviews from one ``/reviews/product/{id}`` page."""
|
||||
data = initial_data(next_data)
|
||||
reviews = dig(data, "data", "reviews") if data else None
|
||||
customer = reviews.get("customerReviews") if isinstance(reviews, dict) else None
|
||||
if not isinstance(customer, list):
|
||||
return []
|
||||
return [normalize_review(r) for r in customer if isinstance(r, dict)]
|
||||
188
surfsense_backend/app/proprietary/platforms/walmart/schemas.py
Normal file
188
surfsense_backend/app/proprietary/platforms/walmart/schemas.py
Normal file
|
|
@ -0,0 +1,188 @@
|
|||
# ruff: noqa: N815
|
||||
"""Input/output models for the public Walmart scraper.
|
||||
|
||||
Two verbs share this module: ``walmart.scrape`` (products + listings) and
|
||||
``walmart.reviews`` (deep paginated reviews). Walmart is a Next.js app that
|
||||
ships its data as JSON in a ``<script id="__NEXT_DATA__">`` tag, so the parsers
|
||||
read structured JSON rather than the rendered DOM — more robust against the
|
||||
constant CSS/layout A/B testing on walmart.com.
|
||||
|
||||
Outputs use ``extra="allow"`` on purpose (same as the Amazon scraper): the wire
|
||||
shape can grow without breaking existing consumers. Only the small, stable,
|
||||
always-populated nested objects are typed; volatile sections stay loose.
|
||||
|
||||
Public, anonymous data only: there is no auth/login field. Deep review
|
||||
pagination uses the public ``/reviews/product/{id}`` page, which robots.txt
|
||||
permits (unlike ``/search``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Typed nested objects (the stable, always-populated ones). #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class Price(BaseModel):
|
||||
"""A monetary amount as Walmart renders it (``price``, ``listPrice``)."""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
value: float | None = None
|
||||
currency: str | None = None
|
||||
|
||||
|
||||
class Seller(BaseModel):
|
||||
"""The offer seller. ``type`` distinguishes Walmart 1P from 3P marketplace."""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
id: str | None = None
|
||||
name: str | None = None
|
||||
type: Literal["WALMART", "MARKETPLACE"] | None = None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Input surfaces. #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class WalmartScrapeInput(BaseModel):
|
||||
"""Full input surface for ``walmart.scrape``.
|
||||
|
||||
``startUrls`` mixes product (``/ip/``), search (``/search``), category
|
||||
(``/cp/``), and browse (``/browse/``) URLs; each is classified and dispatched
|
||||
by :mod:`.url_resolver`. ``extra="allow"`` keeps a verbatim payload valid.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
startUrls: list[str] = Field(min_length=1)
|
||||
maxItemsPerStartUrl: int | None = Field(default=None, ge=0)
|
||||
maxSearchPagesPerStartUrl: int = Field(default=25, ge=1)
|
||||
includeDetails: bool = True
|
||||
includeReviewsSample: bool = True
|
||||
country: str = "us"
|
||||
|
||||
|
||||
class WalmartReviewsInput(BaseModel):
|
||||
"""Full input surface for ``walmart.reviews`` (deep paginated reviews)."""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
itemIds: list[str] = Field(min_length=1)
|
||||
maxReviews: int = Field(default=200, ge=1)
|
||||
sort: Literal["most-recent", "most-helpful", "rating-high", "rating-low"] = (
|
||||
"most-recent"
|
||||
)
|
||||
country: str = "us"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Output items. #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class ProductItem(BaseModel):
|
||||
"""``walmart.scrape`` output item (one per product / listing card).
|
||||
|
||||
Unsourced fields default to ``None``/``[]``; ``extra="allow"`` keeps the
|
||||
contract open so later milestones add fields without breaking consumers.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
||||
|
||||
# Identity / core
|
||||
usItemId: str | None = None
|
||||
name: str | None = None
|
||||
brand: str | None = None
|
||||
url: str | None = None
|
||||
price: Price | None = None
|
||||
listPrice: Price | None = None
|
||||
currency: str | None = None
|
||||
availabilityStatus: str | None = None
|
||||
inStock: bool | None = None
|
||||
|
||||
# Ratings / social
|
||||
stars: float | None = None
|
||||
reviewsCount: int | None = None
|
||||
|
||||
# Commerce
|
||||
seller: Seller | None = None
|
||||
manufacturerName: str | None = None
|
||||
sponsored: bool | None = None
|
||||
|
||||
# Content (detail pages only)
|
||||
shortDescription: str | None = None
|
||||
longDescription: str | None = None
|
||||
thumbnailImage: str | None = None
|
||||
images: list[str] = Field(default_factory=list)
|
||||
breadCrumbs: list[str] = Field(default_factory=list)
|
||||
category: str | None = None
|
||||
specifications: dict | None = None
|
||||
variants: list[dict] = Field(default_factory=list)
|
||||
fulfillment: dict | None = None
|
||||
|
||||
# Embedded free review sample (detail pages only)
|
||||
reviewsSample: dict | None = None
|
||||
|
||||
# Provenance / meta
|
||||
input: str | None = None
|
||||
|
||||
def to_output(self) -> dict[str, Any]:
|
||||
"""Serialize to the flat dict output shape (keeps extras)."""
|
||||
return self.model_dump(by_alias=True, exclude_none=False)
|
||||
|
||||
|
||||
class ReviewItem(BaseModel):
|
||||
"""``walmart.reviews`` output item (one per customer review)."""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
reviewId: str | None = None
|
||||
rating: float | None = None
|
||||
title: str | None = None
|
||||
text: str | None = None
|
||||
submissionTime: str | None = None
|
||||
author: str | None = None
|
||||
verifiedPurchase: bool | None = None
|
||||
positiveFeedback: int | None = None
|
||||
negativeFeedback: int | None = None
|
||||
images: list[str] = Field(default_factory=list)
|
||||
syndicated: bool | None = None
|
||||
sellerResponse: str | None = None
|
||||
usItemId: str | None = None
|
||||
input: str | None = None
|
||||
|
||||
def to_output(self) -> dict[str, Any]:
|
||||
return self.model_dump(exclude_none=False)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Error item (the failure model — pushed into the stream, not raised). #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
ErrorCode = Literal[
|
||||
"invalid_url",
|
||||
"product_not_found",
|
||||
"no_results_found",
|
||||
"reviews_not_found",
|
||||
]
|
||||
|
||||
|
||||
class ErrorItem(BaseModel):
|
||||
"""A per-input failure emitted into the dataset instead of a normal item.
|
||||
|
||||
Consumers tell error items apart from products/reviews by the ``error`` key.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
error: ErrorCode
|
||||
errorDescription: str | None = None
|
||||
input: str | None = None
|
||||
url: str | None = None
|
||||
288
surfsense_backend/app/proprietary/platforms/walmart/scraper.py
Normal file
288
surfsense_backend/app/proprietary/platforms/walmart/scraper.py
Normal file
|
|
@ -0,0 +1,288 @@
|
|||
"""Orchestrate public Walmart product discovery, enrichment, and reviews.
|
||||
|
||||
Two streaming cores: :func:`iter_products` dispatches each start URL to a product
|
||||
or listing flow, and :func:`iter_reviews` paginates the public reviews page per
|
||||
item id. Network and parsing concerns stay isolated in their own modules so
|
||||
markup changes and retry policy can be tested independently.
|
||||
|
||||
The failure model is in-stream error items (dicts with an ``error`` key), never
|
||||
exceptions — identical to the Amazon scraper.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
|
||||
|
||||
from .fetch import fetch_page, gather_bounded
|
||||
from .next_data import extract_next_data
|
||||
from .parsers import parse_listing_page, parse_product, parse_reviews_page
|
||||
from .schemas import (
|
||||
ErrorItem,
|
||||
ProductItem,
|
||||
ReviewItem,
|
||||
WalmartReviewsInput,
|
||||
WalmartScrapeInput,
|
||||
)
|
||||
from .url_resolver import ResolvedUrl, extract_item_id, resolve_url
|
||||
|
||||
__all__ = ["iter_products", "iter_reviews", "scrape_products", "scrape_reviews"]
|
||||
|
||||
_DETAIL_CONCURRENCY = 6
|
||||
_SEARCH_PAGE_LIMIT = 25 # Walmart caps result pagination at 25 pages.
|
||||
_REVIEWS_PAGE_LIMIT = 500 # 10 reviews/page → 5000-review safety ceiling.
|
||||
_DEFAULT_ITEMS_PER_START_URL = 40
|
||||
|
||||
# Friendly sort → Walmart's ``sort`` query value on the reviews page.
|
||||
_REVIEW_SORT = {
|
||||
"most-recent": "submission-desc",
|
||||
"most-helpful": "helpful",
|
||||
"rating-high": "rating-desc",
|
||||
"rating-low": "rating-asc",
|
||||
}
|
||||
|
||||
|
||||
def _error(
|
||||
code: str, description: str, *, input_url: str | None, url: str | None = None
|
||||
) -> dict[str, Any]:
|
||||
return ErrorItem(
|
||||
error=code,
|
||||
errorDescription=description,
|
||||
input=input_url,
|
||||
url=url or input_url,
|
||||
).model_dump()
|
||||
|
||||
|
||||
def _page_url(url: str, page: int) -> str:
|
||||
parsed = urlparse(url)
|
||||
query = dict(parse_qsl(parsed.query, keep_blank_values=True))
|
||||
query["page"] = str(page)
|
||||
return urlunparse(parsed._replace(query=urlencode(query)))
|
||||
|
||||
|
||||
def _product_url(item_id: str) -> str:
|
||||
return f"https://www.walmart.com/ip/{item_id}"
|
||||
|
||||
|
||||
def _reviews_url(item_id: str, page: int, sort: str) -> str:
|
||||
query = urlencode(
|
||||
{"page": page, "sort": sort, "entryPoint": "viewAllReviewsBottom"}
|
||||
)
|
||||
return f"https://www.walmart.com/reviews/product/{item_id}?{query}"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Product / listing flows. #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
async def _product_flow(
|
||||
resolved: ResolvedUrl, input_model: WalmartScrapeInput
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
"""Fetch and parse one product detail page."""
|
||||
url = _product_url(resolved.item_id) if resolved.item_id else resolved.url
|
||||
response = await fetch_page(url, country=input_model.country)
|
||||
if response is None:
|
||||
yield _error(
|
||||
"product_not_found",
|
||||
"The product page could not be loaded after retrying proxy exits.",
|
||||
input_url=resolved.url,
|
||||
)
|
||||
return
|
||||
if response.status in {404, 410}:
|
||||
yield _error(
|
||||
"product_not_found",
|
||||
"The product page was not found.",
|
||||
input_url=resolved.url,
|
||||
)
|
||||
return
|
||||
|
||||
next_data = extract_next_data(response.html)
|
||||
fields = (
|
||||
parse_product(
|
||||
next_data,
|
||||
url=response.url,
|
||||
include_reviews_sample=input_model.includeReviewsSample,
|
||||
)
|
||||
if next_data
|
||||
else {}
|
||||
)
|
||||
if not fields.get("name"):
|
||||
yield _error(
|
||||
"product_not_found",
|
||||
"The response did not contain a recognizable product.",
|
||||
input_url=resolved.url,
|
||||
url=response.url,
|
||||
)
|
||||
return
|
||||
fields["input"] = resolved.url
|
||||
yield ProductItem(**fields).to_output()
|
||||
|
||||
|
||||
async def _listing_flow(
|
||||
resolved: ResolvedUrl, input_model: WalmartScrapeInput
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
"""Page through a search/category/browse listing and optionally enrich."""
|
||||
cap = (
|
||||
input_model.maxItemsPerStartUrl
|
||||
if input_model.maxItemsPerStartUrl is not None
|
||||
else _DEFAULT_ITEMS_PER_START_URL
|
||||
)
|
||||
max_pages = min(input_model.maxSearchPagesPerStartUrl, _SEARCH_PAGE_LIMIT)
|
||||
seen: set[str] = set()
|
||||
emitted = 0
|
||||
for page in range(1, max_pages + 1):
|
||||
response = await fetch_page(
|
||||
_page_url(resolved.url, page), country=input_model.country
|
||||
)
|
||||
next_data = extract_next_data(response.html) if response else None
|
||||
cards = parse_listing_page(next_data) if next_data else []
|
||||
cards = [c for c in cards if c["usItemId"] not in seen]
|
||||
for card in cards:
|
||||
seen.add(card["usItemId"])
|
||||
if not cards:
|
||||
if page == 1:
|
||||
yield _error(
|
||||
"no_results_found",
|
||||
"The listing page did not contain any products.",
|
||||
input_url=resolved.url,
|
||||
)
|
||||
return
|
||||
cards = cards[: max(0, cap - emitted)]
|
||||
|
||||
if not input_model.includeDetails:
|
||||
for card in cards:
|
||||
card["input"] = resolved.url
|
||||
yield ProductItem(**card).to_output()
|
||||
emitted += 1
|
||||
else:
|
||||
|
||||
async def load_card(card: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
product = resolve_url(card["url"]) if card.get("url") else None
|
||||
if product is None or product.item_id is None:
|
||||
card["input"] = resolved.url
|
||||
return [ProductItem(**card).to_output()]
|
||||
return [item async for item in _product_flow(product, input_model)]
|
||||
|
||||
for batch in await gather_bounded(
|
||||
[lambda card=card: load_card(card) for card in cards],
|
||||
concurrency=_DETAIL_CONCURRENCY,
|
||||
):
|
||||
for item in batch:
|
||||
if "error" not in item and emitted >= cap:
|
||||
return
|
||||
yield item
|
||||
if "error" not in item:
|
||||
emitted += 1
|
||||
if emitted >= cap:
|
||||
return
|
||||
|
||||
|
||||
_FLOWS = {"product": _product_flow, "listing": _listing_flow}
|
||||
|
||||
|
||||
async def iter_products(
|
||||
input_model: WalmartScrapeInput,
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
"""Yield product items for every start URL.
|
||||
|
||||
Each URL is classified and dispatched to its per-kind flow. An unrecognized
|
||||
URL yields an ``invalid_url`` error item.
|
||||
"""
|
||||
for url in input_model.startUrls:
|
||||
resolved = resolve_url(url)
|
||||
if resolved is None:
|
||||
yield _error(
|
||||
"invalid_url",
|
||||
"Start URL was malformed or not a recognized Walmart URL.",
|
||||
input_url=url,
|
||||
)
|
||||
continue
|
||||
async for item in _FLOWS[resolved.kind](resolved, input_model):
|
||||
yield item
|
||||
|
||||
|
||||
async def scrape_products(
|
||||
input_model: WalmartScrapeInput, *, limit: int | None = None
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Collect :func:`iter_products` into a list, honoring an optional ``limit``."""
|
||||
from app.capabilities.core.progress import emit_progress
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
async for item in iter_products(input_model):
|
||||
results.append(item)
|
||||
emit_progress("scraping", current=len(results), total=limit, unit="product")
|
||||
if limit is not None and len(results) >= limit:
|
||||
break
|
||||
return results
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Reviews flow (deep pagination). #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
async def _reviews_flow(
|
||||
item_id: str, input_model: WalmartReviewsInput
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
"""Paginate the public reviews page until empty or ``maxReviews`` reached.
|
||||
|
||||
Walmart does not expose a reliable total up front, so the loop stops on the
|
||||
first empty page rather than trusting a page count.
|
||||
"""
|
||||
sort = _REVIEW_SORT[input_model.sort]
|
||||
emitted = 0
|
||||
for page in range(1, _REVIEWS_PAGE_LIMIT + 1):
|
||||
response = await fetch_page(
|
||||
_reviews_url(item_id, page, sort), country=input_model.country
|
||||
)
|
||||
next_data = extract_next_data(response.html) if response else None
|
||||
reviews = parse_reviews_page(next_data) if next_data else []
|
||||
if not reviews:
|
||||
if page == 1 and emitted == 0:
|
||||
yield _error(
|
||||
"reviews_not_found",
|
||||
"The product has no public reviews or could not be loaded.",
|
||||
input_url=item_id,
|
||||
)
|
||||
return
|
||||
for raw in reviews:
|
||||
raw["usItemId"] = item_id
|
||||
raw["input"] = item_id
|
||||
yield ReviewItem(**raw).to_output()
|
||||
emitted += 1
|
||||
if emitted >= input_model.maxReviews:
|
||||
return
|
||||
|
||||
|
||||
async def iter_reviews(
|
||||
input_model: WalmartReviewsInput,
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
"""Yield review items for every requested item id (or resolvable URL)."""
|
||||
for raw_id in input_model.itemIds:
|
||||
item_id = extract_item_id(raw_id)
|
||||
if item_id is None:
|
||||
yield _error(
|
||||
"invalid_url",
|
||||
"Could not extract a Walmart item id from the input.",
|
||||
input_url=raw_id,
|
||||
)
|
||||
continue
|
||||
async for item in _reviews_flow(item_id, input_model):
|
||||
yield item
|
||||
|
||||
|
||||
async def scrape_reviews(
|
||||
input_model: WalmartReviewsInput, *, limit: int | None = None
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Collect :func:`iter_reviews` into a list, honoring an optional ``limit``."""
|
||||
from app.capabilities.core.progress import emit_progress
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
async for item in iter_reviews(input_model):
|
||||
results.append(item)
|
||||
emit_progress("scraping", current=len(results), total=limit, unit="review")
|
||||
if limit is not None and len(results) >= limit:
|
||||
break
|
||||
return results
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
"""Classify a Walmart start URL and extract its identifiers.
|
||||
|
||||
Covers the shapes accepted on input: product pages (``/ip/{slug}/{id}`` or
|
||||
``/ip/{id}``), search (``/search?q=``), category (``/cp/{slug}/{id}``), and
|
||||
browse (``/browse/...``) pages. Category and browse render the same
|
||||
``searchResult`` JSON as search, so they share the ``listing`` kind and parser.
|
||||
|
||||
Pure, no I/O. ``item_id`` is Walmart's ``usItemId`` — the numeric id the review
|
||||
page and detail page are both keyed on.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from typing import Literal
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
ResolvedKind = Literal["product", "listing"]
|
||||
|
||||
# usItemId: the trailing numeric segment of an /ip/ or /reviews/product/ path,
|
||||
# or a bare numeric id passed directly.
|
||||
_ITEM_ID_PATH_RE = re.compile(r"/(?:ip|reviews/product)/(?:[^/?#]+/)*(\d{4,})")
|
||||
_BARE_ID_RE = re.compile(r"^\d{4,}$")
|
||||
|
||||
_LISTING_QUERY_KEYS = ("q", "cat_id", "browse")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResolvedUrl:
|
||||
kind: ResolvedKind
|
||||
url: str
|
||||
item_id: str | None = None
|
||||
domain: str | None = None
|
||||
|
||||
|
||||
def extract_item_id(url: str) -> str | None:
|
||||
"""Pull the numeric ``usItemId`` out of a product/review URL or bare id."""
|
||||
candidate = url.strip()
|
||||
if _BARE_ID_RE.match(candidate):
|
||||
return candidate
|
||||
match = _ITEM_ID_PATH_RE.search(candidate)
|
||||
return match.group(1) if match else None
|
||||
|
||||
|
||||
def resolve_url(url: str) -> ResolvedUrl | None:
|
||||
"""Classify a Walmart start URL into a scrape job, or ``None`` if unrecognized.
|
||||
|
||||
A bare numeric id resolves to a product (its ``usItemId``) so callers can
|
||||
pass item ids directly without constructing a full URL.
|
||||
"""
|
||||
if _BARE_ID_RE.match(url.strip()):
|
||||
return ResolvedUrl("product", url.strip(), item_id=url.strip())
|
||||
|
||||
parsed = urlparse(url)
|
||||
host = (parsed.hostname or "").lower()
|
||||
path = parsed.path or ""
|
||||
|
||||
if "walmart." not in host:
|
||||
return None
|
||||
|
||||
if path.startswith("/ip/"):
|
||||
return ResolvedUrl("product", url, item_id=extract_item_id(path), domain=host)
|
||||
|
||||
query = parse_qs(parsed.query)
|
||||
is_listing = (
|
||||
path.startswith("/search")
|
||||
or path.startswith("/cp/")
|
||||
or path.startswith("/browse/")
|
||||
or any(key in query for key in _LISTING_QUERY_KEYS)
|
||||
)
|
||||
if is_listing:
|
||||
return ResolvedUrl("listing", url, domain=host)
|
||||
|
||||
return None
|
||||
|
|
@ -8,6 +8,7 @@ import app.capabilities.indeed
|
|||
import app.capabilities.instagram
|
||||
import app.capabilities.reddit
|
||||
import app.capabilities.tiktok
|
||||
import app.capabilities.walmart
|
||||
import app.capabilities.web
|
||||
import app.capabilities.youtube # noqa: F401
|
||||
from app.automations.api import router as automations_router
|
||||
|
|
|
|||
80
surfsense_backend/scripts/e2e_walmart_scraper.py
Normal file
80
surfsense_backend/scripts/e2e_walmart_scraper.py
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
"""Manual end-to-end check for the public Walmart scraper.
|
||||
|
||||
Run from the backend directory:
|
||||
|
||||
uv run python scripts/e2e_walmart_scraper.py
|
||||
|
||||
The script requires live network access and the configured residential proxy
|
||||
(US exit). It is intentionally excluded from pytest.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
_BACKEND_ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(_BACKEND_ROOT))
|
||||
for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"):
|
||||
if _candidate.exists():
|
||||
load_dotenv(_candidate)
|
||||
break
|
||||
|
||||
from app.proprietary.platforms.walmart import ( # noqa: E402
|
||||
WalmartReviewsInput,
|
||||
WalmartScrapeInput,
|
||||
scrape_products,
|
||||
scrape_reviews,
|
||||
)
|
||||
|
||||
_PRODUCT_URL = "https://www.walmart.com/ip/212092810"
|
||||
_SEARCH_URL = "https://www.walmart.com/search?q=air+fryer"
|
||||
|
||||
|
||||
def _check(label: str, passed: bool) -> bool:
|
||||
print(f"[{'PASS' if passed else 'FAIL'}] {label}")
|
||||
return passed
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
ok = True
|
||||
|
||||
product_items = await scrape_products(
|
||||
WalmartScrapeInput(startUrls=[_PRODUCT_URL]), limit=1
|
||||
)
|
||||
product = product_items[0] if product_items else {}
|
||||
print(json.dumps(product, indent=2, ensure_ascii=False)[:2500])
|
||||
ok &= _check(
|
||||
"product detail has id and name",
|
||||
bool(product.get("usItemId") and product.get("name")),
|
||||
)
|
||||
|
||||
search_items = await scrape_products(
|
||||
WalmartScrapeInput(
|
||||
startUrls=[_SEARCH_URL], maxItemsPerStartUrl=3, includeDetails=False
|
||||
),
|
||||
limit=3,
|
||||
)
|
||||
ok &= _check(
|
||||
"search returns product cards",
|
||||
bool(search_items) and any("name" in item for item in search_items),
|
||||
)
|
||||
|
||||
item_id = product.get("usItemId") or "212092810"
|
||||
reviews = await scrape_reviews(
|
||||
WalmartReviewsInput(itemIds=[item_id], maxReviews=15), limit=15
|
||||
)
|
||||
ok &= _check(
|
||||
"reviews returned with rating and text",
|
||||
bool(reviews) and any(r.get("rating") and r.get("text") for r in reviews),
|
||||
)
|
||||
|
||||
return 0 if ok else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(asyncio.run(main()))
|
||||
|
|
@ -41,6 +41,7 @@ _EXPECTED_SUBAGENTS = frozenset(
|
|||
"onedrive",
|
||||
"reddit",
|
||||
"tiktok",
|
||||
"walmart",
|
||||
"web_crawler",
|
||||
"youtube",
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,38 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from app.capabilities.walmart.reviews.executor import build_reviews_executor
|
||||
from app.capabilities.walmart.reviews.schemas import ReviewsInput
|
||||
from app.proprietary.platforms.walmart import WalmartReviewsInput
|
||||
|
||||
|
||||
class _FakeScraper:
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[tuple[WalmartReviewsInput, int | None]] = []
|
||||
|
||||
async def __call__(
|
||||
self, input_model: WalmartReviewsInput, *, limit: int | None = None
|
||||
) -> list[dict]:
|
||||
self.calls.append((input_model, limit))
|
||||
return [{"reviewId": "r1", "rating": 5}]
|
||||
|
||||
|
||||
async def test_executor_maps_agent_input_to_scraper_input():
|
||||
scraper = _FakeScraper()
|
||||
execute = build_reviews_executor(scraper)
|
||||
|
||||
output = await execute(
|
||||
ReviewsInput(
|
||||
urls=["https://www.walmart.com/ip/123456"],
|
||||
item_ids=["222"],
|
||||
max_reviews=50,
|
||||
sort_by="most-helpful",
|
||||
)
|
||||
)
|
||||
|
||||
assert output.items[0].reviewId == "r1"
|
||||
input_model, limit = scraper.calls[0]
|
||||
assert input_model.itemIds == ["https://www.walmart.com/ip/123456", "222"]
|
||||
assert input_model.maxReviews == 50
|
||||
assert input_model.sort == "most-helpful"
|
||||
# limit is the pre-flight worst case: 2 sources * 50 reviews
|
||||
assert limit == 100
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app.capabilities.walmart.reviews.schemas import ReviewsInput, ReviewsOutput
|
||||
|
||||
|
||||
def test_estimated_units_scale_with_sources_and_max_reviews():
|
||||
payload = ReviewsInput(
|
||||
urls=["https://www.walmart.com/ip/1"], item_ids=["222"], max_reviews=150
|
||||
)
|
||||
|
||||
# 2 sources * 150 reviews each
|
||||
assert payload.estimated_units == 300
|
||||
|
||||
|
||||
def test_at_least_one_source_is_required():
|
||||
with pytest.raises(ValidationError):
|
||||
ReviewsInput()
|
||||
|
||||
|
||||
def test_sources_merge_urls_then_item_ids():
|
||||
payload = ReviewsInput(urls=["https://www.walmart.com/ip/1"], item_ids=["222"])
|
||||
|
||||
assert payload.sources() == ["https://www.walmart.com/ip/1", "222"]
|
||||
|
||||
|
||||
def test_error_items_are_not_billable():
|
||||
output = ReviewsOutput(
|
||||
items=[
|
||||
{"reviewId": "r1", "rating": 5},
|
||||
{"error": "reviews_not_found"},
|
||||
]
|
||||
)
|
||||
|
||||
assert output.billable_units == 1
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from app.capabilities.walmart.scrape.executor import build_scrape_executor
|
||||
from app.capabilities.walmart.scrape.schemas import MAX_WALMART_RESULTS, ScrapeInput
|
||||
from app.proprietary.platforms.walmart import WalmartScrapeInput
|
||||
|
||||
|
||||
class _FakeScraper:
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[tuple[WalmartScrapeInput, int | None]] = []
|
||||
|
||||
async def __call__(
|
||||
self, input_model: WalmartScrapeInput, *, limit: int | None = None
|
||||
) -> list[dict]:
|
||||
self.calls.append((input_model, limit))
|
||||
return [{"usItemId": "123", "name": "Product"}]
|
||||
|
||||
|
||||
async def test_executor_maps_agent_input_to_scraper_input():
|
||||
scraper = _FakeScraper()
|
||||
execute = build_scrape_executor(scraper)
|
||||
|
||||
output = await execute(
|
||||
ScrapeInput(
|
||||
search_terms=["air fryer"],
|
||||
urls=["https://www.walmart.com/ip/123456"],
|
||||
max_items=5,
|
||||
include_details=False,
|
||||
include_reviews_sample=False,
|
||||
)
|
||||
)
|
||||
|
||||
assert output.items[0].usItemId == "123"
|
||||
input_model, limit = scraper.calls[0]
|
||||
assert input_model.startUrls == [
|
||||
"https://www.walmart.com/ip/123456",
|
||||
"https://www.walmart.com/search?q=air+fryer",
|
||||
]
|
||||
assert input_model.maxItemsPerStartUrl == 5
|
||||
assert input_model.includeDetails is False
|
||||
assert input_model.includeReviewsSample is False
|
||||
assert limit == MAX_WALMART_RESULTS
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app.capabilities.walmart.scrape.schemas import (
|
||||
MAX_WALMART_RESULTS,
|
||||
ScrapeInput,
|
||||
ScrapeOutput,
|
||||
)
|
||||
|
||||
|
||||
def test_estimated_units_cover_search_and_direct_sources():
|
||||
payload = ScrapeInput(
|
||||
search_terms=["air fryer", "blender"],
|
||||
urls=["https://www.walmart.com/ip/123456"],
|
||||
max_items=20,
|
||||
)
|
||||
|
||||
# (2 search + 1 direct source) * 20 items each
|
||||
assert payload.estimated_units == 60
|
||||
|
||||
|
||||
def test_estimated_units_respect_hard_run_ceiling():
|
||||
payload = ScrapeInput(search_terms=["x"] * 20, max_items=100)
|
||||
assert payload.estimated_units == MAX_WALMART_RESULTS
|
||||
|
||||
|
||||
def test_at_least_one_source_is_required():
|
||||
with pytest.raises(ValidationError):
|
||||
ScrapeInput()
|
||||
|
||||
|
||||
def test_combined_sources_are_capped():
|
||||
with pytest.raises(ValidationError):
|
||||
ScrapeInput(urls=["u"] * 11, search_terms=["t"] * 10)
|
||||
|
||||
|
||||
def test_start_urls_synthesize_a_search_url_per_term():
|
||||
payload = ScrapeInput(
|
||||
urls=["https://www.walmart.com/ip/1"], search_terms=["air fryer"]
|
||||
)
|
||||
|
||||
assert payload.start_urls() == [
|
||||
"https://www.walmart.com/ip/1",
|
||||
"https://www.walmart.com/search?q=air+fryer",
|
||||
]
|
||||
|
||||
|
||||
def test_error_items_are_not_billable():
|
||||
output = ScrapeOutput(
|
||||
items=[
|
||||
{"usItemId": "123", "name": "Product"},
|
||||
{"error": "product_not_found", "errorDescription": "Missing"},
|
||||
]
|
||||
)
|
||||
|
||||
assert output.billable_units == 1
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from app.capabilities.core import BillingUnit
|
||||
from app.capabilities.core.store import get_capability
|
||||
from app.capabilities.walmart.reviews.schemas import ReviewsInput, ReviewsOutput
|
||||
from app.capabilities.walmart.scrape.schemas import ScrapeInput, ScrapeOutput
|
||||
|
||||
|
||||
def test_walmart_scrape_is_registered_and_metered():
|
||||
capability = get_capability("walmart.scrape")
|
||||
|
||||
assert capability.input_schema is ScrapeInput
|
||||
assert capability.output_schema is ScrapeOutput
|
||||
assert capability.billing_unit is BillingUnit.WALMART_PRODUCT
|
||||
|
||||
|
||||
def test_walmart_reviews_is_registered_and_metered():
|
||||
capability = get_capability("walmart.reviews")
|
||||
|
||||
assert capability.input_schema is ReviewsInput
|
||||
assert capability.output_schema is ReviewsOutput
|
||||
assert capability.billing_unit is BillingUnit.WALMART_REVIEW
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
<!doctype html><html><head><title>Robot or human?</title></head><body>
|
||||
<div id="px-captcha">Activate and hold the button to confirm that you're human.</div>
|
||||
<p>Robot or human?</p>
|
||||
</body></html>
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
<!doctype html><html><head><title>laptop - Walmart.com</title></head><body>
|
||||
<script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"initialData":{"searchResult":{"itemStacks":[{"count":2,"items":[{"__typename":"Product","usItemId":"791595618","name":"Restored Acer Chromebook","brand":null,"canonicalUrl":"/ip/Acer-Chromebook/791595618","averageRating":3.5,"numberOfReviews":46,"sellerId":"AB4FE","sellerName":"Ellison ProTech","availabilityStatusV2":{"value":"IN_STOCK"},"imageInfo":{"thumbnailUrl":"https://i5.walmartimages.com/a.jpg"},"priceInfo":{"currentPrice":{"price":79.99,"currencyUnit":"USD"}},"isSponsoredFlag":false},{"__typename":"Product","usItemId":"999001","name":"HP Laptop 15","canonicalUrl":"/ip/HP-Laptop/999001","isOutOfStock":true,"price":499.0}]}]}}}}}</script>
|
||||
</body></html>
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
<!doctype html><html><head><title>Midea AC</title></head><body>
|
||||
<script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"initialData":{"data":{"product":{"usItemId":"212092810","id":"ABC123","name":"Midea 5,000 BTU Window Air Conditioner","brand":"Midea","availabilityStatus":"IN_STOCK","averageRating":4.5,"numberOfReviews":6287,"manufacturerName":"Midea","shortDescription":"Cools a small room fast.","sellerName":"Walmart.com","sellerId":"F55CT","priceInfo":{"currentPrice":{"price":149.0,"currencyUnit":"USD"},"wasPrice":{"price":199.0,"currencyUnit":"USD"}},"imageInfo":{"thumbnailUrl":"https://i5.walmartimages.com/t.jpg","allImages":[{"url":"https://i5.walmartimages.com/1.jpg"},{"url":"https://i5.walmartimages.com/2.jpg"}]},"category":{"categoryPathId":"0:1072864:133026","path":[{"name":"Home","url":"/cp/home/1072864"},{"name":"Air Conditioners","url":"/cp/air-conditioners/133026"}]},"variantCriteria":[{"name":"capacity"}]},"idml":{"longDescription":"<p>A powerful window unit.</p>"},"reviews":{"averageOverallRating":4.5,"totalReviewCount":6287,"aspects":[{"name":"Value","score":90}],"customerReviews":[{"reviewId":"r1","rating":5,"reviewTitle":"Great AC","reviewText":"Very cold air.","reviewSubmissionTime":"2024-04-28","userNickname":"Anne","positiveFeedback":3,"negativeFeedback":0,"badges":[{"id":"VerifiedPurchaser"}]}]}}}}}}</script>
|
||||
</body></html>
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
<!doctype html><html><head><title>Reviews</title></head><body>
|
||||
<script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"initialData":{"data":{"reviews":{"customerReviews":[{"reviewId":"296013686","rating":5,"reviewTitle":"Awesome","reviewText":"The PS5 is awesome!","reviewSubmissionTime":"12/20/2022","userNickname":"JohnPaul","positiveFeedback":10,"negativeFeedback":3,"badges":[{"id":"VerifiedPurchaser"}],"photos":[{"normalUrl":"https://i5.walmartimages.com/r.jpg"}],"clientResponses":[{"response":"Thanks for the review!"}],"syndicationSource":null},{"reviewId":"296013687","rating":4,"reviewTitle":"Good","reviewText":"Solid console.","reviewSubmissionTime":"12/21/2022","userNickname":"Sam","positiveFeedback":1,"negativeFeedback":0,"badges":[]}]}}}}}}</script>
|
||||
</body></html>
|
||||
138
surfsense_backend/tests/unit/platforms/walmart/test_flows.py
Normal file
138
surfsense_backend/tests/unit/platforms/walmart/test_flows.py
Normal file
|
|
@ -0,0 +1,138 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from app.proprietary.platforms.walmart import (
|
||||
WalmartReviewsInput,
|
||||
WalmartScrapeInput,
|
||||
scrape_products,
|
||||
scrape_reviews,
|
||||
scraper,
|
||||
)
|
||||
from app.proprietary.platforms.walmart.fetch import FetchResult
|
||||
|
||||
_FIXTURES = Path(__file__).parent / "fixtures"
|
||||
|
||||
|
||||
def _fixture(name: str) -> str:
|
||||
return (_FIXTURES / name).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def _response(url: str, html: str, status: int = 200) -> FetchResult:
|
||||
return FetchResult(status=status, html=html, url=url, cookies={})
|
||||
|
||||
|
||||
async def test_product_flow_emits_parsed_item(monkeypatch):
|
||||
async def fetch_page(url: str, **_kwargs):
|
||||
return _response(url, _fixture("product.html"))
|
||||
|
||||
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
|
||||
items = await scrape_products(
|
||||
WalmartScrapeInput(startUrls=["https://www.walmart.com/ip/212092810"])
|
||||
)
|
||||
|
||||
assert len(items) == 1
|
||||
assert items[0]["usItemId"] == "212092810"
|
||||
assert items[0]["name"].startswith("Midea")
|
||||
|
||||
|
||||
async def test_product_flow_maps_not_found_to_error_item(monkeypatch):
|
||||
async def fetch_page(url: str, **_kwargs):
|
||||
return _response(url, "", status=404)
|
||||
|
||||
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
|
||||
items = await scrape_products(
|
||||
WalmartScrapeInput(startUrls=["https://www.walmart.com/ip/212092810"])
|
||||
)
|
||||
|
||||
assert items[0]["error"] == "product_not_found"
|
||||
|
||||
|
||||
async def test_listing_flow_card_only_honors_cap(monkeypatch):
|
||||
calls: list[str] = []
|
||||
|
||||
async def fetch_page(url: str, **_kwargs):
|
||||
calls.append(url)
|
||||
html = _fixture("listing.html") if "page=1" in url else "<html></html>"
|
||||
return _response(url, html)
|
||||
|
||||
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
|
||||
items = await scrape_products(
|
||||
WalmartScrapeInput(
|
||||
startUrls=["https://www.walmart.com/search?q=laptop"],
|
||||
maxItemsPerStartUrl=1,
|
||||
includeDetails=False,
|
||||
)
|
||||
)
|
||||
|
||||
assert len(items) == 1
|
||||
assert items[0]["usItemId"] == "791595618"
|
||||
assert len(calls) == 1
|
||||
|
||||
|
||||
async def test_listing_flow_enriches_with_detail_pages(monkeypatch):
|
||||
async def fetch_page(url: str, **_kwargs):
|
||||
if "/ip/" in url:
|
||||
return _response(url, _fixture("product.html"))
|
||||
html = _fixture("listing.html") if "page=1" in url else "<html></html>"
|
||||
return _response(url, html)
|
||||
|
||||
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
|
||||
items = await scrape_products(
|
||||
WalmartScrapeInput(
|
||||
startUrls=["https://www.walmart.com/search?q=laptop"],
|
||||
maxItemsPerStartUrl=2,
|
||||
includeDetails=True,
|
||||
)
|
||||
)
|
||||
|
||||
# Both cards enrich to the same product fixture (detail fetch wins).
|
||||
assert all(item["longDescription"] for item in items)
|
||||
|
||||
|
||||
async def test_invalid_url_yields_error_item(monkeypatch):
|
||||
items = await scrape_products(
|
||||
WalmartScrapeInput(startUrls=["https://example.com/not-walmart"])
|
||||
)
|
||||
assert items[0]["error"] == "invalid_url"
|
||||
|
||||
|
||||
async def test_reviews_flow_paginates_until_empty(monkeypatch):
|
||||
calls: list[str] = []
|
||||
|
||||
async def fetch_page(url: str, **_kwargs):
|
||||
calls.append(url)
|
||||
html = _fixture("reviews.html") if "page=1" in url else "<html></html>"
|
||||
return _response(url, html)
|
||||
|
||||
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
|
||||
items = await scrape_reviews(
|
||||
WalmartReviewsInput(itemIds=["212092810"], maxReviews=100)
|
||||
)
|
||||
|
||||
assert len(items) == 2
|
||||
assert items[0]["reviewId"] == "296013686"
|
||||
# page=1 returned records, page=2 empty → stop. Two fetches total.
|
||||
assert len(calls) == 2
|
||||
|
||||
|
||||
async def test_reviews_flow_honors_max_reviews(monkeypatch):
|
||||
async def fetch_page(url: str, **_kwargs):
|
||||
return _response(url, _fixture("reviews.html"))
|
||||
|
||||
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
|
||||
items = await scrape_reviews(
|
||||
WalmartReviewsInput(itemIds=["212092810"], maxReviews=1)
|
||||
)
|
||||
|
||||
assert len(items) == 1
|
||||
|
||||
|
||||
async def test_reviews_flow_maps_empty_to_error_item(monkeypatch):
|
||||
async def fetch_page(url: str, **_kwargs):
|
||||
return _response(url, "<html></html>")
|
||||
|
||||
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
|
||||
items = await scrape_reviews(WalmartReviewsInput(itemIds=["212092810"]))
|
||||
|
||||
assert items[0]["error"] == "reviews_not_found"
|
||||
|
|
@ -0,0 +1,93 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from app.proprietary.platforms.walmart.fetch import is_blocked
|
||||
from app.proprietary.platforms.walmart.next_data import extract_next_data
|
||||
from app.proprietary.platforms.walmart.parsers import (
|
||||
parse_listing_page,
|
||||
parse_product,
|
||||
parse_reviews_page,
|
||||
)
|
||||
from app.proprietary.platforms.walmart.url_resolver import extract_item_id, resolve_url
|
||||
|
||||
_FIXTURES = Path(__file__).parent / "fixtures"
|
||||
|
||||
|
||||
def _fixture(name: str) -> str:
|
||||
return (_FIXTURES / name).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_product_parser_extracts_core_fields_and_review_sample():
|
||||
data = extract_next_data(_fixture("product.html"))
|
||||
item = parse_product(data, url="https://www.walmart.com/ip/212092810")
|
||||
|
||||
assert item["usItemId"] == "212092810"
|
||||
assert item["name"].startswith("Midea")
|
||||
assert item["price"] == {"value": 149.0, "currency": "USD"}
|
||||
assert item["listPrice"] == {"value": 199.0, "currency": "USD"}
|
||||
assert item["stars"] == 4.5
|
||||
assert item["reviewsCount"] == 6287
|
||||
assert item["inStock"] is True
|
||||
assert item["seller"] == {"id": "F55CT", "name": "Walmart.com", "type": "WALMART"}
|
||||
assert item["longDescription"] == "<p>A powerful window unit.</p>"
|
||||
assert item["images"] == [
|
||||
"https://i5.walmartimages.com/1.jpg",
|
||||
"https://i5.walmartimages.com/2.jpg",
|
||||
]
|
||||
# Walmart ships category.path as breadcrumb objects, not a string.
|
||||
assert item["breadCrumbs"] == ["Home", "Air Conditioners"]
|
||||
assert item["category"] == "Air Conditioners"
|
||||
assert item["reviewsSample"]["totalReviewCount"] == 6287
|
||||
assert item["reviewsSample"]["topReviews"][0]["verifiedPurchase"] is True
|
||||
|
||||
|
||||
def test_listing_parser_normalizes_cards_and_marketplace_seller():
|
||||
data = extract_next_data(_fixture("listing.html"))
|
||||
cards = parse_listing_page(data)
|
||||
|
||||
assert [c["usItemId"] for c in cards] == ["791595618", "999001"]
|
||||
first = cards[0]
|
||||
assert first["url"] == "https://www.walmart.com/ip/Acer-Chromebook/791595618"
|
||||
assert first["price"] == {"value": 79.99, "currency": "USD"}
|
||||
assert first["seller"]["type"] == "MARKETPLACE"
|
||||
assert first["inStock"] is True
|
||||
# Fallback price + out-of-stock derivation for the second card.
|
||||
assert cards[1]["price"] == {"value": 499.0}
|
||||
assert cards[1]["inStock"] is False
|
||||
|
||||
|
||||
def test_reviews_parser_extracts_all_records():
|
||||
data = extract_next_data(_fixture("reviews.html"))
|
||||
reviews = parse_reviews_page(data)
|
||||
|
||||
assert len(reviews) == 2
|
||||
assert reviews[0]["reviewId"] == "296013686"
|
||||
assert reviews[0]["author"] == "JohnPaul"
|
||||
assert reviews[0]["verifiedPurchase"] is True
|
||||
assert reviews[0]["images"] == ["https://i5.walmartimages.com/r.jpg"]
|
||||
assert reviews[0]["sellerResponse"] == "Thanks for the review!"
|
||||
assert reviews[1]["verifiedPurchase"] is False
|
||||
|
||||
|
||||
def test_block_detection_handles_status_and_body():
|
||||
assert is_blocked(_fixture("blocked.html"), 200)
|
||||
assert is_blocked("", 412)
|
||||
assert is_blocked("", 429)
|
||||
assert not is_blocked(_fixture("product.html"), 200)
|
||||
|
||||
|
||||
def test_missing_next_data_yields_none():
|
||||
assert extract_next_data("<html><body>no script here</body></html>") is None
|
||||
assert parse_product(extract_next_data("") or {}, url="x") == {}
|
||||
|
||||
|
||||
def test_url_resolver_classifies_and_extracts_ids():
|
||||
assert resolve_url("https://www.walmart.com/ip/Foo/123456789").kind == "product"
|
||||
assert (
|
||||
extract_item_id("https://www.walmart.com/reviews/product/212092810")
|
||||
== "212092810"
|
||||
)
|
||||
assert resolve_url("https://www.walmart.com/search?q=tv").kind == "listing"
|
||||
assert resolve_url("https://www.walmart.com/cp/tvs/3944").kind == "listing"
|
||||
assert resolve_url("https://example.com/ip/1") is None
|
||||
|
|
@ -1,7 +1,8 @@
|
|||
"""Scraper tools: one MCP surface per SurfSense platform capability.
|
||||
|
||||
Web crawl, Google Search, Reddit, YouTube, Google Maps, Indeed, and Amazon each
|
||||
get a tool that maps a natural-language request to the workspace's scraper. Two run-history tools
|
||||
Web crawl, Google Search, Reddit, YouTube, Google Maps, Amazon, Indeed, and
|
||||
Walmart each get a tool that maps a natural-language request to the workspace's
|
||||
scraper. Two run-history tools
|
||||
list and fetch past runs, so a large result truncated inline can be retrieved in
|
||||
full later. Each platform lives in its own module under platforms/.
|
||||
"""
|
||||
|
|
@ -21,6 +22,7 @@ from .platforms import (
|
|||
instagram,
|
||||
reddit,
|
||||
tiktok,
|
||||
walmart,
|
||||
web,
|
||||
youtube,
|
||||
)
|
||||
|
|
@ -35,6 +37,7 @@ _REGISTRARS = (
|
|||
google_maps,
|
||||
indeed,
|
||||
amazon,
|
||||
walmart,
|
||||
run_history,
|
||||
)
|
||||
|
||||
|
|
|
|||
140
surfsense_mcp/mcp_server/features/scrapers/platforms/walmart.py
Normal file
140
surfsense_mcp/mcp_server/features/scrapers/platforms/walmart.py
Normal file
|
|
@ -0,0 +1,140 @@
|
|||
"""Walmart scraper tools: products/listings and deep reviews."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Annotated, Literal
|
||||
|
||||
from mcp.server.fastmcp import FastMCP
|
||||
from pydantic import Field
|
||||
|
||||
from ....core.client import SurfSenseClient
|
||||
from ....core.rendering import ResponseFormatParam
|
||||
from ....core.workspace_context import WorkspaceContext, WorkspaceParam
|
||||
from ..annotations import SCRAPE
|
||||
from ..capability import run_scraper
|
||||
|
||||
ReviewSort = Literal["most-recent", "most-helpful", "rating-high", "rating-low"]
|
||||
|
||||
|
||||
def register(mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext) -> None:
|
||||
"""Register the Walmart product and review tools."""
|
||||
|
||||
@mcp.tool(
|
||||
name="surfsense_walmart_scrape",
|
||||
title="Scrape Walmart products",
|
||||
annotations=SCRAPE,
|
||||
structured_output=False,
|
||||
)
|
||||
async def walmart_scrape(
|
||||
urls: Annotated[
|
||||
list[str] | None,
|
||||
Field(
|
||||
description="Walmart product (/ip/), search (/search), category "
|
||||
"(/cp/), or browse (/browse/) URLs. Provide urls OR search_terms."
|
||||
),
|
||||
] = None,
|
||||
search_terms: Annotated[
|
||||
list[str] | None,
|
||||
Field(
|
||||
description="Search phrases run on walmart.com, e.g. ['air fryer']. "
|
||||
"Provide search_terms OR urls."
|
||||
),
|
||||
] = None,
|
||||
max_items: Annotated[
|
||||
int,
|
||||
Field(ge=1, le=100, description="Max products per search term or listing URL."),
|
||||
] = 10,
|
||||
include_details: Annotated[
|
||||
bool,
|
||||
Field(
|
||||
description="Fetch full product detail pages. False returns faster "
|
||||
"card-only results from listings."
|
||||
),
|
||||
] = True,
|
||||
include_reviews_sample: Annotated[
|
||||
bool,
|
||||
Field(
|
||||
description="Include the free on-page review sample (rating "
|
||||
"distribution, aspects, top reviews) on detail pages."
|
||||
),
|
||||
] = True,
|
||||
workspace: WorkspaceParam = None,
|
||||
response_format: ResponseFormatParam = "markdown",
|
||||
) -> str:
|
||||
"""Scrape public Walmart product data by URL or search term.
|
||||
|
||||
Use this for product research: title, price, list price, rating and
|
||||
review count, availability, seller (Walmart 1P vs marketplace), images,
|
||||
description, variants, and a sample of on-page reviews. Only public,
|
||||
anonymous data — no login. For a product's full review history use
|
||||
surfsense_walmart_reviews instead.
|
||||
Example: search_terms=['air fryer'], max_items=5.
|
||||
"""
|
||||
return await run_scraper(
|
||||
client,
|
||||
context,
|
||||
platform="walmart",
|
||||
verb="scrape",
|
||||
payload={
|
||||
"urls": urls,
|
||||
"search_terms": search_terms,
|
||||
"max_items": max_items,
|
||||
"include_details": include_details,
|
||||
"include_reviews_sample": include_reviews_sample,
|
||||
},
|
||||
workspace=workspace,
|
||||
response_format=response_format,
|
||||
)
|
||||
|
||||
@mcp.tool(
|
||||
name="surfsense_walmart_reviews",
|
||||
title="Fetch Walmart reviews",
|
||||
annotations=SCRAPE,
|
||||
structured_output=False,
|
||||
)
|
||||
async def walmart_reviews(
|
||||
urls: Annotated[
|
||||
list[str] | None,
|
||||
Field(
|
||||
description="Walmart product URLs (/ip/...). Provide urls OR item_ids."
|
||||
),
|
||||
] = None,
|
||||
item_ids: Annotated[
|
||||
list[str] | None,
|
||||
Field(
|
||||
description="Walmart numeric item ids (usItemId) from "
|
||||
"surfsense_walmart_scrape."
|
||||
),
|
||||
] = None,
|
||||
max_reviews: Annotated[
|
||||
int,
|
||||
Field(ge=1, le=5000, description="Max reviews per product (10 per page)."),
|
||||
] = 200,
|
||||
sort_by: Annotated[
|
||||
ReviewSort, Field(description="Review ordering.")
|
||||
] = "most-recent",
|
||||
workspace: WorkspaceParam = None,
|
||||
response_format: ResponseFormatParam = "markdown",
|
||||
) -> str:
|
||||
"""Fetch deep paginated customer reviews for Walmart products.
|
||||
|
||||
Use this to read the full review history on specific products; get urls
|
||||
or item_ids from surfsense_walmart_scrape first if you only have a name.
|
||||
Returns rating, title, text, author, verified-purchase flag, images, and
|
||||
seller response per review.
|
||||
Example: item_ids=['212092810'], sort_by='most-helpful', max_reviews=100.
|
||||
"""
|
||||
return await run_scraper(
|
||||
client,
|
||||
context,
|
||||
platform="walmart",
|
||||
verb="reviews",
|
||||
payload={
|
||||
"urls": urls,
|
||||
"item_ids": item_ids,
|
||||
"max_reviews": max_reviews,
|
||||
"sort_by": sort_by,
|
||||
},
|
||||
workspace=workspace,
|
||||
response_format=response_format,
|
||||
)
|
||||
|
|
@ -16,7 +16,7 @@ import type { FaqItem } from "@/lib/connectors-marketing/types";
|
|||
const canonicalUrl = "https://www.surfsense.com/mcp-server";
|
||||
|
||||
const metaDescription =
|
||||
"The SurfSense MCP server gives Claude, Cursor, and any MCP client native tools for your workspace: scrape Reddit, YouTube, Instagram, TikTok, Amazon, Google Maps, Google Search, and the web, plus full knowledge base access. One API key.";
|
||||
"The SurfSense MCP server gives Claude, Cursor, and any MCP client native tools for your workspace: scrape Reddit, YouTube, Instagram, TikTok, Amazon, Walmart, Google Maps, Google Search, and the web, plus full knowledge base access. One API key.";
|
||||
|
||||
export const metadata: Metadata = {
|
||||
title: "SurfSense MCP Server: Scraper APIs and Knowledge Base as Agent Tools",
|
||||
|
|
@ -103,6 +103,8 @@ const TOOL_GROUPS = [
|
|||
"surfsense_google_maps_reviews",
|
||||
"surfsense_google_search",
|
||||
"surfsense_amazon_scrape",
|
||||
"surfsense_walmart_scrape",
|
||||
"surfsense_walmart_reviews",
|
||||
"surfsense_web_crawl",
|
||||
"surfsense_list_scraper_runs",
|
||||
"surfsense_get_scraper_run",
|
||||
|
|
@ -134,7 +136,7 @@ const FAQ: FaqItem[] = [
|
|||
{
|
||||
question: "What is the SurfSense MCP server?",
|
||||
answer:
|
||||
"It is a Model Context Protocol server that exposes your SurfSense workspace to MCP clients like Claude Code, Cursor, and Claude Desktop. Your agents get native tools for every scraper API (Reddit, YouTube, Instagram, TikTok, Amazon, Google Maps, Google Search, web crawl) and for searching, reading, and writing your knowledge base.",
|
||||
"It is a Model Context Protocol server that exposes your SurfSense workspace to MCP clients like Claude Code, Cursor, and Claude Desktop. Your agents get native tools for every scraper API (Reddit, YouTube, Instagram, TikTok, Amazon, Walmart, Google Maps, Google Search, web crawl) and for searching, reading, and writing your knowledge base.",
|
||||
},
|
||||
{
|
||||
question: "Which MCP clients does it work with?",
|
||||
|
|
@ -222,9 +224,9 @@ export default function McpServerPage() {
|
|||
</h1>
|
||||
<p className="mt-5 max-w-xl text-base leading-relaxed text-muted-foreground sm:text-lg">
|
||||
The SurfSense MCP server hands Claude, Cursor, or any MCP client the whole platform:
|
||||
scrape Reddit, YouTube, Instagram, TikTok, Amazon, Google Maps, Google Search, and
|
||||
the open web, and search, read, and write your knowledge base. One API key, typed
|
||||
tools, pay as you go.
|
||||
scrape Reddit, YouTube, Instagram, TikTok, Amazon, Walmart, Google Maps, Google
|
||||
Search, and the open web, and search, read, and write your knowledge base. One API
|
||||
key, typed tools, pay as you go.
|
||||
</p>
|
||||
<div className="mt-8 flex flex-wrap items-center gap-3">
|
||||
<Button asChild size="lg">
|
||||
|
|
|
|||
|
|
@ -658,8 +658,8 @@ export function HeroSection() {
|
|||
>
|
||||
SurfSense is the open-source NotebookLM alternative for AI agents, an open web
|
||||
research platform with live data connectors. Your AI agents research the live web with
|
||||
structured data from Reddit, YouTube, Instagram, TikTok, Amazon, Google Maps, Google
|
||||
Search, and any page on the open web.
|
||||
structured data from Reddit, YouTube, Instagram, TikTok, Amazon, Walmart, Google
|
||||
Maps, Google Search, and any page on the open web.
|
||||
</p>
|
||||
|
||||
<div className="relative mb-4 flex w-full flex-col justify-center gap-y-2 sm:flex-row sm:justify-start sm:space-y-0 sm:space-x-4">
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ const PATHS: {
|
|||
eyebrow: "For developers & agents",
|
||||
title: "The whole platform is programmable",
|
||||
description:
|
||||
"Everything SurfSense agents can do is a typed REST API: scrape Reddit, YouTube, TikTok, Amazon, Google Maps, Google Search, and the open web, search the knowledge base, run automations. One key, JSON in and out, $5 free credit, pay as you go. Already running agents in Claude, Cursor, or your own harness? The SurfSense MCP server hands them the same tools natively.",
|
||||
"Everything SurfSense agents can do is a typed REST API: scrape Reddit, YouTube, TikTok, Amazon, Walmart, Google Maps, Google Search, and the open web, search the knowledge base, run automations. One key, JSON in and out, $5 free credit, pay as you go. Already running agents in Claude, Cursor, or your own harness? The SurfSense MCP server hands them the same tools natively.",
|
||||
links: [
|
||||
{ label: "Read the docs", href: "/docs" },
|
||||
{ label: "SurfSense MCP server", href: "/mcp-server" },
|
||||
|
|
|
|||
|
|
@ -76,11 +76,11 @@ export function SoftwareApplicationJsonLd() {
|
|||
"Free self-hosted from the open-source repo; cloud starts with $5 of free credit, then pay as you go",
|
||||
},
|
||||
description:
|
||||
"SurfSense is an open-source NotebookLM alternative for AI agents. It researches the live web with platform-native connectors for Reddit, YouTube, TikTok, Amazon, Google Maps, Google Search, and any page on the open web, through one API or MCP server.",
|
||||
"SurfSense is an open-source NotebookLM alternative for AI agents. It researches the live web with platform-native connectors for Reddit, YouTube, TikTok, Amazon, Walmart, Google Maps, Google Search, and any page on the open web, through one API or MCP server.",
|
||||
url: "https://www.surfsense.com",
|
||||
downloadUrl: "https://github.com/MODSetter/SurfSense/releases",
|
||||
featureList: [
|
||||
"Platform-native connectors: Reddit, YouTube, TikTok, Amazon, Google Maps, Google Search, Web Crawl",
|
||||
"Platform-native connectors: Reddit, YouTube, TikTok, Amazon, Walmart, Google Maps, Google Search, Web Crawl",
|
||||
"MCP server that exposes every connector as a native agent tool",
|
||||
"Agent harness with retries, structured output, and credit metering",
|
||||
"Live web research with cited briefs and alerts",
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ Connectors bring data into SurfSense — either as searchable knowledge or as li
|
|||
<Card
|
||||
icon={<Zap />}
|
||||
title="Native Connectors"
|
||||
description="SurfSense's built-in scraper APIs: Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed, Amazon, and Web Crawl — usable in chat, the API Playground, or via REST"
|
||||
description="SurfSense's built-in scraper APIs: Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed, Amazon, Walmart, and Web Crawl — usable in chat, the API Playground, or via REST"
|
||||
href="/docs/connectors/native"
|
||||
/>
|
||||
<Card
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
---
|
||||
title: Native Connectors
|
||||
description: SurfSense's built-in scraper APIs for Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed, Amazon, and the web
|
||||
description: SurfSense's built-in scraper APIs for Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed, Amazon, Walmart, and the web
|
||||
---
|
||||
|
||||
import { Card, Cards } from 'fumadocs-ui/components/card';
|
||||
|
|
@ -48,6 +48,11 @@ Native connectors are SurfSense's own scraper APIs — built into the platform,
|
|||
description="Public product data: prices, ratings, offers, sellers, and best-seller ranks"
|
||||
href="/docs/connectors/native/amazon"
|
||||
/>
|
||||
<Card
|
||||
title="Walmart"
|
||||
description="Public product data and deep reviews: prices, ratings, sellers, and variants"
|
||||
href="/docs/connectors/native/walmart"
|
||||
/>
|
||||
<Card
|
||||
title="Web Crawl"
|
||||
description="Scrape any page or spider a whole site into clean markdown"
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@
|
|||
"google-search",
|
||||
"indeed",
|
||||
"amazon",
|
||||
"walmart",
|
||||
"web-crawl"
|
||||
],
|
||||
"defaultOpen": false
|
||||
|
|
|
|||
71
surfsense_web/content/docs/connectors/native/walmart.mdx
Normal file
71
surfsense_web/content/docs/connectors/native/walmart.mdx
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
---
|
||||
title: Walmart
|
||||
description: Scrape public Walmart product data and deep reviews as structured JSON
|
||||
---
|
||||
|
||||
The Walmart scraper returns public product data as structured JSON: price and list price, rating and review count, availability, 1P and marketplace sellers, variants, specifications, and a free on-page review sample. A second verb pages the full public review history for a product. It only uses public, anonymous data — no login or seller account. Give it search terms or Walmart product, search, category, or browse URLs.
|
||||
|
||||
Walmart data is the US marketplace (`walmart.com`). The scraper reads the JSON Walmart server-renders into each page rather than the rendered DOM, which keeps extraction stable against the frequent front-end A/B testing on the site.
|
||||
|
||||
## Scrape products
|
||||
|
||||
```bash
|
||||
POST /api/v1/workspaces/{workspace_id}/scrapers/walmart/scrape
|
||||
```
|
||||
|
||||
### Inputs
|
||||
|
||||
| Field | Default | Description |
|
||||
|-------|---------|-------------|
|
||||
| `urls` | — | Walmart product (`/ip/`), search (`/search`), category (`/cp/`), or browse (`/browse/`) URLs. Provide `urls` or `search_terms` |
|
||||
| `search_terms` | — | Search phrases run on `walmart.com`, e.g. `air fryer`. Provide `search_terms` or `urls` |
|
||||
| `max_items` | `10` | Max products per search term or category/browse URL (1–100) |
|
||||
| `include_details` | `true` | Fetch full product detail pages; `false` returns faster card-only results |
|
||||
| `include_reviews_sample` | `true` | Attach the free on-page review sample from each detail page. For full history use the reviews verb |
|
||||
|
||||
At least one of `urls` or `search_terms` is required, with up to 20 combined sources per call.
|
||||
|
||||
```bash
|
||||
curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/walmart/scrape" \
|
||||
-H "Authorization: Bearer $SURFSENSE_API_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"search_terms": ["air fryer"],
|
||||
"max_items": 5,
|
||||
"include_reviews_sample": true
|
||||
}'
|
||||
```
|
||||
|
||||
The response is `{ "items": [...] }` — one item per product, plus structured error items for any input that could not be resolved. Billing is per returned product; error items are never billed.
|
||||
|
||||
## Fetch reviews
|
||||
|
||||
```bash
|
||||
POST /api/v1/workspaces/{workspace_id}/scrapers/walmart/reviews
|
||||
```
|
||||
|
||||
### Inputs
|
||||
|
||||
| Field | Default | Description |
|
||||
|-------|---------|-------------|
|
||||
| `urls` | — | Walmart product (`/ip/`) or reviews URLs to fetch reviews for. Provide `urls` or `item_ids` |
|
||||
| `item_ids` | — | Walmart numeric item ids (`usItemId`) to fetch reviews for. Provide `item_ids` or `urls` |
|
||||
| `max_reviews` | `200` | Max reviews per product (1–5000); reviews are paged 10 at a time |
|
||||
| `sort_by` | `most-recent` | Review ordering: `most-recent`, `most-helpful`, `rating-high`, or `rating-low` |
|
||||
|
||||
At least one of `urls` or `item_ids` is required, with up to 20 combined sources per call.
|
||||
|
||||
```bash
|
||||
curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/walmart/reviews" \
|
||||
-H "Authorization: Bearer $SURFSENSE_API_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"urls": ["https://www.walmart.com/ip/1234567890"],
|
||||
"max_reviews": 500,
|
||||
"sort_by": "most-recent"
|
||||
}'
|
||||
```
|
||||
|
||||
The response is `{ "items": [...] }` — one item per review with rating, title, text, author, date, verified-purchase flag, helpful votes, images, and any seller response. Billing is per returned review; error items are never billed.
|
||||
|
||||
For the full input and output JSON schemas and generated code snippets in your language, open **API Playground → Walmart** in your workspace.
|
||||
|
|
@ -7,7 +7,7 @@ import { Tab, Tabs } from 'fumadocs-ui/components/tabs';
|
|||
|
||||
# SurfSense MCP Server
|
||||
|
||||
The SurfSense MCP server exposes your workspace to any [Model Context Protocol](https://modelcontextprotocol.io/) client. Your agent gets 26 native, typed tools: every scraper (Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed, Amazon, web crawl), full knowledge-base access (search, read, add, upload, update, delete), and a workspace selector.
|
||||
The SurfSense MCP server exposes your workspace to any [Model Context Protocol](https://modelcontextprotocol.io/) client. Your agent gets 28 native, typed tools: every scraper (Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed, Amazon, Walmart, web crawl), full knowledge-base access (search, read, add, upload, update, delete), and a workspace selector.
|
||||
|
||||
Connect it two ways: the **hosted** server at `https://mcp.surfsense.com/mcp` (nothing to install — just an API key), or run it yourself over **stdio** against any SurfSense backend, cloud or self-hosted.
|
||||
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ export const CHAT_EXAMPLE_CATEGORIES: ChatExampleCategory[] = [
|
|||
"Analyze the comments on [channel]'s last 10 videos and cluster the complaints",
|
||||
"What is Reddit saying about [topic] this week?",
|
||||
"Pull the top TikTok videos for [hashtag] and summarize the trend",
|
||||
"Summarize the top complaints in the Walmart reviews for [product]",
|
||||
],
|
||||
},
|
||||
{
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import { instagram } from "./instagram";
|
|||
import { reddit } from "./reddit";
|
||||
import { tiktok } from "./tiktok";
|
||||
import type { ConnectorPageContent } from "./types";
|
||||
import { walmart } from "./walmart";
|
||||
import { webCrawl } from "./web-crawl";
|
||||
import { youtube } from "./youtube";
|
||||
|
||||
|
|
@ -21,6 +22,7 @@ const CONNECTOR_LIST: ConnectorPageContent[] = [
|
|||
googleSearch,
|
||||
indeed,
|
||||
amazon,
|
||||
walmart,
|
||||
webCrawl,
|
||||
];
|
||||
|
||||
|
|
|
|||
273
surfsense_web/lib/connectors-marketing/walmart.tsx
Normal file
273
surfsense_web/lib/connectors-marketing/walmart.tsx
Normal file
|
|
@ -0,0 +1,273 @@
|
|||
import { IconBrandWalmart } from "@tabler/icons-react";
|
||||
import type { ConnectorPageContent } from "./types";
|
||||
|
||||
export const walmart: ConnectorPageContent = {
|
||||
slug: "walmart",
|
||||
name: "Walmart",
|
||||
cardTitle: "Walmart Product & Review API",
|
||||
icon: IconBrandWalmart,
|
||||
|
||||
metaTitle: "Walmart Product & Review API | SurfSense",
|
||||
metaDescription:
|
||||
"Scrape public Walmart product data and deep customer reviews as structured JSON: prices, ratings, sellers, variants, availability, and full review history. Start free.",
|
||||
keywords: [
|
||||
"walmart product api",
|
||||
"walmart scraper api",
|
||||
"walmart review scraper",
|
||||
"walmart reviews api",
|
||||
"scrape walmart product data",
|
||||
"walmart price scraper",
|
||||
"walmart product data api",
|
||||
"walmart price tracking api",
|
||||
"walmart competitor monitoring",
|
||||
"walmart seller data",
|
||||
"walmart marketplace api",
|
||||
"ecommerce product api",
|
||||
],
|
||||
|
||||
h1: "Walmart Product and Review API",
|
||||
heroLede:
|
||||
"The SurfSense Walmart API scrapes public Walmart.com listings as structured JSON: price and list price, rating and review count, availability, 1P and marketplace sellers, variants, and specifications. A second verb pages the full public review history — ratings, text, authors, verified-purchase flags, images, and seller responses. Point your AI agents at a search term or product URL — no login, only public data.",
|
||||
|
||||
transcript: {
|
||||
prompt: "Pull the ratings and top complaints for this Walmart air fryer",
|
||||
toolCall:
|
||||
'walmart.reviews({ urls: ["walmart.com/ip/..."],\n max_reviews: 500, sort_by: "most-recent" })',
|
||||
rows: [
|
||||
{
|
||||
primary: "Ninja 4-Qt Air Fryer — $79.00 (was $99.00)",
|
||||
secondary: "4.6 stars · 8,204 reviews · In Stock",
|
||||
tag: "-20%",
|
||||
},
|
||||
{
|
||||
primary: "512 reviews paged, 71% verified purchase",
|
||||
secondary: "sorted most-recent · 132 with photos",
|
||||
tag: "reviews",
|
||||
},
|
||||
{
|
||||
primary: 'Top complaint: "basket coating flaking after 3 months"',
|
||||
secondary: "surfaced across 24 recent 1-2 star reviews",
|
||||
tag: "theme",
|
||||
},
|
||||
],
|
||||
resultSummary: "1 product · 512 reviews · surfaced in 6.4s",
|
||||
},
|
||||
|
||||
extractIntro:
|
||||
"Give the scraper Walmart product, search, category, or browse URLs, or plain search terms. The scrape verb returns product cards and full detail with a free on-page review sample; the reviews verb pages the complete public review history for any product. Walmart data is US marketplace (walmart.com), server-rendered as JSON for stability.",
|
||||
extractFields: [
|
||||
{
|
||||
label: "Product core",
|
||||
description:
|
||||
"Name, item id (usItemId), brand, price, list price, availability, in-stock flag, and canonical URL for every product.",
|
||||
},
|
||||
{
|
||||
label: "Ratings & review sample",
|
||||
description:
|
||||
"Average stars and total review count on every product, plus a free sample of on-page reviews when you fetch full detail.",
|
||||
},
|
||||
{
|
||||
label: "Deep reviews",
|
||||
description:
|
||||
"The reviews verb pages the full public review history: rating, title, text, author, date, verified-purchase flag, helpful votes, images, and seller responses.",
|
||||
},
|
||||
{
|
||||
label: "Sellers",
|
||||
description:
|
||||
"The seller behind each product, typed as Walmart first-party (1P) or third-party marketplace (3P), with id and name.",
|
||||
},
|
||||
{
|
||||
label: "Variants & media",
|
||||
description:
|
||||
"Product variants (color, size, count) with their own item ids and prices, plus the gallery and thumbnail images.",
|
||||
},
|
||||
{
|
||||
label: "Content & taxonomy",
|
||||
description:
|
||||
"Short and long descriptions, specifications, category, and breadcrumb trail for catalog enrichment and classification.",
|
||||
},
|
||||
],
|
||||
|
||||
useCasesHeading: "What teams do with the Walmart API",
|
||||
useCases: [
|
||||
{
|
||||
title: "Review mining and product research",
|
||||
description:
|
||||
"Page the full review history for a product and brief an agent to cluster complaints, track sentiment over time, and separate verified-purchase signal from noise. Walmart exposes far more review depth than most marketplaces — use it to inform your roadmap or listing copy.",
|
||||
},
|
||||
{
|
||||
title: "Price and buy-box tracking",
|
||||
description:
|
||||
"Watch your own and competitors' prices, list prices, and which seller wins the offer (Walmart 1P vs a third-party marketplace seller). Run the same URLs on a schedule and diff the results to alert on undercuts and stockouts.",
|
||||
},
|
||||
{
|
||||
title: "Seller and marketplace intelligence",
|
||||
description:
|
||||
"See which third-party sellers are winning offers on the products that matter to you, and how first-party Walmart pricing compares, to size the marketplace opportunity in your category.",
|
||||
},
|
||||
{
|
||||
title: "Catalog and assortment enrichment",
|
||||
description:
|
||||
"Enrich a catalog by item id with structured attributes, variants, specifications, and images, and discover products from search or category URLs to map an entire assortment.",
|
||||
},
|
||||
],
|
||||
|
||||
comparison: {
|
||||
heading: "A Walmart scraper API built for agents",
|
||||
intro:
|
||||
"Most Walmart data APIs bill per request, split product and review data across endpoints, and leave the discovery-to-detail logic to you. Here is how SurfSense compares.",
|
||||
columnLabel: "Typical Walmart data API",
|
||||
rows: [
|
||||
{
|
||||
feature: "Pricing",
|
||||
official: "Per-request pricing that climbs fast at scale",
|
||||
surfsense: "Pay per product or per review returned, with a free tier to start",
|
||||
},
|
||||
{
|
||||
feature: "Discovery",
|
||||
official: "Separate search and detail endpoints you stitch together",
|
||||
surfsense: "One verb takes search terms or product, search, category, and browse URLs",
|
||||
},
|
||||
{
|
||||
feature: "Reviews",
|
||||
official: "A separate paid endpoint, often capped shallow",
|
||||
surfsense: "A dedicated verb pages the full public review history with photos and seller replies",
|
||||
},
|
||||
{
|
||||
feature: "Data access",
|
||||
official: "Requires accounts, keys, or approval for many providers",
|
||||
surfsense: "Public, anonymous data only — no login or seller account",
|
||||
},
|
||||
{
|
||||
feature: "Agent-ready",
|
||||
official: "No; you wire the harness yourself",
|
||||
surfsense: "MCP server exposes walmart.scrape and walmart.reviews as native tools",
|
||||
},
|
||||
],
|
||||
},
|
||||
|
||||
api: {
|
||||
platform: "walmart",
|
||||
verb: "scrape",
|
||||
mcpTool: "walmart.scrape",
|
||||
requestBody: {
|
||||
search_terms: ["air fryer"],
|
||||
max_items: 5,
|
||||
include_reviews_sample: true,
|
||||
},
|
||||
},
|
||||
|
||||
schema: {
|
||||
requestNote:
|
||||
"walmart.scrape: provide urls or search_terms (at least one), up to 20 combined sources per call. For the full review history, use the walmart.reviews verb with product urls or item_ids.",
|
||||
request: [
|
||||
{
|
||||
name: "urls",
|
||||
type: "string[]",
|
||||
description:
|
||||
"Walmart product (/ip/), search (/search), category (/cp/), or browse (/browse/) URLs. Provide urls or search_terms.",
|
||||
},
|
||||
{
|
||||
name: "search_terms",
|
||||
type: "string[]",
|
||||
description:
|
||||
"Search phrases run on walmart.com, e.g. 'air fryer'. Provide search_terms or urls.",
|
||||
},
|
||||
{
|
||||
name: "max_items",
|
||||
type: "integer",
|
||||
defaultValue: "10",
|
||||
description: "Max products per search term or category/browse URL, 1 to 100.",
|
||||
},
|
||||
{
|
||||
name: "include_details",
|
||||
type: "boolean",
|
||||
defaultValue: "true",
|
||||
description: "Fetch full product detail pages; false returns faster card-only results.",
|
||||
},
|
||||
{
|
||||
name: "include_reviews_sample",
|
||||
type: "boolean",
|
||||
defaultValue: "true",
|
||||
description:
|
||||
"Attach the free on-page review sample from each detail page. For full history use walmart.reviews.",
|
||||
},
|
||||
],
|
||||
responseNote:
|
||||
"The response is { items: [...] } with one item per product. One returned product is one billable unit; error items are never billed.",
|
||||
response: [
|
||||
{
|
||||
name: "name / usItemId / brand",
|
||||
type: "string",
|
||||
description: "Product identity: display name, Walmart item id, and brand.",
|
||||
},
|
||||
{
|
||||
name: "price / listPrice",
|
||||
type: "object",
|
||||
description: "Current price and strike-through list price, each with value and currency.",
|
||||
},
|
||||
{
|
||||
name: "stars / reviewsCount",
|
||||
type: "number",
|
||||
description: "Average star rating and total public review count.",
|
||||
},
|
||||
{
|
||||
name: "seller",
|
||||
type: "object",
|
||||
description: "Offer seller with id, name, and type (WALMART 1P or MARKETPLACE 3P).",
|
||||
},
|
||||
{
|
||||
name: "availabilityStatus / inStock",
|
||||
type: "string",
|
||||
description: "Availability label and a boolean in-stock flag.",
|
||||
},
|
||||
{
|
||||
name: "variants",
|
||||
type: "object[]",
|
||||
description: "Product variants (color, size, count) with their own item ids and prices.",
|
||||
},
|
||||
{
|
||||
name: "reviewsSample",
|
||||
type: "object",
|
||||
description: "Free sample of on-page reviews returned with full detail pages.",
|
||||
},
|
||||
],
|
||||
},
|
||||
|
||||
faq: [
|
||||
{
|
||||
question: "What is the Walmart product API?",
|
||||
answer:
|
||||
"It returns a Walmart listing's data as structured JSON instead of raw HTML. The SurfSense Walmart API scrapes public product pages and gives you price, rating, seller, variants, availability, and specifications as clean JSON your agents can read.",
|
||||
},
|
||||
{
|
||||
question: "Can I get the full Walmart review history?",
|
||||
answer:
|
||||
"Yes. The walmart.reviews verb pages the complete public review history for a product — rating, title, text, author, date, verified-purchase flag, helpful votes, images, and seller responses. Pass product URLs or item ids and set max_reviews and sort_by.",
|
||||
},
|
||||
{
|
||||
question: "Does it only use public data?",
|
||||
answer:
|
||||
"It does. The scraper collects public, anonymous Walmart data only — no login or seller account. That covers product details, prices, ratings, public reviews, seller identity, variants, and availability.",
|
||||
},
|
||||
{
|
||||
question: "How do I look up a product by item id or search term?",
|
||||
answer:
|
||||
"Pass a product URL (which contains the item id) or a phrase in search_terms to discover products. You can also pass search, category, and browse URLs. walmart.scrape handles discovery; walmart.reviews takes urls or item_ids directly.",
|
||||
},
|
||||
{
|
||||
question: "Which Walmart marketplace is supported?",
|
||||
answer:
|
||||
"The scraper targets the US marketplace at walmart.com. Data is read from the server-rendered JSON Walmart ships in the page, which keeps extraction stable against the frequent front-end A/B testing on the site.",
|
||||
},
|
||||
],
|
||||
|
||||
related: [
|
||||
{ label: "Amazon Product API", href: "/amazon" },
|
||||
{ label: "Google Maps API", href: "/google-maps" },
|
||||
{ label: "Google Search API", href: "/google-search" },
|
||||
{ label: "Reddit API", href: "/reddit" },
|
||||
{ label: "SurfSense MCP Server", href: "/mcp-server" },
|
||||
{ label: "Read the docs", href: "/docs" },
|
||||
],
|
||||
};
|
||||
|
|
@ -7,6 +7,7 @@ import {
|
|||
InstagramIcon,
|
||||
RedditIcon,
|
||||
TikTokIcon,
|
||||
WalmartIcon,
|
||||
WebIcon,
|
||||
YouTubeIcon,
|
||||
} from "./platform-icons";
|
||||
|
|
@ -103,6 +104,15 @@ export const PLAYGROUND_PLATFORMS: PlaygroundPlatform[] = [
|
|||
icon: AmazonIcon,
|
||||
verbs: [{ name: "amazon.scrape", verb: "scrape", label: "Scrape" }],
|
||||
},
|
||||
{
|
||||
id: "walmart",
|
||||
label: "Walmart",
|
||||
icon: WalmartIcon,
|
||||
verbs: [
|
||||
{ name: "walmart.scrape", verb: "scrape", label: "Scrape" },
|
||||
{ name: "walmart.reviews", verb: "reviews", label: "Reviews" },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "web",
|
||||
label: "Web",
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ function brandIcon(src: string, alt: string) {
|
|||
}
|
||||
|
||||
export const AmazonIcon = brandIcon("/connectors/amazon.svg", "Amazon");
|
||||
export const WalmartIcon = brandIcon("/connectors/walmart.svg", "Walmart");
|
||||
export const RedditIcon = brandIcon("/connectors/reddit.svg", "Reddit");
|
||||
export const YouTubeIcon = brandIcon("/connectors/youtube.svg", "YouTube");
|
||||
export const InstagramIcon = brandIcon("/connectors/instagram.svg", "Instagram");
|
||||
|
|
|
|||
11
surfsense_web/public/connectors/walmart.svg
Normal file
11
surfsense_web/public/connectors/walmart.svg
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
<svg width="48" height="48" viewBox="0 0 48 48" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<title>Walmart</title>
|
||||
<g fill="#ffc220">
|
||||
<rect x="21.4" y="3.5" width="5.2" height="14" rx="2.6" transform="rotate(0 24 24)" />
|
||||
<rect x="21.4" y="3.5" width="5.2" height="14" rx="2.6" transform="rotate(60 24 24)" />
|
||||
<rect x="21.4" y="3.5" width="5.2" height="14" rx="2.6" transform="rotate(120 24 24)" />
|
||||
<rect x="21.4" y="3.5" width="5.2" height="14" rx="2.6" transform="rotate(180 24 24)" />
|
||||
<rect x="21.4" y="3.5" width="5.2" height="14" rx="2.6" transform="rotate(240 24 24)" />
|
||||
<rect x="21.4" y="3.5" width="5.2" height="14" rx="2.6" transform="rotate(300 24 24)" />
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 711 B |
Loading…
Add table
Add a link
Reference in a new issue