diff --git a/docker/.env.example b/docker/.env.example index 355dbb6c0..78c0bd93a 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -441,7 +441,7 @@ SURFSENSE_ENABLE_DOOM_LOOP=true # WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE=3000 # Debit the credit wallet per *item returned* by the platform-native scrapers -# (Reddit, Google Search, Google Maps, Amazon, YouTube). Default FALSE keeps scraping +# (Reddit, Google Search, Google Maps, Amazon, Walmart, YouTube). Default FALSE keeps scraping # effectively free for self-hosted installs. Each rate is micro-USD per item, # config-driven: = round(USD_per_1000_items * 1_000). Defaults sit # at/above Apify's first-party actor rates (we charge no subscription tiers, @@ -459,6 +459,8 @@ SURFSENSE_ENABLE_DOOM_LOOP=true # TIKTOK_MICROS_PER_USER=2500 # TIKTOK_MICROS_PER_COMMENT=1500 # INDEED_SCRAPE_MICROS_PER_JOB=3500 +# WALMART_MICROS_PER_PRODUCT=3500 +# WALMART_MICROS_PER_REVIEW=1500 # Safety ceiling on per-call premium reservation, in micro-USD ($1.00 default). # QUOTA_MAX_RESERVE_MICROS=1000000 diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index 9af1f7645..266985e6c 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -277,8 +277,8 @@ MICROS_PER_PAGE=1000 # WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE=3000 # Debit the credit wallet per *item returned* by the platform-native scrapers -# (Reddit, Google Search, Google Maps, Amazon, YouTube). Default FALSE keeps -# scraping effectively free for self-hosted/OSS installs; hosted set TRUE. +# (Reddit, Google Search, Google Maps, Amazon, Walmart, YouTube). Default FALSE +# keeps scraping effectively free for self-hosted/OSS installs; hosted set TRUE. # Each rate is micro-USD per item, fully config-driven (no hardcoded rate): # = round(USD_per_1000_items * 1_000) # 3500 == $3.50/1000 | 5000 == $5/1000 | 2000 == $2/1000 @@ -302,6 +302,11 @@ MICROS_PER_PAGE=1000 # Browser-listing retries when a feed is empty (profile feed is withheld from # flagged IPs; each retry draws a fresh rotating exit IP). Set to 1 for a static IP. # TIKTOK_LISTING_MAX_ATTEMPTS=3 +# Walmart products (server-rendered JSON behind residential proxies) priced like +# Amazon; reviews are 10/page (many light requests) so priced on the per-review +# market like Google Maps reviews. +# WALMART_MICROS_PER_PRODUCT=3500 +# WALMART_MICROS_PER_REVIEW=1500 # Low-balance warning threshold (micro-USD), surfaced to the UI. Default $0.50. CREDIT_LOW_BALANCE_WARNING_MICROS=500000 diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py b/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py index 681675c8c..0fb2fc336 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py @@ -40,6 +40,7 @@ SUBAGENT_TO_REQUIRED_CONNECTOR_MAP: dict[str, frozenset[str]] = { "reddit": frozenset(), "instagram": frozenset(), "tiktok": frozenset(), + "walmart": frozenset(), "mcp_discovery": frozenset( { "SLACK_CONNECTOR", diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/private.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/private.md index 82444634a..56e933253 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/private.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/private.md @@ -6,7 +6,7 @@ are changing, and what is being published across the open web — and to put that research to work alongside their own knowledge base. You do this by dispatching **specialist subagents** via the `task` tool: -- **Live web data** — Reddit, YouTube, Instagram, TikTok, Amazon, Google +- **Live web data** — Reddit, YouTube, Instagram, TikTok, Amazon, Walmart, Google Maps, Google Search, and the web crawler return structured, current platform data (posts, comments, transcripts, videos, products, reviews, SERPs, full page content). diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/team.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/team.md index 38cec63dc..a16a8bfd2 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/team.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/team.md @@ -6,7 +6,7 @@ are changing, and what is being published across the open web — and to put that research to work alongside the team's shared knowledge base. You do this by dispatching **specialist subagents** via the `task` tool: -- **Live web data** — Reddit, YouTube, Instagram, TikTok, Amazon, Google +- **Live web data** — Reddit, YouTube, Instagram, TikTok, Amazon, Walmart, Google Maps, Google Search, and the web crawler return structured, current platform data (posts, comments, transcripts, videos, products, reviews, SERPs, full page content). diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md index 3bae85262..60192fb39 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md @@ -2,7 +2,8 @@ CRITICAL — ground factual answers in what you actually receive this turn: - **live platform data** via the market specialists — `task(reddit, ...)`, `task(youtube, ...)`, `task(instagram, ...)`, - `task(tiktok, ...)`, `task(amazon, ...)`, `task(google_maps, ...)`, + `task(tiktok, ...)`, `task(amazon, ...)`, `task(walmart, ...)`, + `task(google_maps, ...)`, `task(google_search, ...)`, `task(web_crawler, ...)`. Anything about competitors, markets, rankings, reviews, or audience sentiment is answered from what these return **this turn**, never from your training data: your diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md index fb818cabc..7e2a0f983 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md @@ -32,7 +32,9 @@ about a brand, product, or topic is answered from the platform where they say it — `task(reddit, …)` for community discussion and threads, `task(youtube, …)` for video content, transcripts, and comment sections, `task(tiktok, …)` for short-form video trends by hashtag or search, -`task(google_maps, …)` for customer reviews of physical businesses. Web +`task(google_maps, …)` for customer reviews of physical businesses, +`task(amazon, …)` / `task(walmart, …)` for product ratings and customer +reviews of retail products (Walmart pages the full review history). Web search only finds articles *about* the conversation; the platform specialists return the conversation itself, structured and current. For competitive questions ("what are people saying about X", "how is Y diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/__init__.py new file mode 100644 index 000000000..8126071cf --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/__init__.py @@ -0,0 +1 @@ +"""``walmart`` builtin subagent: structured public Walmart product data and reviews.""" diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/agent.py new file mode 100644 index 000000000..794b050a2 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/agent.py @@ -0,0 +1,43 @@ +"""``walmart`` route: ``SurfSenseSubagentSpec`` builder for deepagents.""" + +from __future__ import annotations + +from typing import Any + +from langchain_core.language_models import BaseChatModel +from langchain_core.tools import BaseTool + +from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( + read_md_file, +) +from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec +from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( + pack_subagent, +) + +from .tools.index import NAME, RULESET, load_tools + + +def build_subagent( + *, + dependencies: dict[str, Any], + model: BaseChatModel | None = None, + middleware_stack: dict[str, Any] | None = None, + mcp_tools: list[BaseTool] | None = None, +) -> SurfSenseSubagentSpec: + tools = [*load_tools(dependencies=dependencies), *(mcp_tools or [])] + description = ( + read_md_file(__package__, "description").strip() + or "Scrapes public Walmart product data and reviews for a URL or search term." + ) + system_prompt = read_md_file(__package__, "system_prompt").strip() + return pack_subagent( + name=NAME, + description=description, + system_prompt=system_prompt, + tools=tools, + ruleset=RULESET, + dependencies=dependencies, + model=model, + middleware_stack=middleware_stack, + ) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/description.md new file mode 100644 index 000000000..88c6b7c28 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/description.md @@ -0,0 +1,2 @@ +Walmart product specialist: scrapes public Walmart listings and returns structured product data — title, item id (usItemId), brand, price and list price, star rating and review count, availability, images, features, seller, and product variants — plus deep paginated customer reviews (rating, text, author, verified-purchase flag, images, and seller responses). Works from a search term (e.g. "air fryer") or from Walmart product (/ip/...), search, category, or browse URLs, and can pull many reviews per product by item id or URL. Only public, anonymous US Walmart data — no login or seller account. +Use it for product research, price tracking, catalog enrichment by item id, and review mining. Triggers include "find X on Walmart", "Walmart price of X", "reviews for this Walmart product", "compare these Walmart products", and "look up this Walmart URL/item id". Not for general web search (use the Google Search specialist), reading an arbitrary non-Walmart page (use the web crawling specialist), or other marketplaces. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/system_prompt.md new file mode 100644 index 000000000..bf30fe2bb --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/system_prompt.md @@ -0,0 +1,68 @@ +You are the SurfSense Walmart sub-agent. +You receive delegated instructions from a supervisor agent and return structured results for supervisor synthesis. + + +Answer the delegated question from public Walmart product data and reviews gathered with your verbs, comparing against earlier results already in this conversation when the task calls for it. + + + +- `walmart_scrape` — product details and search/category listings +- `walmart_reviews` — deep paginated reviews for a product +- `read_run` / `search_run` (free readers for stored scrape output) + + + +- Discovering products: call `walmart_scrape` with `search_terms` (e.g. ["air fryer"]). +- Specific products: pass Walmart product URLs (/ip/...) or search/category/browse URLs in `urls`. +- Faster listings: set `include_details=false` to return card-only results without opening each product page. +- Sampled reviews: `walmart_scrape` returns a small on-page review sample by default (`include_reviews_sample=true`); disable it when reviews are irrelevant. +- Deep review mining: use `walmart_reviews` with product `urls` or numeric `item_ids` (usItemId); raise `max_reviews` and set `sort_by` (most-recent, most-helpful, rating-high, rating-low) as the task needs. Reviews are billed per review, so keep `max_reviews` to what the task actually requires. +- Batch multiple URLs or search terms into one call rather than many single-source calls. + +- Comparison requests: pull the current products, compare against prior values already in this conversation's earlier tool results, and report concrete deltas (price up/down, rating change, stock changes). + + + +- Use only tools in ``. +- Report only results present in the tool output. Never invent titles, item ids, prices, ratings, or reviews. +- `walmart_scrape`: provide at least one of `urls` or `search_terms`. +- `walmart_reviews`: provide at least one of `urls` or `item_ids`. + + + +- Do not perform general web search — that is the Google Search specialist's job. +- Do not read or extract an arbitrary non-Walmart page — return the URL for the web crawling specialist. +- Do not generate deliverables or perform connector mutations; return findings for the supervisor to act on. +- Only public, anonymous Walmart data — never anything behind a login or seller account. + + + +- Report uncertainty explicitly when evidence is incomplete or conflicting. +- Never present unverified claims as facts. + + + +- Underspecified request — no usable search term, URL, or item id — return `status=blocked` with the missing fields. +- Tool failure: return `status=error` with a concise recovery `next_step`. +- No useful evidence: return `status=blocked` with a narrower query or the scope you still need. + + + +Return **only** one JSON object (no markdown/prose): +{ + "status": "success" | "partial" | "blocked" | "error", + "action_summary": string, + "evidence": { + "findings": string[], + "sources": string[], + "confidence": "high" | "medium" | "low" + }, + "next_step": string | null, + "missing_fields": string[] | null, + "assumptions": string[] | null +} + +Route-specific rules: +- `evidence.findings`: max 10 entries, each a single sentence stating one distinct product, review theme, or delta. Do not paste raw payloads. +- `evidence.sources`: max 10 URLs, one per finding when applicable. List each URL once. + diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/tools/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/tools/index.py new file mode 100644 index 000000000..ac3e5ed81 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/tools/index.py @@ -0,0 +1,28 @@ +"""``walmart`` sub-agent tools: the Walmart product scrape and reviews verbs.""" + +from __future__ import annotations + +from typing import Any + +from langchain_core.tools import BaseTool + +from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset +from app.capabilities.core.access.agent import build_capability_tools +from app.capabilities.walmart.reviews.definition import WALMART_REVIEWS +from app.capabilities.walmart.scrape.definition import WALMART_SCRAPE + +NAME = "walmart" + +RULESET = Ruleset(origin=NAME, rules=[]) + +_CI_VERBS = [WALMART_SCRAPE, WALMART_REVIEWS] + + +def load_tools( + *, dependencies: dict[str, Any] | None = None, **kwargs: Any +) -> list[BaseTool]: + d = {**(dependencies or {}), **kwargs} + return build_capability_tools( + workspace_id=d.get("workspace_id"), + capabilities=_CI_VERBS, + ) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py index d9ed4a5ff..5ecd2721c 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py @@ -45,6 +45,9 @@ from app.agents.chat.multi_agent_chat.subagents.builtins.reddit.agent import ( from app.agents.chat.multi_agent_chat.subagents.builtins.tiktok.agent import ( build_subagent as build_tiktok_subagent, ) +from app.agents.chat.multi_agent_chat.subagents.builtins.walmart.agent import ( + build_subagent as build_walmart_subagent, +) from app.agents.chat.multi_agent_chat.subagents.builtins.web_crawler.agent import ( build_subagent as build_web_crawler_subagent, ) @@ -100,6 +103,7 @@ SUBAGENT_BUILDERS_BY_NAME: dict[str, SubagentBuilder] = { "onedrive": build_onedrive_subagent, "reddit": build_reddit_subagent, "tiktok": build_tiktok_subagent, + "walmart": build_walmart_subagent, "web_crawler": build_web_crawler_subagent, "youtube": build_youtube_subagent, } diff --git a/surfsense_backend/app/capabilities/core/billing.py b/surfsense_backend/app/capabilities/core/billing.py index d0d059b0e..cdaf1eb9d 100644 --- a/surfsense_backend/app/capabilities/core/billing.py +++ b/surfsense_backend/app/capabilities/core/billing.py @@ -42,6 +42,8 @@ _PLATFORM_RATE_KEYS: dict[BillingUnit, str] = { BillingUnit.TIKTOK_USER: "TIKTOK_MICROS_PER_USER", BillingUnit.TIKTOK_COMMENT: "TIKTOK_MICROS_PER_COMMENT", BillingUnit.INDEED_JOB: "INDEED_SCRAPE_MICROS_PER_JOB", + BillingUnit.WALMART_PRODUCT: "WALMART_MICROS_PER_PRODUCT", + BillingUnit.WALMART_REVIEW: "WALMART_MICROS_PER_REVIEW", } @@ -65,6 +67,8 @@ _UNIT_NOUNS: dict[BillingUnit, str] = { BillingUnit.TIKTOK_USER: "profile", BillingUnit.TIKTOK_COMMENT: "comment", BillingUnit.INDEED_JOB: "job", + BillingUnit.WALMART_PRODUCT: "product", + BillingUnit.WALMART_REVIEW: "review", } diff --git a/surfsense_backend/app/capabilities/core/types.py b/surfsense_backend/app/capabilities/core/types.py index d3d5c089a..8b1d664c1 100644 --- a/surfsense_backend/app/capabilities/core/types.py +++ b/surfsense_backend/app/capabilities/core/types.py @@ -32,6 +32,8 @@ class BillingUnit(StrEnum): TIKTOK_USER = "tiktok_user" TIKTOK_COMMENT = "tiktok_comment" INDEED_JOB = "indeed_job" + WALMART_PRODUCT = "walmart_product" + WALMART_REVIEW = "walmart_review" class BillableInput(Protocol): diff --git a/surfsense_backend/app/capabilities/walmart/__init__.py b/surfsense_backend/app/capabilities/walmart/__init__.py new file mode 100644 index 000000000..1031f4230 --- /dev/null +++ b/surfsense_backend/app/capabilities/walmart/__init__.py @@ -0,0 +1,6 @@ +"""``walmart.*`` namespace: platform-native Walmart data verbs.""" + +from __future__ import annotations + +from app.capabilities.walmart.reviews import definition as _reviews # noqa: F401 +from app.capabilities.walmart.scrape import definition as _scrape # noqa: F401 diff --git a/surfsense_backend/app/capabilities/walmart/reviews/__init__.py b/surfsense_backend/app/capabilities/walmart/reviews/__init__.py new file mode 100644 index 000000000..473d60402 --- /dev/null +++ b/surfsense_backend/app/capabilities/walmart/reviews/__init__.py @@ -0,0 +1,3 @@ +"""Walmart deep review scraping capability.""" + +from __future__ import annotations diff --git a/surfsense_backend/app/capabilities/walmart/reviews/definition.py b/surfsense_backend/app/capabilities/walmart/reviews/definition.py new file mode 100644 index 000000000..6984ee031 --- /dev/null +++ b/surfsense_backend/app/capabilities/walmart/reviews/definition.py @@ -0,0 +1,24 @@ +"""``walmart.reviews`` capability registration (billed per review; see config +``WALMART_MICROS_PER_REVIEW``).""" + +from __future__ import annotations + +from app.capabilities.core import BillingUnit, Capability, register_capability +from app.capabilities.walmart.reviews.executor import build_reviews_executor +from app.capabilities.walmart.reviews.schemas import ReviewsInput, ReviewsOutput + +WALMART_REVIEWS = Capability( + name="walmart.reviews", + description=( + "Fetch deep paginated public Walmart product reviews with ratings, text, " + "authors, verified-purchase flags, images, and seller responses. Use " + "product urls or item ids." + ), + input_schema=ReviewsInput, + output_schema=ReviewsOutput, + executor=build_reviews_executor(), + billing_unit=BillingUnit.WALMART_REVIEW, + docs_url="/docs/connectors/native/walmart", +) + +register_capability(WALMART_REVIEWS) diff --git a/surfsense_backend/app/capabilities/walmart/reviews/executor.py b/surfsense_backend/app/capabilities/walmart/reviews/executor.py new file mode 100644 index 000000000..ba1ee0367 --- /dev/null +++ b/surfsense_backend/app/capabilities/walmart/reviews/executor.py @@ -0,0 +1,40 @@ +"""``walmart.reviews`` executor: verb input → scraper → review items.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable + +from app.capabilities.core import Executor +from app.capabilities.core.progress import emit_progress +from app.capabilities.walmart.reviews.schemas import ReviewsInput, ReviewsOutput +from app.proprietary.platforms.walmart import WalmartReviewsInput, scrape_reviews + +ReviewsFn = Callable[..., Awaitable[list[dict]]] + + +def build_reviews_executor(scrape_fn: ReviewsFn | None = None) -> Executor: + """Bind the capability input mapping to a replaceable reviews scraper function.""" + scrape_fn = scrape_fn or scrape_reviews + + async def execute(payload: ReviewsInput) -> ReviewsOutput: + input_model = WalmartReviewsInput( + itemIds=payload.sources(), + maxReviews=payload.max_reviews, + sort=payload.sort_by, + ) + emit_progress( + "starting", + "Fetching Walmart reviews", + total=payload.estimated_units, + unit="review", + ) + items = await scrape_fn(input_model, limit=payload.estimated_units) + emit_progress( + "done", + f"Scraped {sum('error' not in item for item in items)} review(s)", + current=len(items), + unit="review", + ) + return ReviewsOutput(items=items) + + return execute diff --git a/surfsense_backend/app/capabilities/walmart/reviews/schemas.py b/surfsense_backend/app/capabilities/walmart/reviews/schemas.py new file mode 100644 index 000000000..faf56ed5b --- /dev/null +++ b/surfsense_backend/app/capabilities/walmart/reviews/schemas.py @@ -0,0 +1,72 @@ +"""``walmart.reviews`` I/O contracts. + +A lean surface over ``WalmartReviewsInput``; the scraper's ``ReviewItem`` is +reused verbatim as the output element. Accepts product URLs or bare item ids — +both resolve to a ``usItemId`` the reviews page is keyed on. +""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, Field, model_validator + +from app.proprietary.platforms.walmart import ReviewItem + +MAX_WALMART_REVIEW_SOURCES = 20 + + +class ReviewsInput(BaseModel): + urls: list[str] = Field( + default_factory=list, + max_length=MAX_WALMART_REVIEW_SOURCES, + description=( + "Walmart product URLs (/ip/...) or reviews URLs to fetch reviews for. " + "Provide these OR item_ids (at least one is required)." + ), + ) + item_ids: list[str] = Field( + default_factory=list, + max_length=MAX_WALMART_REVIEW_SOURCES, + description="Walmart numeric item ids (usItemId) to fetch reviews for.", + ) + max_reviews: int = Field( + default=200, + ge=1, + le=5000, + description="Max reviews to return per product (10 per page).", + ) + sort_by: Literal["most-recent", "most-helpful", "rating-high", "rating-low"] = ( + Field(default="most-recent", description="Review ordering.") + ) + + @model_validator(mode="after") + def _require_a_source(self) -> ReviewsInput: + if not (self.urls or self.item_ids): + raise ValueError("Provide at least one of 'urls' or 'item_ids'.") + return self + + def sources(self) -> list[str]: + """URLs and item ids merged; the scraper resolves each to a usItemId.""" + return [*self.urls, *self.item_ids] + + @property + def estimated_units(self) -> int: + """Worst-case billable reviews: up to ``max_reviews`` per source.""" + return (len(self.urls) + len(self.item_ids)) * self.max_reviews + + +class ReviewsOutput(BaseModel): + items: list[ReviewItem] = Field( + default_factory=list, + description="One item per review, in the scraper's emission order.", + ) + + @property + def billable_units(self) -> int: + """One returned review = one billable unit; error items are not billed.""" + return sum( + 1 + for item in self.items + if not (item.model_extra and item.model_extra.get("error")) + ) diff --git a/surfsense_backend/app/capabilities/walmart/scrape/__init__.py b/surfsense_backend/app/capabilities/walmart/scrape/__init__.py new file mode 100644 index 000000000..bb3e8f794 --- /dev/null +++ b/surfsense_backend/app/capabilities/walmart/scrape/__init__.py @@ -0,0 +1,3 @@ +"""Walmart product scraping capability.""" + +from __future__ import annotations diff --git a/surfsense_backend/app/capabilities/walmart/scrape/definition.py b/surfsense_backend/app/capabilities/walmart/scrape/definition.py new file mode 100644 index 000000000..6165a8992 --- /dev/null +++ b/surfsense_backend/app/capabilities/walmart/scrape/definition.py @@ -0,0 +1,22 @@ +"""Registration for the ``walmart.scrape`` capability.""" + +from __future__ import annotations + +from app.capabilities.core import BillingUnit, Capability, register_capability +from app.capabilities.walmart.scrape.executor import build_scrape_executor +from app.capabilities.walmart.scrape.schemas import ScrapeInput, ScrapeOutput + +WALMART_SCRAPE = Capability( + name="walmart.scrape", + description=( + "Scrape public Walmart product details, search/category listings, " + "prices, sellers, variants, availability, and a sample of on-page reviews." + ), + input_schema=ScrapeInput, + output_schema=ScrapeOutput, + executor=build_scrape_executor(), + billing_unit=BillingUnit.WALMART_PRODUCT, + docs_url="/docs/connectors/native/walmart", +) + +register_capability(WALMART_SCRAPE) diff --git a/surfsense_backend/app/capabilities/walmart/scrape/executor.py b/surfsense_backend/app/capabilities/walmart/scrape/executor.py new file mode 100644 index 000000000..4c4b073ee --- /dev/null +++ b/surfsense_backend/app/capabilities/walmart/scrape/executor.py @@ -0,0 +1,46 @@ +"""Executor for the ``walmart.scrape`` capability.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable + +from app.capabilities.core import Executor +from app.capabilities.core.progress import emit_progress +from app.capabilities.walmart.scrape.schemas import ( + MAX_WALMART_RESULTS, + ScrapeInput, + ScrapeOutput, +) +from app.proprietary.platforms.walmart import WalmartScrapeInput, scrape_products + +ScrapeFn = Callable[..., Awaitable[list[dict]]] + + +def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor: + """Bind the capability input mapping to a replaceable scraper function.""" + scrape_fn = scrape_fn or scrape_products + + async def execute(payload: ScrapeInput) -> ScrapeOutput: + input_model = WalmartScrapeInput( + startUrls=payload.start_urls(), + maxItemsPerStartUrl=payload.max_items, + includeDetails=payload.include_details, + includeReviewsSample=payload.include_reviews_sample, + ) + emit_progress( + "starting", + "Scraping Walmart products", + total=payload.estimated_units, + unit="product", + ) + items = await scrape_fn(input_model, limit=MAX_WALMART_RESULTS) + emit_progress( + "done", + f"Scraped {sum('error' not in item for item in items)} product(s)", + current=len(items), + total=payload.estimated_units, + unit="product", + ) + return ScrapeOutput(items=items) + + return execute diff --git a/surfsense_backend/app/capabilities/walmart/scrape/schemas.py b/surfsense_backend/app/capabilities/walmart/scrape/schemas.py new file mode 100644 index 000000000..2d42dc371 --- /dev/null +++ b/surfsense_backend/app/capabilities/walmart/scrape/schemas.py @@ -0,0 +1,64 @@ +"""Input and output contracts for ``walmart.scrape``.""" + +from __future__ import annotations + +from urllib.parse import quote_plus + +from pydantic import BaseModel, Field, model_validator + +from app.proprietary.platforms.walmart import ProductItem + +MAX_WALMART_SOURCES = 20 +MAX_WALMART_RESULTS = 1000 + + +class ScrapeInput(BaseModel): + """Agent-facing controls for public Walmart product discovery and enrichment.""" + + urls: list[str] = Field(default_factory=list, max_length=MAX_WALMART_SOURCES) + search_terms: list[str] = Field( + default_factory=list, max_length=MAX_WALMART_SOURCES + ) + max_items: int = Field(default=10, ge=1, le=100) + include_details: bool = True + include_reviews_sample: bool = True + + @model_validator(mode="after") + def _require_source(self) -> ScrapeInput: + if not (self.urls or self.search_terms): + raise ValueError("Provide at least one URL or search term.") + if len(self.urls) + len(self.search_terms) > MAX_WALMART_SOURCES: + raise ValueError( + f"Provide no more than {MAX_WALMART_SOURCES} combined sources." + ) + return self + + def start_urls(self) -> list[str]: + """Direct URLs plus a search URL synthesized per search term.""" + searches = [ + f"https://www.walmart.com/search?q={quote_plus(term)}" + for term in self.search_terms + ] + return [*self.urls, *searches] + + @property + def estimated_units(self) -> int: + """Worst-case returned products within the hard per-run ceiling.""" + search_products = len(self.search_terms) * self.max_items + direct_products = len(self.urls) * self.max_items + return min(search_products + direct_products, MAX_WALMART_RESULTS) + + +class ScrapeOutput(BaseModel): + """Products and structured per-input errors in emission order.""" + + items: list[ProductItem] = Field(default_factory=list) + + @property + def billable_units(self) -> int: + """Count successful products; error items are never billed.""" + return sum( + 1 + for item in self.items + if not (item.model_extra and item.model_extra.get("error")) + ) diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index f9edc3fdf..8e0ac12bc 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -741,6 +741,12 @@ class Config: INDEED_SCRAPE_MICROS_PER_JOB = int( os.getenv("INDEED_SCRAPE_MICROS_PER_JOB", "3500") ) + # Walmart products come from server-rendered JSON behind residential proxies, + # priced alongside Amazon's per-product meter. + WALMART_MICROS_PER_PRODUCT = int(os.getenv("WALMART_MICROS_PER_PRODUCT", "3500")) + # Reviews are 10 per page (many light requests per product), priced on the + # cheaper per-review market like the Google Maps review meter. + WALMART_MICROS_PER_REVIEW = int(os.getenv("WALMART_MICROS_PER_REVIEW", "1500")) # Retry an empty listing draw on a fresh rotating IP. Set to 1 for a static # proxy, where every retry re-hits the same exit. TIKTOK_LISTING_MAX_ATTEMPTS = int(os.getenv("TIKTOK_LISTING_MAX_ATTEMPTS", "3")) diff --git a/surfsense_backend/app/proprietary/platforms/walmart/README.md b/surfsense_backend/app/proprietary/platforms/walmart/README.md new file mode 100644 index 000000000..982ee5400 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/walmart/README.md @@ -0,0 +1,52 @@ +# Walmart Scraper + +Two verbs read Walmart's public, anonymous pages: `walmart.scrape` (products + +search/category/browse listings, per-product billing) and `walmart.reviews` +(deep paginated reviews, per-review billing). + +## Scope + +The scraper reads public pages available to anonymous visitors — no login, no +account cookies. Data is extracted from the Next.js `__NEXT_DATA__` JSON blob +embedded in each page (with an `__APP_DATA__` fallback), not from the rendered +DOM, because Walmart obfuscates CSS classes and A/B-tests layout constantly. + +`walmart.scrape` returns a free sample of on-page reviews (rating distribution, +aspects, top reviews) under `reviewsSample`. `walmart.reviews` fetches the full +review history from the public `/reviews/product/{usItemId}` page, which +robots.txt permits (unlike `/search`). + +## Architecture + +- `schemas.py` defines the stable input, product, review, and error models. +- `url_resolver.py` classifies product (`/ip/`) vs listing (`/search`, `/cp/`, + `/browse/`) URLs and extracts the numeric `usItemId`. +- `next_data.py` extracts and navigates the hidden Next.js JSON state. +- `fetch.py` owns proxy-aware HTTP access (US-pinned), block detection, and + retries. +- `parsers.py` contains pure, defensive JSON parsers. +- `scraper.py` coordinates discovery, enrichment, pagination, concurrency, + limits, and in-stream error items. + +## Anti-bot + +Walmart runs Akamai (edge/TLS) + PerimeterX/HUMAN (behavioral JS). Requests go +through US residential proxies with TLS-impersonated headers; blocked responses +(body markers, `412`/`429`/`503`, or the `200`-OK CAPTCHA body) rotate to a +fresh proxy exit. + +Known ceilings and upgrade paths (see `fetch.py` / `scraper.py` `ponytail:` +notes): reviews page at 10/page; search capped at Walmart's 25-page limit; +session warming (seed `_px3`/`_pxhd` on a sticky exit) is the next lever if +block rates on the SSR pages climb, and the `/orchestra/*` GraphQL API is +deliberately avoided (rotating persisted-query hashes make it brittle). + +## Verification + +Offline fixtures cover the parsers and both flows: + + uv run pytest tests/unit/platforms/walmart/ + +A manual live check (requires network + residential proxy): + + uv run python scripts/e2e_walmart_scraper.py diff --git a/surfsense_backend/app/proprietary/platforms/walmart/__init__.py b/surfsense_backend/app/proprietary/platforms/walmart/__init__.py new file mode 100644 index 000000000..83b7cf277 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/walmart/__init__.py @@ -0,0 +1,27 @@ +"""Platform-native Walmart scraper (products, listings, reviews).""" + +from .schemas import ( + ErrorItem, + ProductItem, + ReviewItem, + WalmartReviewsInput, + WalmartScrapeInput, +) +from .scraper import ( + iter_products, + iter_reviews, + scrape_products, + scrape_reviews, +) + +__all__ = [ + "ErrorItem", + "ProductItem", + "ReviewItem", + "WalmartReviewsInput", + "WalmartScrapeInput", + "iter_products", + "iter_reviews", + "scrape_products", + "scrape_reviews", +] diff --git a/surfsense_backend/app/proprietary/platforms/walmart/fetch.py b/surfsense_backend/app/proprietary/platforms/walmart/fetch.py new file mode 100644 index 000000000..851b6a674 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/walmart/fetch.py @@ -0,0 +1,171 @@ +"""Network access for public Walmart pages. + +Mirrors the Amazon fetch layer: every request goes through the configured +residential proxy (pinned to the US, since Walmart geo-locks inventory), and any +response that looks like an anti-bot interstitial is retried on a fresh proxy +exit. Ordinary HTTP failures are returned to the caller for domain-specific +handling. + +Walmart runs Akamai (edge/TLS) + PerimeterX/HUMAN (behavioral JS). Two Walmart +specifics differ from Amazon: + +* Walmart serves CAPTCHA with an HTTP ``200`` body ("Robot or human?"), so block + detection scans the body, never the status alone. +* ``412`` is PerimeterX's rejection code and is treated as blocked → rotate. + +``ponytail:`` MVP hits only the server-rendered ``__NEXT_DATA__`` pages, which +TLS impersonation + residential proxies clear without seeding PerimeterX cookies. +If block rates on those pages climb, the upgrade path is a warmed sticky session +(seed ``_px3``/``_pxhd``/``ACID`` from a homepage fetch, reuse exit + cookies) — +the same shape as Amazon's ``get_location_session``. +""" + +from __future__ import annotations + +import asyncio +import logging +import time +from collections.abc import Awaitable, Callable +from dataclasses import dataclass, field +from typing import Any + +from scrapling.fetchers import AsyncFetcher + +from app.utils.proxy import get_geo_proxy_url, get_sticky_proxy_url + +logger = logging.getLogger(__name__) + +_MAX_IP_ATTEMPTS = 6 +_REQUEST_TIMEOUT_S = 30 +_HEADERS = { + "Accept-Language": "en-US,en;q=0.9", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", +} +_BLOCK_MARKERS = ( + "robot or human", + "px-captcha", + "/blocked", + "verify you are a human", + "access to this page has been denied", +) + + +@dataclass(frozen=True) +class FetchResult: + """The response details needed by scraper flows.""" + + status: int + html: str + url: str + cookies: dict[str, str] + headers: dict[str, str] = field(default_factory=dict) + + +async def gather_bounded[T]( + factories: list[Callable[[], Awaitable[T]]], *, concurrency: int +) -> list[T]: + """Run async factories concurrently while preserving input order.""" + if not factories: + return [] + semaphore = asyncio.Semaphore(max(1, concurrency)) + + async def run(factory: Callable[[], Awaitable[T]]) -> T: + async with semaphore: + return await factory() + + return await asyncio.gather(*(run(factory) for factory in factories)) + + +def is_blocked( + html: str | None, status: int, headers: dict[str, str] | None = None +) -> bool: + """Return whether a response is a Walmart anti-bot interstitial. + + ``412`` is PerimeterX's rejection; ``429``/``503`` are throttles. Walmart also + serves CAPTCHA with a ``200`` body, so the body is scanned regardless of + status. + """ + if status in {412, 429, 503}: + return True + text = (html or "")[:200_000].lower() + return any(marker in text for marker in _BLOCK_MARKERS) + + +def _response_url(page: Any, fallback: str) -> str: + value = getattr(page, "url", None) + return str(value) if value else fallback + + +def _response_cookies(page: Any) -> dict[str, str]: + cookies = getattr(page, "cookies", None) + return dict(cookies) if isinstance(cookies, dict) else {} + + +def _response_headers(page: Any) -> dict[str, str]: + headers = getattr(page, "headers", None) + return dict(headers) if isinstance(headers, dict) else {} + + +def _selected_proxy( + proxy: str | None, country: str, attempt: int, url: str +) -> str | None: + if proxy is not None: + return proxy + if attempt > 1: + session_id = f"walmart-{attempt}-{abs(hash((url, time.time_ns()))):x}" + return get_sticky_proxy_url(session_id, country) + return get_geo_proxy_url(country) + + +async def fetch_page( + url: str, + *, + cookies: dict[str, str] | None = None, + proxy: str | None = None, + country: str = "us", + rotate_on_block: bool = True, +) -> FetchResult | None: + """Fetch a page and retry blocked responses with fresh proxy exits.""" + attempts = _MAX_IP_ATTEMPTS if rotate_on_block else 1 + for attempt in range(1, attempts + 1): + selected_proxy = _selected_proxy(proxy, country, attempt, url) + started = time.perf_counter() + try: + page = await AsyncFetcher.get( + url, + headers={**_HEADERS}, + cookies=cookies or {}, + proxy=selected_proxy, + stealthy_headers=True, + timeout=_REQUEST_TIMEOUT_S, + ) + except Exception as exc: + logger.warning("Walmart request failed for %s: %s", url, exc) + if proxy is not None: + return None + continue + + status = int(getattr(page, "status", 0) or 0) + html = getattr(page, "html_content", None) or "" + response_headers = _response_headers(page) + logger.info( + "[walmart][perf] status=%s attempt=%s fetch_ms=%.1f url=%s", + status, + attempt, + (time.perf_counter() - started) * 1000, + url, + ) + if rotate_on_block and is_blocked(html, status, response_headers): + logger.info( + "Walmart blocked proxy attempt %s/%s for %s", attempt, attempts, url + ) + continue + return FetchResult( + status=status, + html=html, + url=_response_url(page, url), + cookies=_response_cookies(page), + headers=response_headers, + ) + logger.warning("Walmart exhausted %s proxy attempts for %s", attempts, url) + return None diff --git a/surfsense_backend/app/proprietary/platforms/walmart/next_data.py b/surfsense_backend/app/proprietary/platforms/walmart/next_data.py new file mode 100644 index 000000000..2994fcbf6 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/walmart/next_data.py @@ -0,0 +1,69 @@ +"""Extract Walmart's hidden Next.js JSON state from page HTML. + +Every serious Walmart scraper (Scrapfly, Oxylabs, ScrapeOps, Apify) reads the +``', re.DOTALL) +_APP_DATA_RE = re.compile(r'', re.DOTALL) + + +def extract_next_data(html: str | None) -> dict[str, Any] | None: + """Return the parsed Next.js state object, or ``None`` when absent/invalid. + + Tries ``__NEXT_DATA__`` first, then the ``__APP_DATA__`` fallback so a single + Walmart layout experiment does not blank the whole extractor. + """ + if not html: + return None + for pattern in (_NEXT_DATA_RE, _APP_DATA_RE): + match = pattern.search(html) + if not match: + continue + try: + data = json.loads(match.group(1)) + except (ValueError, TypeError): + logger.warning("Walmart hidden JSON present but did not parse") + continue + if isinstance(data, dict): + return data + return None + + +def dig(obj: Any, *keys: str | int) -> Any: + """Walk nested dict/list keys, returning ``None`` on any miss. + + Tolerates the layout drift between Walmart's ``initialData`` variants without + a cascade of ``if key in ...`` guards at every call site. + """ + current = obj + for key in keys: + if isinstance(key, int): + if not isinstance(current, list) or not -len(current) <= key < len(current): + return None + current = current[key] + else: + if not isinstance(current, dict) or key not in current: + return None + current = current[key] + return current + + +def initial_data(next_data: dict[str, Any]) -> dict[str, Any] | None: + """The ``props.pageProps.initialData`` node shared by every page type.""" + node = dig(next_data, "props", "pageProps", "initialData") + return node if isinstance(node, dict) else None diff --git a/surfsense_backend/app/proprietary/platforms/walmart/parsers.py b/surfsense_backend/app/proprietary/platforms/walmart/parsers.py new file mode 100644 index 000000000..5ea32c4d1 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/walmart/parsers.py @@ -0,0 +1,246 @@ +"""Pure parsers for Walmart's hidden ``__NEXT_DATA__`` JSON. + +Product, listing, and review data all live in the same Next.js state tree; these +functions navigate it defensively (via :func:`.next_data.dig`) and normalize the +fields into the stable output shape. Missing sections yield ``None``/``[]`` so an +isolated schema change never discards an otherwise usable record. +""" + +from __future__ import annotations + +from typing import Any +from urllib.parse import urljoin + +from .next_data import dig, initial_data + +_WALMART_ORIGIN = "https://www.walmart.com" + + +# --------------------------------------------------------------------------- # +# Shared helpers. # +# --------------------------------------------------------------------------- # + + +def _price(node: Any) -> dict[str, Any] | None: + """Normalize a Walmart price node into ``{value, currency}``.""" + if not isinstance(node, dict): + return None + value = node.get("price") + currency = node.get("currencyUnit") + if value is None and currency is None: + return None + return {"value": value, "currency": currency} + + +def _seller(raw: dict[str, Any]) -> dict[str, Any] | None: + name = raw.get("sellerName") or raw.get("sellerDisplayName") + seller_id = raw.get("sellerId") + if not name and not seller_id: + return None + is_walmart = bool(name) and "walmart" in name.lower() + return { + "id": seller_id, + "name": name, + "type": "WALMART" if is_walmart else "MARKETPLACE", + } + + +def _absolute(url: str | None) -> str | None: + return urljoin(_WALMART_ORIGIN, url) if url else None + + +# --------------------------------------------------------------------------- # +# Listing / search cards. # +# --------------------------------------------------------------------------- # + + +def _listing_card(raw: dict[str, Any]) -> dict[str, Any] | None: + """Normalize one search/category result item into a product card.""" + item_id = raw.get("usItemId") or raw.get("id") + name = raw.get("name") + if not item_id or not name: + return None + price_info = raw.get("priceInfo") or {} + availability = raw.get("availabilityStatusV2") or {} + image = raw.get("imageInfo") or {} + return { + "usItemId": str(item_id), + "name": name, + "brand": raw.get("brand"), + "url": _absolute(raw.get("canonicalUrl")), + "price": _price(price_info.get("currentPrice")) + or ({"value": raw.get("price")} if raw.get("price") is not None else None), + "listPrice": _price(price_info.get("wasPrice")), + "stars": raw.get("averageRating"), + "reviewsCount": raw.get("numberOfReviews"), + "seller": _seller(raw), + "availabilityStatus": availability.get("value") + or ("OUT_OF_STOCK" if raw.get("isOutOfStock") else None), + "inStock": (availability.get("value") == "IN_STOCK") + if availability.get("value") + else (raw.get("isOutOfStock") is False if "isOutOfStock" in raw else None), + "thumbnailImage": image.get("thumbnailUrl") or raw.get("image"), + "sponsored": raw.get("isSponsoredFlag"), + } + + +def parse_listing_page(next_data: dict[str, Any]) -> list[dict[str, Any]]: + """Extract normalized product cards from a search/category/browse page.""" + data = initial_data(next_data) + if data is None: + return [] + stacks = dig(data, "searchResult", "itemStacks") + if not isinstance(stacks, list): + return [] + cards: list[dict[str, Any]] = [] + seen: set[str] = set() + for stack in stacks: + for raw in (stack or {}).get("items") or []: + if not isinstance(raw, dict) or raw.get("__typename") not in ( + None, + "Product", + ): + continue + card = _listing_card(raw) + if card and card["usItemId"] not in seen: + seen.add(card["usItemId"]) + cards.append(card) + return cards + + +# --------------------------------------------------------------------------- # +# Product detail. # +# --------------------------------------------------------------------------- # + + +def _images(image_info: dict[str, Any]) -> list[str]: + images: list[str] = [] + for entry in image_info.get("allImages") or []: + url = entry.get("url") if isinstance(entry, dict) else None + if url and url not in images: + images.append(url) + return images + + +def _breadcrumbs(product: dict[str, Any]) -> list[str]: + """Category breadcrumb names, root→leaf. + + Walmart ships ``category.path`` as a list of ``{name, url}`` nodes, e.g. + ``[{"name": "Home Improvement"}, ..., {"name": "Air Conditioners"}]`` — not + a string. + """ + path = dig(product, "category", "path") + if not isinstance(path, list): + return [] + return [node["name"] for node in path if isinstance(node, dict) and node.get("name")] + + +def _reviews_sample( + reviews: dict[str, Any] | None, limit: int = 10 +) -> dict[str, Any] | None: + if not isinstance(reviews, dict): + return None + customer = reviews.get("customerReviews") or [] + return { + "averageOverallRating": reviews.get("averageOverallRating"), + "totalReviewCount": reviews.get("totalReviewCount"), + "aspects": reviews.get("aspects") or [], + "topReviews": [ + normalize_review(r) for r in customer[:limit] if isinstance(r, dict) + ], + } + + +def parse_product( + next_data: dict[str, Any], *, url: str, include_reviews_sample: bool = True +) -> dict[str, Any]: + """Extract normalized product fields from a product detail page.""" + data = initial_data(next_data) + product = dig(data, "data", "product") if data else None + if not isinstance(product, dict): + return {} + idml = dig(data, "data", "idml") if data else None + reviews = dig(data, "data", "reviews") if data else None + price_info = product.get("priceInfo") or {} + image_info = product.get("imageInfo") or {} + availability = product.get("availabilityStatus") + breadcrumbs = _breadcrumbs(product) + + fields: dict[str, Any] = { + "usItemId": str(product.get("usItemId") or product.get("id") or "") or None, + "name": product.get("name"), + "brand": product.get("brand"), + "url": url, + "price": _price(price_info.get("currentPrice")), + "listPrice": _price(price_info.get("wasPrice")), + "currency": dig(price_info, "currentPrice", "currencyUnit"), + "availabilityStatus": availability, + "inStock": availability == "IN_STOCK" if availability else None, + "stars": product.get("averageRating"), + "reviewsCount": product.get("numberOfReviews"), + "seller": _seller(product), + "manufacturerName": product.get("manufacturerName"), + "shortDescription": product.get("shortDescription"), + "longDescription": (idml or {}).get("longDescription") + if isinstance(idml, dict) + else None, + "thumbnailImage": image_info.get("thumbnailUrl"), + "images": _images(image_info), + "category": breadcrumbs[-1] if breadcrumbs else None, + "breadCrumbs": breadcrumbs, + "variants": product.get("variantCriteria") or [], + } + if include_reviews_sample: + fields["reviewsSample"] = _reviews_sample(reviews) + return {key: value for key, value in fields.items() if value is not None} + + +# --------------------------------------------------------------------------- # +# Reviews (deep pagination). # +# --------------------------------------------------------------------------- # + + +def normalize_review(raw: dict[str, Any]) -> dict[str, Any]: + """Normalize one Walmart ``customerReviews`` record into a ``ReviewItem`` dict.""" + badges = raw.get("badges") or [] + verified = any( + isinstance(b, dict) and b.get("id") == "VerifiedPurchaser" for b in badges + ) + photos = raw.get("photos") or raw.get("media") or [] + images: list[str] = [] + for photo in photos: + if not isinstance(photo, dict): + continue + url = photo.get("normalUrl") or dig(photo, "sizes", "normal", "url") + if url and url not in images: + images.append(url) + responses = raw.get("clientResponses") or [] + seller_response = None + if responses and isinstance(responses[0], dict): + seller_response = responses[0].get("response") or responses[0].get( + "responseText" + ) + return { + "reviewId": raw.get("reviewId"), + "rating": raw.get("rating"), + "title": raw.get("reviewTitle"), + "text": raw.get("reviewText"), + "submissionTime": raw.get("reviewSubmissionTime"), + "author": raw.get("userNickname"), + "verifiedPurchase": verified, + "positiveFeedback": raw.get("positiveFeedback"), + "negativeFeedback": raw.get("negativeFeedback"), + "images": images, + "syndicated": bool(raw.get("syndicationSource")), + "sellerResponse": seller_response, + } + + +def parse_reviews_page(next_data: dict[str, Any]) -> list[dict[str, Any]]: + """Extract normalized reviews from one ``/reviews/product/{id}`` page.""" + data = initial_data(next_data) + reviews = dig(data, "data", "reviews") if data else None + customer = reviews.get("customerReviews") if isinstance(reviews, dict) else None + if not isinstance(customer, list): + return [] + return [normalize_review(r) for r in customer if isinstance(r, dict)] diff --git a/surfsense_backend/app/proprietary/platforms/walmart/schemas.py b/surfsense_backend/app/proprietary/platforms/walmart/schemas.py new file mode 100644 index 000000000..ea25f2eff --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/walmart/schemas.py @@ -0,0 +1,188 @@ +# ruff: noqa: N815 +"""Input/output models for the public Walmart scraper. + +Two verbs share this module: ``walmart.scrape`` (products + listings) and +``walmart.reviews`` (deep paginated reviews). Walmart is a Next.js app that +ships its data as JSON in a `` + diff --git a/surfsense_backend/tests/unit/platforms/walmart/fixtures/product.html b/surfsense_backend/tests/unit/platforms/walmart/fixtures/product.html new file mode 100644 index 000000000..183ddff6b --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/walmart/fixtures/product.html @@ -0,0 +1,3 @@ +Midea AC + + diff --git a/surfsense_backend/tests/unit/platforms/walmart/fixtures/reviews.html b/surfsense_backend/tests/unit/platforms/walmart/fixtures/reviews.html new file mode 100644 index 000000000..b0d27e5b6 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/walmart/fixtures/reviews.html @@ -0,0 +1,3 @@ +Reviews + + diff --git a/surfsense_backend/tests/unit/platforms/walmart/test_flows.py b/surfsense_backend/tests/unit/platforms/walmart/test_flows.py new file mode 100644 index 000000000..045aa99d5 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/walmart/test_flows.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +from pathlib import Path + +from app.proprietary.platforms.walmart import ( + WalmartReviewsInput, + WalmartScrapeInput, + scrape_products, + scrape_reviews, + scraper, +) +from app.proprietary.platforms.walmart.fetch import FetchResult + +_FIXTURES = Path(__file__).parent / "fixtures" + + +def _fixture(name: str) -> str: + return (_FIXTURES / name).read_text(encoding="utf-8") + + +def _response(url: str, html: str, status: int = 200) -> FetchResult: + return FetchResult(status=status, html=html, url=url, cookies={}) + + +async def test_product_flow_emits_parsed_item(monkeypatch): + async def fetch_page(url: str, **_kwargs): + return _response(url, _fixture("product.html")) + + monkeypatch.setattr(scraper, "fetch_page", fetch_page) + items = await scrape_products( + WalmartScrapeInput(startUrls=["https://www.walmart.com/ip/212092810"]) + ) + + assert len(items) == 1 + assert items[0]["usItemId"] == "212092810" + assert items[0]["name"].startswith("Midea") + + +async def test_product_flow_maps_not_found_to_error_item(monkeypatch): + async def fetch_page(url: str, **_kwargs): + return _response(url, "", status=404) + + monkeypatch.setattr(scraper, "fetch_page", fetch_page) + items = await scrape_products( + WalmartScrapeInput(startUrls=["https://www.walmart.com/ip/212092810"]) + ) + + assert items[0]["error"] == "product_not_found" + + +async def test_listing_flow_card_only_honors_cap(monkeypatch): + calls: list[str] = [] + + async def fetch_page(url: str, **_kwargs): + calls.append(url) + html = _fixture("listing.html") if "page=1" in url else "" + return _response(url, html) + + monkeypatch.setattr(scraper, "fetch_page", fetch_page) + items = await scrape_products( + WalmartScrapeInput( + startUrls=["https://www.walmart.com/search?q=laptop"], + maxItemsPerStartUrl=1, + includeDetails=False, + ) + ) + + assert len(items) == 1 + assert items[0]["usItemId"] == "791595618" + assert len(calls) == 1 + + +async def test_listing_flow_enriches_with_detail_pages(monkeypatch): + async def fetch_page(url: str, **_kwargs): + if "/ip/" in url: + return _response(url, _fixture("product.html")) + html = _fixture("listing.html") if "page=1" in url else "" + return _response(url, html) + + monkeypatch.setattr(scraper, "fetch_page", fetch_page) + items = await scrape_products( + WalmartScrapeInput( + startUrls=["https://www.walmart.com/search?q=laptop"], + maxItemsPerStartUrl=2, + includeDetails=True, + ) + ) + + # Both cards enrich to the same product fixture (detail fetch wins). + assert all(item["longDescription"] for item in items) + + +async def test_invalid_url_yields_error_item(monkeypatch): + items = await scrape_products( + WalmartScrapeInput(startUrls=["https://example.com/not-walmart"]) + ) + assert items[0]["error"] == "invalid_url" + + +async def test_reviews_flow_paginates_until_empty(monkeypatch): + calls: list[str] = [] + + async def fetch_page(url: str, **_kwargs): + calls.append(url) + html = _fixture("reviews.html") if "page=1" in url else "" + return _response(url, html) + + monkeypatch.setattr(scraper, "fetch_page", fetch_page) + items = await scrape_reviews( + WalmartReviewsInput(itemIds=["212092810"], maxReviews=100) + ) + + assert len(items) == 2 + assert items[0]["reviewId"] == "296013686" + # page=1 returned records, page=2 empty → stop. Two fetches total. + assert len(calls) == 2 + + +async def test_reviews_flow_honors_max_reviews(monkeypatch): + async def fetch_page(url: str, **_kwargs): + return _response(url, _fixture("reviews.html")) + + monkeypatch.setattr(scraper, "fetch_page", fetch_page) + items = await scrape_reviews( + WalmartReviewsInput(itemIds=["212092810"], maxReviews=1) + ) + + assert len(items) == 1 + + +async def test_reviews_flow_maps_empty_to_error_item(monkeypatch): + async def fetch_page(url: str, **_kwargs): + return _response(url, "") + + monkeypatch.setattr(scraper, "fetch_page", fetch_page) + items = await scrape_reviews(WalmartReviewsInput(itemIds=["212092810"])) + + assert items[0]["error"] == "reviews_not_found" diff --git a/surfsense_backend/tests/unit/platforms/walmart/test_parsers.py b/surfsense_backend/tests/unit/platforms/walmart/test_parsers.py new file mode 100644 index 000000000..596131a2e --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/walmart/test_parsers.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +from pathlib import Path + +from app.proprietary.platforms.walmart.fetch import is_blocked +from app.proprietary.platforms.walmart.next_data import extract_next_data +from app.proprietary.platforms.walmart.parsers import ( + parse_listing_page, + parse_product, + parse_reviews_page, +) +from app.proprietary.platforms.walmart.url_resolver import extract_item_id, resolve_url + +_FIXTURES = Path(__file__).parent / "fixtures" + + +def _fixture(name: str) -> str: + return (_FIXTURES / name).read_text(encoding="utf-8") + + +def test_product_parser_extracts_core_fields_and_review_sample(): + data = extract_next_data(_fixture("product.html")) + item = parse_product(data, url="https://www.walmart.com/ip/212092810") + + assert item["usItemId"] == "212092810" + assert item["name"].startswith("Midea") + assert item["price"] == {"value": 149.0, "currency": "USD"} + assert item["listPrice"] == {"value": 199.0, "currency": "USD"} + assert item["stars"] == 4.5 + assert item["reviewsCount"] == 6287 + assert item["inStock"] is True + assert item["seller"] == {"id": "F55CT", "name": "Walmart.com", "type": "WALMART"} + assert item["longDescription"] == "

A powerful window unit.

" + assert item["images"] == [ + "https://i5.walmartimages.com/1.jpg", + "https://i5.walmartimages.com/2.jpg", + ] + # Walmart ships category.path as breadcrumb objects, not a string. + assert item["breadCrumbs"] == ["Home", "Air Conditioners"] + assert item["category"] == "Air Conditioners" + assert item["reviewsSample"]["totalReviewCount"] == 6287 + assert item["reviewsSample"]["topReviews"][0]["verifiedPurchase"] is True + + +def test_listing_parser_normalizes_cards_and_marketplace_seller(): + data = extract_next_data(_fixture("listing.html")) + cards = parse_listing_page(data) + + assert [c["usItemId"] for c in cards] == ["791595618", "999001"] + first = cards[0] + assert first["url"] == "https://www.walmart.com/ip/Acer-Chromebook/791595618" + assert first["price"] == {"value": 79.99, "currency": "USD"} + assert first["seller"]["type"] == "MARKETPLACE" + assert first["inStock"] is True + # Fallback price + out-of-stock derivation for the second card. + assert cards[1]["price"] == {"value": 499.0} + assert cards[1]["inStock"] is False + + +def test_reviews_parser_extracts_all_records(): + data = extract_next_data(_fixture("reviews.html")) + reviews = parse_reviews_page(data) + + assert len(reviews) == 2 + assert reviews[0]["reviewId"] == "296013686" + assert reviews[0]["author"] == "JohnPaul" + assert reviews[0]["verifiedPurchase"] is True + assert reviews[0]["images"] == ["https://i5.walmartimages.com/r.jpg"] + assert reviews[0]["sellerResponse"] == "Thanks for the review!" + assert reviews[1]["verifiedPurchase"] is False + + +def test_block_detection_handles_status_and_body(): + assert is_blocked(_fixture("blocked.html"), 200) + assert is_blocked("", 412) + assert is_blocked("", 429) + assert not is_blocked(_fixture("product.html"), 200) + + +def test_missing_next_data_yields_none(): + assert extract_next_data("no script here") is None + assert parse_product(extract_next_data("") or {}, url="x") == {} + + +def test_url_resolver_classifies_and_extracts_ids(): + assert resolve_url("https://www.walmart.com/ip/Foo/123456789").kind == "product" + assert ( + extract_item_id("https://www.walmart.com/reviews/product/212092810") + == "212092810" + ) + assert resolve_url("https://www.walmart.com/search?q=tv").kind == "listing" + assert resolve_url("https://www.walmart.com/cp/tvs/3944").kind == "listing" + assert resolve_url("https://example.com/ip/1") is None diff --git a/surfsense_mcp/mcp_server/features/scrapers/__init__.py b/surfsense_mcp/mcp_server/features/scrapers/__init__.py index 4700c02e4..c74626d41 100644 --- a/surfsense_mcp/mcp_server/features/scrapers/__init__.py +++ b/surfsense_mcp/mcp_server/features/scrapers/__init__.py @@ -1,7 +1,8 @@ """Scraper tools: one MCP surface per SurfSense platform capability. -Web crawl, Google Search, Reddit, YouTube, Google Maps, Indeed, and Amazon each -get a tool that maps a natural-language request to the workspace's scraper. Two run-history tools +Web crawl, Google Search, Reddit, YouTube, Google Maps, Amazon, Indeed, and +Walmart each get a tool that maps a natural-language request to the workspace's +scraper. Two run-history tools list and fetch past runs, so a large result truncated inline can be retrieved in full later. Each platform lives in its own module under platforms/. """ @@ -21,6 +22,7 @@ from .platforms import ( instagram, reddit, tiktok, + walmart, web, youtube, ) @@ -35,6 +37,7 @@ _REGISTRARS = ( google_maps, indeed, amazon, + walmart, run_history, ) diff --git a/surfsense_mcp/mcp_server/features/scrapers/platforms/walmart.py b/surfsense_mcp/mcp_server/features/scrapers/platforms/walmart.py new file mode 100644 index 000000000..0c11af653 --- /dev/null +++ b/surfsense_mcp/mcp_server/features/scrapers/platforms/walmart.py @@ -0,0 +1,140 @@ +"""Walmart scraper tools: products/listings and deep reviews.""" + +from __future__ import annotations + +from typing import Annotated, Literal + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from ....core.client import SurfSenseClient +from ....core.rendering import ResponseFormatParam +from ....core.workspace_context import WorkspaceContext, WorkspaceParam +from ..annotations import SCRAPE +from ..capability import run_scraper + +ReviewSort = Literal["most-recent", "most-helpful", "rating-high", "rating-low"] + + +def register(mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext) -> None: + """Register the Walmart product and review tools.""" + + @mcp.tool( + name="surfsense_walmart_scrape", + title="Scrape Walmart products", + annotations=SCRAPE, + structured_output=False, + ) + async def walmart_scrape( + urls: Annotated[ + list[str] | None, + Field( + description="Walmart product (/ip/), search (/search), category " + "(/cp/), or browse (/browse/) URLs. Provide urls OR search_terms." + ), + ] = None, + search_terms: Annotated[ + list[str] | None, + Field( + description="Search phrases run on walmart.com, e.g. ['air fryer']. " + "Provide search_terms OR urls." + ), + ] = None, + max_items: Annotated[ + int, + Field(ge=1, le=100, description="Max products per search term or listing URL."), + ] = 10, + include_details: Annotated[ + bool, + Field( + description="Fetch full product detail pages. False returns faster " + "card-only results from listings." + ), + ] = True, + include_reviews_sample: Annotated[ + bool, + Field( + description="Include the free on-page review sample (rating " + "distribution, aspects, top reviews) on detail pages." + ), + ] = True, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Scrape public Walmart product data by URL or search term. + + Use this for product research: title, price, list price, rating and + review count, availability, seller (Walmart 1P vs marketplace), images, + description, variants, and a sample of on-page reviews. Only public, + anonymous data — no login. For a product's full review history use + surfsense_walmart_reviews instead. + Example: search_terms=['air fryer'], max_items=5. + """ + return await run_scraper( + client, + context, + platform="walmart", + verb="scrape", + payload={ + "urls": urls, + "search_terms": search_terms, + "max_items": max_items, + "include_details": include_details, + "include_reviews_sample": include_reviews_sample, + }, + workspace=workspace, + response_format=response_format, + ) + + @mcp.tool( + name="surfsense_walmart_reviews", + title="Fetch Walmart reviews", + annotations=SCRAPE, + structured_output=False, + ) + async def walmart_reviews( + urls: Annotated[ + list[str] | None, + Field( + description="Walmart product URLs (/ip/...). Provide urls OR item_ids." + ), + ] = None, + item_ids: Annotated[ + list[str] | None, + Field( + description="Walmart numeric item ids (usItemId) from " + "surfsense_walmart_scrape." + ), + ] = None, + max_reviews: Annotated[ + int, + Field(ge=1, le=5000, description="Max reviews per product (10 per page)."), + ] = 200, + sort_by: Annotated[ + ReviewSort, Field(description="Review ordering.") + ] = "most-recent", + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Fetch deep paginated customer reviews for Walmart products. + + Use this to read the full review history on specific products; get urls + or item_ids from surfsense_walmart_scrape first if you only have a name. + Returns rating, title, text, author, verified-purchase flag, images, and + seller response per review. + Example: item_ids=['212092810'], sort_by='most-helpful', max_reviews=100. + """ + return await run_scraper( + client, + context, + platform="walmart", + verb="reviews", + payload={ + "urls": urls, + "item_ids": item_ids, + "max_reviews": max_reviews, + "sort_by": sort_by, + }, + workspace=workspace, + response_format=response_format, + ) diff --git a/surfsense_web/app/(home)/mcp-server/page.tsx b/surfsense_web/app/(home)/mcp-server/page.tsx index ab7769ee9..439e1ae6f 100644 --- a/surfsense_web/app/(home)/mcp-server/page.tsx +++ b/surfsense_web/app/(home)/mcp-server/page.tsx @@ -16,7 +16,7 @@ import type { FaqItem } from "@/lib/connectors-marketing/types"; const canonicalUrl = "https://www.surfsense.com/mcp-server"; const metaDescription = - "The SurfSense MCP server gives Claude, Cursor, and any MCP client native tools for your workspace: scrape Reddit, YouTube, Instagram, TikTok, Amazon, Google Maps, Google Search, and the web, plus full knowledge base access. One API key."; + "The SurfSense MCP server gives Claude, Cursor, and any MCP client native tools for your workspace: scrape Reddit, YouTube, Instagram, TikTok, Amazon, Walmart, Google Maps, Google Search, and the web, plus full knowledge base access. One API key."; export const metadata: Metadata = { title: "SurfSense MCP Server: Scraper APIs and Knowledge Base as Agent Tools", @@ -103,6 +103,8 @@ const TOOL_GROUPS = [ "surfsense_google_maps_reviews", "surfsense_google_search", "surfsense_amazon_scrape", + "surfsense_walmart_scrape", + "surfsense_walmart_reviews", "surfsense_web_crawl", "surfsense_list_scraper_runs", "surfsense_get_scraper_run", @@ -134,7 +136,7 @@ const FAQ: FaqItem[] = [ { question: "What is the SurfSense MCP server?", answer: - "It is a Model Context Protocol server that exposes your SurfSense workspace to MCP clients like Claude Code, Cursor, and Claude Desktop. Your agents get native tools for every scraper API (Reddit, YouTube, Instagram, TikTok, Amazon, Google Maps, Google Search, web crawl) and for searching, reading, and writing your knowledge base.", + "It is a Model Context Protocol server that exposes your SurfSense workspace to MCP clients like Claude Code, Cursor, and Claude Desktop. Your agents get native tools for every scraper API (Reddit, YouTube, Instagram, TikTok, Amazon, Walmart, Google Maps, Google Search, web crawl) and for searching, reading, and writing your knowledge base.", }, { question: "Which MCP clients does it work with?", @@ -222,9 +224,9 @@ export default function McpServerPage() {

The SurfSense MCP server hands Claude, Cursor, or any MCP client the whole platform: - scrape Reddit, YouTube, Instagram, TikTok, Amazon, Google Maps, Google Search, and - the open web, and search, read, and write your knowledge base. One API key, typed - tools, pay as you go. + scrape Reddit, YouTube, Instagram, TikTok, Amazon, Walmart, Google Maps, Google + Search, and the open web, and search, read, and write your knowledge base. One API + key, typed tools, pay as you go.