Merge pull request #1604 from AnishSarkar22/feat/amazon-scraper

feat(amazon): add public multi-marketplace product scraping capability

docker/.env.example

@@ -441,7 +441,7 @@ SURFSENSE_ENABLE_DOOM_LOOP=true
 # WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE=3000
 
 # Debit the credit wallet per *item returned* by the platform-native scrapers
-# (Reddit, Google Search, Google Maps, YouTube). Default FALSE keeps scraping
+# (Reddit, Google Search, Google Maps, Amazon, YouTube). Default FALSE keeps scraping
 # effectively free for self-hosted installs. Each rate is micro-USD per item,
 # config-driven: <KEY> = round(USD_per_1000_items * 1_000). Defaults sit
 # at/above Apify's first-party actor rates (we charge no subscription tiers,
@@ -452,6 +452,7 @@ SURFSENSE_ENABLE_DOOM_LOOP=true
 # GOOGLE_SEARCH_MICROS_PER_SERP=5500
 # GOOGLE_MAPS_MICROS_PER_PLACE=3500
 # GOOGLE_MAPS_MICROS_PER_REVIEW=1500
+# AMAZON_MICROS_PER_PRODUCT=3500
 # YOUTUBE_MICROS_PER_VIDEO=2500
 # YOUTUBE_MICROS_PER_COMMENT=1500
 # TIKTOK_MICROS_PER_VIDEO=3500

surfsense_backend/.env.example

@@ -277,8 +277,8 @@ MICROS_PER_PAGE=1000
 # WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE=3000
 
 # Debit the credit wallet per *item returned* by the platform-native scrapers
-# (Reddit, Google Search, Google Maps, YouTube). Default FALSE keeps scraping
-# effectively free for self-hosted/OSS installs; hosted deployments set TRUE.
+# (Reddit, Google Search, Google Maps, Amazon, YouTube). Default FALSE keeps
+# scraping effectively free for self-hosted/OSS installs; hosted set TRUE.
 # Each rate is micro-USD per item, fully config-driven (no hardcoded rate):
 #   <KEY> = round(USD_per_1000_items * 1_000)
 #   3500 == $3.50/1000 | 5000 == $5/1000 | 2000 == $2/1000
@@ -290,6 +290,7 @@ MICROS_PER_PAGE=1000
 # GOOGLE_SEARCH_MICROS_PER_SERP=5500
 # GOOGLE_MAPS_MICROS_PER_PLACE=3500
 # GOOGLE_MAPS_MICROS_PER_REVIEW=1500
+# AMAZON_MICROS_PER_PRODUCT=3500
 # YOUTUBE_MICROS_PER_VIDEO=2500
 # YOUTUBE_MICROS_PER_COMMENT=1500
 # INSTAGRAM_SCRAPE_MICROS_PER_ITEM=3500

surfsense_backend/app/agents/chat/multi_agent_chat/constants.py

@@ -29,6 +29,7 @@ CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS: dict[str, str] = {
 # connected app. Tokens are searchable-type strings (Composio Gmail/Calendar
 # map to the GOOGLE_* tokens in connector_searchable_types).
 SUBAGENT_TO_REQUIRED_CONNECTOR_MAP: dict[str, frozenset[str]] = {
+    "amazon": frozenset(),
     "deliverables": frozenset(),
     "knowledge_base": frozenset(),
     "web_crawler": frozenset(),

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/amazon/__init__.py

@@ -0,0 +1 @@
+"""``amazon`` builtin subagent: structured public Amazon product data."""

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/amazon/agent.py

@@ -0,0 +1,43 @@
+"""``amazon`` route: ``SurfSenseSubagentSpec`` builder for deepagents."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from langchain_core.language_models import BaseChatModel
+from langchain_core.tools import BaseTool
+
+from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import (
+    read_md_file,
+)
+from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec
+from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import (
+    pack_subagent,
+)
+
+from .tools.index import NAME, RULESET, load_tools
+
+
+def build_subagent(
+    *,
+    dependencies: dict[str, Any],
+    model: BaseChatModel | None = None,
+    middleware_stack: dict[str, Any] | None = None,
+    mcp_tools: list[BaseTool] | None = None,
+) -> SurfSenseSubagentSpec:
+    tools = [*load_tools(dependencies=dependencies), *(mcp_tools or [])]
+    description = (
+        read_md_file(__package__, "description").strip()
+        or "Scrapes public Amazon product data for a URL or search term."
+    )
+    system_prompt = read_md_file(__package__, "system_prompt").strip()
+    return pack_subagent(
+        name=NAME,
+        description=description,
+        system_prompt=system_prompt,
+        tools=tools,
+        ruleset=RULESET,
+        dependencies=dependencies,
+        model=model,
+        middleware_stack=middleware_stack,
+    )

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/amazon/description.md

@@ -0,0 +1,2 @@
+Amazon product specialist: scrapes public Amazon listings and returns structured product data — title, ASIN, brand, price and list price, star rating and review breakdown, availability, images, features, best-seller ranks, marketplace offers, third-party sellers, product variants, and on-page customer reviews. Works from a search term (e.g. "mechanical keyboard") or from Amazon product, search, category, best-seller, or short (a.co / amzn.to) URLs, on any Amazon marketplace domain. Only public, anonymous data — no login or seller account. Can enrich results with extra offers and seller profiles, expand product variants, and localize pricing/availability by country and ZIP.
+Use it for product research, price and buy-box tracking, review mining, catalog enrichment by ASIN, and best-seller/category monitoring. Triggers include "find X on Amazon", "Amazon price of X", "reviews for this Amazon product", "compare these Amazon products", "top-selling X on Amazon", and "look up this ASIN/Amazon URL". Not for general web search (use the Google Search specialist), reading an arbitrary non-Amazon page (use the web crawling specialist), or other marketplaces.

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/amazon/system_prompt.md

@@ -0,0 +1,67 @@
+You are the SurfSense Amazon sub-agent.
+You receive delegated instructions from a supervisor agent and return structured results for supervisor synthesis.
+
+<goal>
+Answer the delegated question from public Amazon product data gathered with your verb, comparing against earlier results already in this conversation when the task calls for it.
+</goal>
+
+<available_tools>
+- `amazon_scrape`
+- `read_run` / `search_run` (free readers for stored scrape output)
+</available_tools>
+
+<playbook>
+- Discovering products: call `amazon_scrape` with `search_terms` (e.g. ["mechanical keyboard"]), setting `domain` when a non-US marketplace matters (e.g. "www.amazon.co.uk").
+- Specific products: pass Amazon product URLs (or search / category / best-seller / short a.co URLs) in `urls`.
+- Faster listings: set `include_details=false` to return card-only results without opening each product page.
+- Pricing and buy box: raise `max_offers` to pull additional marketplace offers; set `include_sellers=true` to attach seller profiles.
+- Variants: raise `max_variants` to return variants as separate results; set `include_variant_prices=true` for per-variant prices.
+- Localized pricing/availability: set `country_code` and `zip_code`.
+- Batch multiple URLs or search terms into one call rather than many single-source calls.
+<include snippet="run_reader"/>
+- Comparison requests: pull the current products, compare against prior values already in this conversation's earlier tool results, and report concrete deltas (price up/down, rating change, rank moves, stock changes).
+</playbook>
+
+<tool_policy>
+- Use only tools in `<available_tools>`.
+- Report only results present in the tool output. Never invent titles, ASINs, prices, ratings, or rankings.
+- Provide at least one of `urls` or `search_terms`; they cannot be combined arbitrarily beyond the source cap.
+</tool_policy>
+
+<out_of_scope>
+- Do not perform general web search — that is the Google Search specialist's job.
+- Do not read or extract an arbitrary non-Amazon page — return the URL for the web crawling specialist.
+- Do not generate deliverables or perform connector mutations; return findings for the supervisor to act on.
+- Only public, anonymous Amazon data — never anything behind a login or seller account.
+</out_of_scope>
+
+<safety>
+- Report uncertainty explicitly when evidence is incomplete or conflicting.
+- Never present unverified claims as facts.
+</safety>
+
+<failure_policy>
+- Underspecified request — no usable search term or URL — return `status=blocked` with the missing fields.
+- Tool failure: return `status=error` with a concise recovery `next_step`.
+- No useful evidence: return `status=blocked` with a narrower query or the scope you still need.
+</failure_policy>
+
+<output_contract>
+Return **only** one JSON object (no markdown/prose):
+{
+  "status": "success" | "partial" | "blocked" | "error",
+  "action_summary": string,
+  "evidence": {
+    "findings": string[],
+    "sources": string[],
+    "confidence": "high" | "medium" | "low"
+  },
+  "next_step": string | null,
+  "missing_fields": string[] | null,
+  "assumptions": string[] | null
+}
+<include snippet="output_contract_base"/>
+Route-specific rules:
+- `evidence.findings`: max 10 entries, each a single sentence stating one distinct product or delta. Do not paste raw payloads.
+- `evidence.sources`: max 10 URLs, one per finding when applicable. List each URL once.
+</output_contract>

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/amazon/tools/index.py

@@ -0,0 +1,27 @@
+"""``amazon`` sub-agent tools: the Amazon product scrape capability verb."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from langchain_core.tools import BaseTool
+
+from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset
+from app.capabilities.amazon.scrape.definition import AMAZON_SCRAPE
+from app.capabilities.core.access.agent import build_capability_tools
+
+NAME = "amazon"
+
+RULESET = Ruleset(origin=NAME, rules=[])
+
+_CI_VERBS = [AMAZON_SCRAPE]
+
+
+def load_tools(
+    *, dependencies: dict[str, Any] | None = None, **kwargs: Any
+) -> list[BaseTool]:
+    d = {**(dependencies or {}), **kwargs}
+    return build_capability_tools(
+        workspace_id=d.get("workspace_id"),
+        capabilities=_CI_VERBS,
+    )

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py

@@ -12,6 +12,9 @@ from langchain_core.tools import BaseTool
 from app.agents.chat.multi_agent_chat.constants import (
     SUBAGENT_TO_REQUIRED_CONNECTOR_MAP,
 )
+from app.agents.chat.multi_agent_chat.subagents.builtins.amazon.agent import (
+    build_subagent as build_amazon_subagent,
+)
 from app.agents.chat.multi_agent_chat.subagents.builtins.deliverables.agent import (
     build_subagent as build_deliverables_subagent,
 )
@@ -80,6 +83,7 @@ class SubagentBuilder(Protocol):
 
 
 SUBAGENT_BUILDERS_BY_NAME: dict[str, SubagentBuilder] = {
+    "amazon": build_amazon_subagent,
     "deliverables": build_deliverables_subagent,
     "dropbox": build_dropbox_subagent,
     "google_drive": build_google_drive_subagent,

surfsense_backend/app/capabilities/__init__.py

@@ -2,4 +2,6 @@
 
 from __future__ import annotations
 
+from app.capabilities import amazon as _amazon  # noqa: F401
+
 __all__: list[str] = []

surfsense_backend/app/capabilities/amazon/__init__.py

@@ -0,0 +1,5 @@
+"""Amazon capability namespace."""
+
+from __future__ import annotations
+
+from app.capabilities.amazon.scrape import definition as _scrape  # noqa: F401

surfsense_backend/app/capabilities/amazon/scrape/__init__.py

@@ -0,0 +1,3 @@
+"""Amazon product scraping capability."""
+
+from __future__ import annotations

surfsense_backend/app/capabilities/amazon/scrape/definition.py

@@ -0,0 +1,22 @@
+"""Registration for the ``amazon.scrape`` capability."""
+
+from __future__ import annotations
+
+from app.capabilities.amazon.scrape.executor import build_scrape_executor
+from app.capabilities.amazon.scrape.schemas import ScrapeInput, ScrapeOutput
+from app.capabilities.core import BillingUnit, Capability, register_capability
+
+AMAZON_SCRAPE = Capability(
+    name="amazon.scrape",
+    description=(
+        "Scrape public Amazon product details, search results, offers, sellers, "
+        "best-seller rankings, and on-page reviews."
+    ),
+    input_schema=ScrapeInput,
+    output_schema=ScrapeOutput,
+    executor=build_scrape_executor(),
+    billing_unit=BillingUnit.AMAZON_PRODUCT,
+    docs_url="/docs/connectors/native/amazon",
+)
+
+register_capability(AMAZON_SCRAPE)

surfsense_backend/app/capabilities/amazon/scrape/executor.py

@@ -0,0 +1,59 @@
+"""Executor for the ``amazon.scrape`` capability."""
+
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable
+from urllib.parse import quote_plus
+
+from app.capabilities.amazon.scrape.schemas import (
+    MAX_AMAZON_RESULTS,
+    ScrapeInput,
+    ScrapeOutput,
+)
+from app.capabilities.core import Executor
+from app.capabilities.core.progress import emit_progress
+from app.proprietary.platforms.amazon import AmazonScrapeInput, scrape_products
+
+ScrapeFn = Callable[..., Awaitable[list[dict]]]
+
+
+def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor:
+    """Bind the capability input mapping to a replaceable scraper function."""
+    scrape_fn = scrape_fn or scrape_products
+
+    async def execute(payload: ScrapeInput) -> ScrapeOutput:
+        search_urls = [
+            f"https://{payload.domain}/s?k={quote_plus(term)}"
+            for term in payload.search_terms
+        ]
+        input_model = AmazonScrapeInput(
+            categoryOrProductUrls=[
+                {"url": url} for url in [*payload.urls, *search_urls]
+            ],
+            maxItemsPerStartUrl=payload.max_items,
+            language=payload.language,
+            countryCode=payload.country_code,
+            zipCode=payload.zip_code,
+            scrapeProductDetails=payload.include_details,
+            maxOffers=payload.max_offers,
+            scrapeSellers=payload.include_sellers,
+            maxProductVariantsAsSeparateResults=payload.max_variants,
+            scrapeProductVariantPrices=payload.include_variant_prices,
+        )
+        emit_progress(
+            "starting",
+            "Scraping Amazon products",
+            total=payload.estimated_units,
+            unit="product",
+        )
+        items = await scrape_fn(input_model, limit=MAX_AMAZON_RESULTS)
+        emit_progress(
+            "done",
+            f"Scraped {sum('error' not in item for item in items)} product(s)",
+            current=len(items),
+            total=payload.estimated_units,
+            unit="product",
+        )
+        return ScrapeOutput(items=items)
+
+    return execute

surfsense_backend/app/capabilities/amazon/scrape/schemas.py

@@ -0,0 +1,61 @@
+"""Input and output contracts for ``amazon.scrape``."""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, Field, model_validator
+
+from app.proprietary.platforms.amazon import ProductItem
+
+MAX_AMAZON_SOURCES = 20
+MAX_AMAZON_RESULTS = 1000
+
+
+class ScrapeInput(BaseModel):
+    """Agent-facing controls for public product discovery and enrichment."""
+
+    urls: list[str] = Field(default_factory=list, max_length=MAX_AMAZON_SOURCES)
+    search_terms: list[str] = Field(default_factory=list, max_length=MAX_AMAZON_SOURCES)
+    max_items: int = Field(default=10, ge=1, le=100)
+    domain: str = Field(
+        default="www.amazon.com", pattern=r"^(?:www\.)?amazon\.[a-z.]+$"
+    )
+    language: str | None = None
+    country_code: str | None = Field(default=None, min_length=2, max_length=2)
+    zip_code: str | None = Field(default=None, min_length=1, max_length=20)
+    include_details: bool = True
+    max_offers: int = Field(default=0, ge=0, le=100)
+    include_sellers: bool = False
+    max_variants: int = Field(default=0, ge=0, le=100)
+    include_variant_prices: bool = False
+
+    @model_validator(mode="after")
+    def _require_source(self) -> ScrapeInput:
+        if not (self.urls or self.search_terms):
+            raise ValueError("Provide at least one URL or search term.")
+        if len(self.urls) + len(self.search_terms) > MAX_AMAZON_SOURCES:
+            raise ValueError(
+                f"Provide no more than {MAX_AMAZON_SOURCES} combined sources."
+            )
+        return self
+
+    @property
+    def estimated_units(self) -> int:
+        """Worst-case returned products within the hard per-run ceiling."""
+        search_products = len(self.search_terms) * self.max_items
+        direct_products = len(self.urls) * (1 + self.max_variants)
+        return min(search_products + direct_products, MAX_AMAZON_RESULTS)
+
+
+class ScrapeOutput(BaseModel):
+    """Products and structured per-input errors in emission order."""
+
+    items: list[ProductItem] = Field(default_factory=list)
+
+    @property
+    def billable_units(self) -> int:
+        """Count successful products; error items are never billed."""
+        return sum(
+            1
+            for item in self.items
+            if not (item.model_extra and item.model_extra.get("error"))
+        )

surfsense_backend/app/capabilities/core/billing.py

@@ -33,6 +33,7 @@ _PLATFORM_RATE_KEYS: dict[BillingUnit, str] = {
     BillingUnit.GOOGLE_SEARCH_SERP: "GOOGLE_SEARCH_MICROS_PER_SERP",
     BillingUnit.GOOGLE_MAPS_PLACE: "GOOGLE_MAPS_MICROS_PER_PLACE",
     BillingUnit.GOOGLE_MAPS_REVIEW: "GOOGLE_MAPS_MICROS_PER_REVIEW",
+    BillingUnit.AMAZON_PRODUCT: "AMAZON_MICROS_PER_PRODUCT",
     BillingUnit.YOUTUBE_VIDEO: "YOUTUBE_MICROS_PER_VIDEO",
     BillingUnit.YOUTUBE_COMMENT: "YOUTUBE_MICROS_PER_COMMENT",
     BillingUnit.INSTAGRAM_ITEM: "INSTAGRAM_SCRAPE_MICROS_PER_ITEM",
@@ -54,6 +55,7 @@ _UNIT_NOUNS: dict[BillingUnit, str] = {
     BillingUnit.GOOGLE_SEARCH_SERP: "SERP",
     BillingUnit.GOOGLE_MAPS_PLACE: "place",
     BillingUnit.GOOGLE_MAPS_REVIEW: "review",
+    BillingUnit.AMAZON_PRODUCT: "product",
     BillingUnit.YOUTUBE_VIDEO: "video",
     BillingUnit.YOUTUBE_COMMENT: "comment",
     BillingUnit.INSTAGRAM_ITEM: "item",

surfsense_backend/app/capabilities/core/types.py

@@ -23,6 +23,7 @@ class BillingUnit(StrEnum):
     GOOGLE_SEARCH_SERP = "google_search_serp"
     GOOGLE_MAPS_PLACE = "google_maps_place"
     GOOGLE_MAPS_REVIEW = "google_maps_review"
+    AMAZON_PRODUCT = "amazon_product"
     YOUTUBE_VIDEO = "youtube_video"
     YOUTUBE_COMMENT = "youtube_comment"
     INSTAGRAM_ITEM = "instagram_item"

surfsense_backend/app/config/__init__.py

@@ -697,9 +697,8 @@ class Config:
     # per item; retune with an env change + restart (no code/migration):
     #   <KEY> = round(USD_per_1000_items * 1_000)
     #   $3.50/1000 -> 3500 | $5.00/1000 -> 5000 | $2.00/1000 -> 2000
-    # Defaults sit at/above Apify's first-party actor rates (Jul 2026), which
-    # is justified because SurfSense charges no subscription tiers, no
-    # per-run actor-start fees, and no separate proxy/compute/storage billing.
+    # Defaults include margin for proxy, compute, and storage costs while
+    # remaining independently adjustable for each platform.
     PLATFORM_SCRAPE_BILLING_ENABLED = (
         os.getenv("PLATFORM_SCRAPE_BILLING_ENABLED", "FALSE").upper() == "TRUE"
     )
@@ -715,6 +714,7 @@ class Config:
     GOOGLE_MAPS_MICROS_PER_REVIEW = int(
         os.getenv("GOOGLE_MAPS_MICROS_PER_REVIEW", "1500")
     )
+    AMAZON_MICROS_PER_PRODUCT = int(os.getenv("AMAZON_MICROS_PER_PRODUCT", "3500"))
     YOUTUBE_MICROS_PER_VIDEO = int(os.getenv("YOUTUBE_MICROS_PER_VIDEO", "2500"))
     # Kept separate from the video rate so comments can be re-tuned toward the
     # cheaper per-comment market ($0.40-2.00/1k) without touching video pricing.

surfsense_backend/app/proprietary/platforms/amazon/README.md

@@ -0,0 +1,50 @@
+# Amazon Product Scraper
+
+Current status: product details, search and category discovery, offers, sellers,
+best-seller rankings, on-page reviews, localized delivery sessions, and product
+variants are implemented. The `amazon.scrape` capability exposes the scraper
+with per-product billing and a hard per-run result ceiling.
+
+## Scope
+
+The scraper reads public pages available to anonymous visitors. It does not log
+in, use account cookies, or retrieve account-gated content. Delivery
+localization uses an anonymous session cookie pinned to the same proxy exit.
+
+Deep review pagination currently requires an account on Amazon and is therefore
+out of scope. Reviews embedded in a public product page are returned in
+`productPageReviews` and `productPageReviewsFromOtherCountries`.
+
+## Architecture
+
+- `schemas.py` defines the stable input, product, and error models.
+- `url_resolver.py` classifies product, search, best-sellers, and shortened URLs.
+- `fetch.py` owns proxy-aware HTTP access, block detection, retries, and
+  anonymous location sessions.
+- `parsers.py` contains pure, defensive HTML parsers.
+- `scraper.py` coordinates discovery, enrichment, concurrency, limits, and
+  in-stream errors.
+
+## Implementation progress
+
+- Done: public product detail parsing and shortened-link dispatch.
+- Done: search/category paging with card-only and detailed modes.
+- Done: public offers and seller enrichment.
+- Done: best-seller rankings and product-page reviews.
+- Done: anonymous localized delivery sessions on sticky proxy exits.
+- Done: variant expansion, variant prices, capability registration, and billing.
+
+## Verification
+
+Offline fixtures cover every parser and flow:
+
+```bash
+cd surfsense_backend
+uv run pytest tests/unit/platforms/amazon tests/unit/capabilities/amazon
+```
+
+The manual live check requires proxy credentials:
+
+```bash
+uv run python scripts/e2e_amazon_scraper.py
+```

surfsense_backend/app/proprietary/platforms/amazon/__init__.py

@@ -0,0 +1,12 @@
+"""Platform-native Amazon Product Scraper."""
+
+from .schemas import AmazonScrapeInput, ErrorItem, ProductItem
+from .scraper import iter_products, scrape_products
+
+__all__ = [
+    "AmazonScrapeInput",
+    "ErrorItem",
+    "ProductItem",
+    "iter_products",
+    "scrape_products",
+]

surfsense_backend/app/proprietary/platforms/amazon/fetch.py

@@ -0,0 +1,388 @@
+"""Network access for public Amazon pages.
+
+All requests use the configured residential proxy. Responses that contain an
+anti-bot interstitial are retried with a fresh proxy exit, while ordinary HTTP
+failures are returned to the caller for domain-specific error handling.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import re
+import time
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass, field
+from typing import Any
+
+from scrapling.fetchers import AsyncFetcher
+
+from app.utils.proxy import get_geo_proxy_url, get_sticky_proxy_url
+
+logger = logging.getLogger(__name__)
+
+_MAX_IP_ATTEMPTS = 8
+_REQUEST_TIMEOUT_S = 30
+_HEADERS = {
+    "Accept-Language": "en-US,en;q=0.9",
+    "Accept": "text/html,application/xhtml+xml,application/json;q=0.9,*/*;q=0.8",
+}
+_BLOCK_MARKERS = (
+    "/errors/validatecaptcha",
+    "api-services-support@amazon.com",
+    "robot check",
+    "enter the characters you see below",
+    "token.awswaf.com",
+    "/challenge.js",
+    "awswafintegration",
+    "bm-verify=",
+)
+_CSRF_PATTERNS = (
+    re.compile(
+        r"""(?:name|id)=["'](?:anti-csrftoken-a2z|csrf-token)["'][^>]*"""
+        r"""(?:value|content)=["']([^"']+)""",
+        re.IGNORECASE,
+    ),
+    re.compile(
+        r"""["'](?:anti-csrftoken-a2z|csrf-token)["']\s*[:=]\s*["']([^"']+)""",
+        re.IGNORECASE,
+    ),
+)
+
+
+@dataclass(frozen=True)
+class FetchResult:
+    """The response details needed by scraper flows."""
+
+    status: int
+    html: str
+    url: str
+    cookies: dict[str, str]
+    headers: dict[str, str] = field(default_factory=dict)
+
+
+@dataclass
+class LocationSession:
+    """Anonymous delivery-location state pinned to one proxy exit."""
+
+    proxy: str | None
+    cookies: dict[str, str]
+    country_code: str | None
+    zip_code: str
+    location_text: str | None
+    created_at: float
+
+
+async def gather_bounded[T](
+    factories: list[Callable[[], Awaitable[T]]], *, concurrency: int
+) -> list[T]:
+    """Run async factories concurrently while preserving input order."""
+    if not factories:
+        return []
+    semaphore = asyncio.Semaphore(max(1, concurrency))
+
+    async def run(factory: Callable[[], Awaitable[T]]) -> T:
+        async with semaphore:
+            return await factory()
+
+    return await asyncio.gather(*(run(factory) for factory in factories))
+
+
+def is_blocked(
+    html: str | None, status: int, headers: dict[str, str] | None = None
+) -> bool:
+    """Return whether a response is an Amazon anti-bot interstitial.
+
+    ``202`` is a soft anti-bot response Amazon serves to some proxy exits (an
+    empty/accepted body rather than the page), so it is treated as blocked and
+    retried on a fresh exit rather than parsed as a real page.
+    """
+    if status in {202, 429, 503}:
+        return True
+    if _header_value(headers, "x-amzn-waf-action") == "challenge":
+        return True
+    text = (html or "")[:200_000].lower()
+    return any(marker in text for marker in _BLOCK_MARKERS)
+
+
+def _header_value(headers: dict[str, str] | None, name: str) -> str | None:
+    if not headers:
+        return None
+    needle = name.lower()
+    for key, value in headers.items():
+        if key.lower() == needle:
+            return str(value).strip().lower()
+    return None
+
+
+def _response_url(page: Any, fallback: str) -> str:
+    value = getattr(page, "url", None)
+    return str(value) if value else fallback
+
+
+def _response_cookies(page: Any) -> dict[str, str]:
+    cookies = getattr(page, "cookies", None)
+    return dict(cookies) if isinstance(cookies, dict) else {}
+
+
+def _response_headers(page: Any) -> dict[str, str]:
+    headers = getattr(page, "headers", None)
+    return dict(headers) if isinstance(headers, dict) else {}
+
+
+def _selected_proxy(
+    proxy: str | None, country: str | None, attempt: int, url: str
+) -> str | None:
+    if proxy is not None:
+        return proxy
+    if country and attempt > 1:
+        session_id = f"amazon-{country}-{attempt}-{abs(hash((url, time.time_ns()))):x}"
+        return get_sticky_proxy_url(session_id, country)
+    return get_geo_proxy_url(country)
+
+
+async def fetch_page(
+    url: str,
+    *,
+    cookies: dict[str, str] | None = None,
+    proxy: str | None = None,
+    country: str | None = None,
+    accept_language: str | None = None,
+    method: str = "GET",
+    data: dict[str, str] | None = None,
+    headers: dict[str, str] | None = None,
+    rotate_on_block: bool = True,
+) -> FetchResult | None:
+    """Fetch a page and retry blocked responses with fresh proxy exits."""
+    attempts = _MAX_IP_ATTEMPTS if rotate_on_block else 1
+    for attempt in range(1, attempts + 1):
+        selected_proxy = _selected_proxy(proxy, country, attempt, url)
+        started = time.perf_counter()
+        try:
+            request = AsyncFetcher.post if method == "POST" else AsyncFetcher.get
+            request_headers = {**_HEADERS}
+            if accept_language:
+                request_headers["Accept-Language"] = (
+                    f"{accept_language},{accept_language.split('-', 1)[0]};q=0.9"
+                )
+            request_headers.update(headers or {})
+            kwargs: dict[str, Any] = {
+                "headers": request_headers,
+                "cookies": cookies or {},
+                "proxy": selected_proxy,
+                "stealthy_headers": True,
+                "timeout": _REQUEST_TIMEOUT_S,
+            }
+            if method == "POST":
+                kwargs["data"] = data or {}
+            page = await request(url, **kwargs)
+        except Exception as exc:
+            logger.warning("Amazon request failed for %s: %s", url, exc)
+            if proxy is not None:
+                return None
+            continue
+
+        status = int(getattr(page, "status", 0) or 0)
+        html = getattr(page, "html_content", None) or ""
+        response_headers = _response_headers(page)
+        logger.info(
+            "[amazon][perf] method=%s status=%s attempt=%s fetch_ms=%.1f url=%s",
+            method,
+            status,
+            attempt,
+            (time.perf_counter() - started) * 1000,
+            url,
+        )
+        if rotate_on_block and is_blocked(html, status, response_headers):
+            logger.info(
+                "Amazon blocked proxy attempt %s/%s for %s", attempt, attempts, url
+            )
+            continue
+        return FetchResult(
+            status=status,
+            html=html,
+            url=_response_url(page, url),
+            cookies=_response_cookies(page),
+            headers=response_headers,
+        )
+    logger.warning("Amazon exhausted %s proxy attempts for %s", attempts, url)
+    return None
+
+
+async def fetch_html(
+    url: str,
+    *,
+    cookies: dict[str, str] | None = None,
+    proxy: str | None = None,
+    country: str | None = None,
+    accept_language: str | None = None,
+) -> str | None:
+    """Return public page HTML, or ``None`` when no usable response is obtained."""
+    result = await fetch_page(
+        url,
+        cookies=cookies,
+        proxy=proxy,
+        country=country,
+        accept_language=accept_language,
+    )
+    return result.html if result is not None and result.status == 200 else None
+
+
+async def resolve_shortlink(
+    url: str, *, country: str | None = None, accept_language: str | None = None
+) -> str | None:
+    """Follow a shortened Amazon URL and return its final destination."""
+    result = await fetch_page(url, country=country, accept_language=accept_language)
+    return result.url if result is not None and result.status == 200 else None
+
+
+def _origin(domain: str) -> str:
+    return f"https://{domain}"
+
+
+def _csrf_token(html: str) -> str | None:
+    for pattern in _CSRF_PATTERNS:
+        match = pattern.search(html)
+        if match:
+            return match.group(1)
+    return None
+
+
+async def fetch_aod_html(
+    asin: str,
+    domain: str,
+    *,
+    cookies: dict[str, str] | None = None,
+    proxy: str | None = None,
+    country: str | None = None,
+    accept_language: str | None = None,
+) -> str | None:
+    """Fetch the public all-offers panel for one product.
+
+    ``aodAjaxMain`` is a path segment, not a query param. Amazon also serves this
+    panel as a JS-only modal for many (especially US) ASINs, so a 404 here is
+    expected and the caller falls back to the PDP buy-box winner.
+    """
+    url = f"{_origin(domain)}/gp/product/ajax/aodAjaxMain/?asin={asin}"
+    return await fetch_html(
+        url,
+        cookies=cookies,
+        proxy=proxy,
+        country=country,
+        accept_language=accept_language,
+    )
+
+
+async def fetch_seller_html(
+    seller_id: str,
+    domain: str,
+    *,
+    cookies: dict[str, str] | None = None,
+    proxy: str | None = None,
+    country: str | None = None,
+    accept_language: str | None = None,
+) -> str | None:
+    """Fetch a public seller profile."""
+    return await fetch_html(
+        f"{_origin(domain)}/sp?seller={seller_id}",
+        cookies=cookies,
+        proxy=proxy,
+        country=country,
+        accept_language=accept_language,
+    )
+
+
+_LOCATION_SESSION_TTL_S = 30 * 60
+_location_sessions: dict[tuple[str, str, str | None], LocationSession] = {}
+_location_locks: dict[tuple[str, str, str | None], asyncio.Lock] = {}
+
+
+def should_localize(route: str, deliverable_routes: list[str] | None) -> bool:
+    """Return whether a request route should use delivery-location state."""
+    return route.upper() in {value.upper() for value in (deliverable_routes or [])}
+
+
+async def _sticky_proxy(session_id: str, country: str | None = None) -> str | None:
+    """Request a stable proxy exit when the active provider supports it."""
+    try:
+        return get_sticky_proxy_url(session_id, country)
+    except (ImportError, NotImplementedError):
+        return get_geo_proxy_url(country)
+
+
+async def get_location_session(
+    domain: str,
+    *,
+    zip_code: str,
+    country_code: str | None,
+    country: str | None = None,
+    accept_language: str | None = None,
+) -> LocationSession | None:
+    """Create or reuse an anonymous delivery-location session."""
+    key = (domain, zip_code, country_code)
+    cached = _location_sessions.get(key)
+    if cached and time.time() - cached.created_at < _LOCATION_SESSION_TTL_S:
+        return cached
+
+    lock = _location_locks.setdefault(key, asyncio.Lock())
+    async with lock:
+        cached = _location_sessions.get(key)
+        if cached and time.time() - cached.created_at < _LOCATION_SESSION_TTL_S:
+            return cached
+
+        session_id = f"amazon-{abs(hash(key)) & 0xFFFFFFFF:x}"
+        proxy = await _sticky_proxy(session_id, country)
+        home = await fetch_page(
+            f"{_origin(domain)}/?ref_=nav_logo",
+            proxy=proxy,
+            country=country,
+            accept_language=accept_language,
+            rotate_on_block=False,
+        )
+        if home is None or home.status != 200:
+            return None
+        csrf = _csrf_token(home.html)
+        if csrf is None:
+            logger.warning("Amazon location session did not expose a CSRF token")
+            return None
+
+        endpoint = f"{_origin(domain)}/gp/delivery/ajax/address-change.html"
+        payload = {
+            "locationType": "LOCATION_INPUT",
+            "zipCode": zip_code,
+            "storeContext": "generic",
+            "deviceType": "web",
+            "pageType": "Gateway",
+            "actionSource": "glow",
+        }
+        cookies = dict(home.cookies)
+        cookies["anti-csrftoken-a2z"] = csrf
+        changed = await fetch_page(
+            endpoint,
+            cookies=cookies,
+            proxy=proxy,
+            method="POST",
+            data=payload,
+            headers={
+                "anti-csrftoken-a2z": csrf,
+                "X-Requested-With": "XMLHttpRequest",
+                "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
+            },
+            country=country,
+            accept_language=accept_language,
+            rotate_on_block=False,
+        )
+        if changed is None or changed.status != 200:
+            return None
+        cookies.update(changed.cookies)
+        location_text = zip_code
+        session = LocationSession(
+            proxy=proxy,
+            cookies=cookies,
+            country_code=country_code.upper() if country_code else None,
+            zip_code=zip_code,
+            location_text=location_text,
+            created_at=time.time(),
+        )
+        _location_sessions[key] = session
+        return session

surfsense_backend/app/proprietary/platforms/amazon/locale.py

@@ -0,0 +1,33 @@
+"""Marketplace locale helpers for Amazon fetch routing.
+
+Only proxy countries verified against the configured DataImpulse plan are mapped
+here. Unmapped marketplaces fall back to the default proxy exit.
+"""
+
+_MARKETPLACE_COUNTRY = {
+    "com": "us",
+    "co.uk": "gb",
+    "de": "de",
+    "fr": "fr",
+    "it": "it",
+    "es": "es",
+}
+
+_MARKETPLACE_LANGUAGE = {
+    "com": "en-US",
+    "co.uk": "en-GB",
+    "de": "de-DE",
+    "fr": "fr-FR",
+    "it": "it-IT",
+    "es": "es-ES",
+}
+
+
+def proxy_country_for(marketplace: str | None) -> str | None:
+    """Return a confirmed proxy exit country for one Amazon marketplace."""
+    return _MARKETPLACE_COUNTRY.get((marketplace or "").lower())
+
+
+def accept_language_for(marketplace: str | None) -> str:
+    """Return the browser language header that matches the marketplace UI."""
+    return _MARKETPLACE_LANGUAGE.get((marketplace or "").lower(), "en-US")

surfsense_backend/app/proprietary/platforms/amazon/parsers.py

@@ -0,0 +1,586 @@
+"""Pure parsers for public Amazon HTML.
+
+Selectors cover stable element IDs first and use small, explicit fallbacks for
+layout variants. Missing sections return empty or nullable values so isolated
+markup changes do not discard an otherwise usable product.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from html import unescape
+from typing import Any
+from urllib.parse import parse_qs, urljoin, urlparse
+
+from scrapling.parser import Adaptor
+
+_NUMBER_RE = re.compile(r"[\d,.]+")
+_ASIN_RE = re.compile(r"^[A-Z0-9]{10}$")
+_ASIN_IN_TEXT_RE = re.compile(r"\b[A-Z0-9]{10}\b")
+_CURRENCY_BY_SYMBOL = {"$": "USD", "€": "EUR", "£": "GBP", "¥": "JPY", "₹": "INR"}
+_JSON_ASSIGNMENT_RE = re.compile(
+    r"(?:dimensionValuesDisplayData|variationValues|dimensionToAsinMap)"
+    r"""\s*["']?\s*:\s*(\{.*?\})(?:,\s*["']|\s*;)""",
+    re.DOTALL,
+)
+
+
+def _one(node: Any, selector: str) -> Any | None:
+    found = node.css(selector)
+    return found[0] if found else None
+
+
+def _text(node: Any | None) -> str | None:
+    if node is None:
+        return None
+    raw = node.get_all_text(strip=True)
+    return re.sub(r"\s+", " ", raw).strip() or None
+
+
+def _texts(node: Any, selector: str) -> list[str]:
+    values: list[str] = []
+    for match in node.css(selector):
+        value = _text(match)
+        if value and value not in values:
+            values.append(value)
+    return values
+
+
+def _first_text(node: Any, *selectors: str) -> str | None:
+    for selector in selectors:
+        value = _text(_one(node, selector))
+        if value:
+            return value
+    return None
+
+
+def _attr(node: Any | None, name: str) -> str | None:
+    if node is None:
+        return None
+    value = node.attrib.get(name)
+    return str(value).strip() if value else None
+
+
+def _first_attr(node: Any, name: str, *selectors: str) -> str | None:
+    for selector in selectors:
+        value = _attr(_one(node, selector), name)
+        if value:
+            return value
+    return None
+
+
+def _integer(value: str | None) -> int | None:
+    if not value:
+        return None
+    match = _NUMBER_RE.search(value)
+    if not match:
+        return None
+    digits = re.sub(r"\D", "", match.group(0))
+    return int(digits) if digits else None
+
+
+def _float(value: str | None) -> float | None:
+    if not value:
+        return None
+    match = _NUMBER_RE.search(value)
+    if not match:
+        return None
+    token = match.group(0)
+    if "," in token and "." in token:
+        decimal = "," if token.rfind(",") > token.rfind(".") else "."
+        grouping = "." if decimal == "," else ","
+        token = token.replace(grouping, "").replace(decimal, ".")
+    elif (
+        token.count(",") == 1 and "." not in token and len(token.rsplit(",", 1)[1]) <= 2
+    ):
+        token = token.replace(",", ".")
+    else:
+        token = token.replace(",", "")
+    try:
+        return float(token)
+    except ValueError:
+        return None
+
+
+def _price(value: str | None) -> dict[str, Any] | None:
+    amount = _float(value)
+    if amount is None:
+        return None
+    currency = next(
+        (
+            code
+            for symbol, code in _CURRENCY_BY_SYMBOL.items()
+            if symbol in (value or "")
+        ),
+        None,
+    )
+    if currency is None and value:
+        match = re.search(r"\b(USD|EUR|GBP|JPY|INR|CAD|AUD)\b", value)
+        currency = match.group(1) if match else None
+    return {"value": amount, "currency": currency}
+
+
+def _absolute(domain: str, href: str | None) -> str | None:
+    return urljoin(f"https://{domain}", href) if href else None
+
+
+def _asin_from_href(href: str | None) -> str | None:
+    if not href:
+        return None
+    match = re.search(r"/(?:dp|gp/product)/([A-Z0-9]{10})(?:[/?]|$)", href)
+    return match.group(1) if match else None
+
+
+def _embedded_json(html: str, key: str) -> Any | None:
+    """Decode a JSON value assigned to ``key`` inside an inline script."""
+    marker = re.search(rf"""["']?{re.escape(key)}["']?\s*:\s*""", html)
+    if marker is None:
+        return None
+    start = marker.end()
+    while start < len(html) and html[start].isspace():
+        start += 1
+    if start >= len(html) or html[start] not in "[{":
+        return None
+    opening = html[start]
+    closing = "}" if opening == "{" else "]"
+    depth = 0
+    quoted = False
+    escaped = False
+    for index in range(start, len(html)):
+        char = html[index]
+        if quoted:
+            if escaped:
+                escaped = False
+            elif char == "\\":
+                escaped = True
+            elif char == '"':
+                quoted = False
+            continue
+        if char == '"':
+            quoted = True
+        elif char == opening:
+            depth += 1
+        elif char == closing:
+            depth -= 1
+            if depth == 0:
+                try:
+                    return json.loads(html[start : index + 1])
+                except json.JSONDecodeError:
+                    return None
+    return None
+
+
+def _stars_breakdown(doc: Adaptor) -> dict[str, float | None] | None:
+    breakdown: dict[str, float | None] = {}
+    for row in doc.css("#histogramTable tr, [data-hook='rating-histogram'] tr"):
+        label = _first_text(row, ".a-text-left", "td:first-child", "span")
+        percent = _first_text(row, ".a-text-right", "td:last-child")
+        star = _integer(label)
+        value = _float(percent)
+        if star and 1 <= star <= 5 and value is not None:
+            breakdown[f"{star}star"] = value / 100 if value > 1 else value
+    return breakdown or None
+
+
+def _variant_data(
+    html: str,
+) -> tuple[list[str], list[dict[str, Any]], list[dict[str, Any]]]:
+    display = _embedded_json(html, "dimensionValuesDisplayData")
+    variants = _embedded_json(html, "variationValues")
+    asins: list[str] = []
+    details: list[dict[str, Any]] = []
+    attributes: list[dict[str, Any]] = []
+
+    if isinstance(display, dict):
+        for asin, values in display.items():
+            if _ASIN_RE.match(str(asin)):
+                asins.append(str(asin))
+                details.append({"asin": str(asin), "values": values})
+    if isinstance(variants, dict):
+        for name, values in variants.items():
+            attributes.append({"name": name, "values": values})
+        for value in variants.values():
+            if isinstance(value, list):
+                for candidate in value:
+                    if isinstance(candidate, str) and _ASIN_RE.match(candidate):
+                        asins.append(candidate)
+    return list(dict.fromkeys(asins)), details, attributes
+
+
+def _attributes(doc: Adaptor) -> tuple[list[dict[str, str]], dict[str, str]]:
+    pairs: list[dict[str, str]] = []
+    mapped: dict[str, str] = {}
+    for row in doc.css(
+        "#productDetails_techSpec_section_1 tr, "
+        "#productDetails_detailBullets_sections1 tr, "
+        "#detailBullets_feature_div li"
+    ):
+        name = _first_text(row, "th", ".a-text-bold")
+        value = _first_text(row, "td", "span:not(.a-text-bold)")
+        if name and value:
+            name = name.rstrip(": \u200e")
+            pairs.append({"name": name, "value": value})
+            mapped[name] = value
+    return pairs, mapped
+
+
+def _reviews(
+    doc: Adaptor, domain: str
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    local: list[dict[str, Any]] = []
+    foreign: list[dict[str, Any]] = []
+    for node in doc.css("[data-hook='review']"):
+        review = {
+            "id": _attr(node, "id"),
+            "title": _first_text(node, "[data-hook='review-title']"),
+            "stars": _float(_first_text(node, "[data-hook='review-star-rating']")),
+            "text": _first_text(node, "[data-hook='review-body']"),
+            "author": _first_text(node, ".a-profile-name"),
+            "date": _first_text(node, "[data-hook='review-date']"),
+            "verified": _one(node, "[data-hook='avp-badge']") is not None,
+            "helpfulVotes": _integer(
+                _first_text(node, "[data-hook='helpful-vote-statement']")
+            ),
+        }
+        if any(value is not None for value in review.values()):
+            target = (
+                foreign
+                if "other countries" in (_text(node) or "").lower()
+                or "cm_cr_arp_d_rvw_rvwer" in (_attr(node, "class") or "")
+                else local
+            )
+            target.append(review)
+    return local, foreign
+
+
+def parse_product(
+    html: str, *, asin: str, url: str, domain: str | None = None
+) -> dict[str, Any]:
+    """Parse one product detail page into output-model fields."""
+    doc = Adaptor(html)
+    domain = domain or (urlparse(url).hostname or "www.amazon.com")
+    title = _first_text(doc, "#productTitle", "#title")
+    price_text = _first_text(
+        doc,
+        "#corePrice_feature_div .a-offscreen",
+        "#corePriceDisplay_desktop_feature_div .a-offscreen",
+        "#apex_desktop .a-price .a-offscreen",
+        "#priceblock_ourprice",
+        "#priceblock_dealprice",
+        ".priceToPay .a-offscreen",
+        "#price_inside_buybox",
+        "#centerCol .a-price .a-offscreen",
+    )
+    list_price_text = _first_text(
+        doc,
+        "#corePrice_feature_div .basisPrice .a-offscreen",
+        "#corePriceDisplay_desktop_feature_div .basisPrice .a-offscreen",
+        ".priceBlockStrikePriceString",
+        "#centerCol .a-price.a-text-price .a-offscreen",
+    )
+    availability = _first_text(doc, "#availability", "#outOfStock")
+    availability_lower = (availability or "").lower()
+    in_stock = (
+        None
+        if availability is None
+        else not any(
+            word in availability_lower for word in ("unavailable", "out of stock")
+        )
+    )
+    brand_link = _one(doc, "#bylineInfo")
+    brand_text = _text(brand_link)
+    brand = (
+        re.sub(r"^(visit the |brand:\s*)| store$", "", brand_text, flags=re.IGNORECASE)
+        if brand_text
+        else None
+    )
+    variant_asins, variant_details, variant_attributes = _variant_data(html)
+    attrs, attrs_mapped = _attributes(doc)
+    local_reviews, foreign_reviews = _reviews(doc, domain)
+
+    high_res: list[str] = []
+    gallery: list[str] = []
+    for image in doc.css("#altImages img, #landingImage, #imgTagWrapperId img"):
+        thumb = _attr(image, "src")
+        if thumb and thumb not in gallery:
+            gallery.append(thumb)
+        dynamic = _attr(image, "data-a-dynamic-image")
+        if dynamic:
+            try:
+                decoded = json.loads(unescape(dynamic))
+                for candidate in decoded:
+                    if candidate not in high_res:
+                        high_res.append(candidate)
+            except (json.JSONDecodeError, TypeError):
+                pass
+        zoom = _attr(image, "data-old-hires")
+        if zoom and zoom not in high_res:
+            high_res.append(zoom)
+
+    ranks: list[dict[str, Any]] = []
+    for node in doc.css(
+        "#detailBulletsWrapper_feature_div li, #productDetails_detailBullets_sections1 tr"
+    ):
+        text = _text(node)
+        if not text or "best sellers rank" not in text.lower():
+            continue
+        for rank, category in re.findall(r"#([\d,]+)\s+in\s+([^#(]+)", text):
+            ranks.append(
+                {"rank": int(rank.replace(",", "")), "category": category.strip()}
+            )
+
+    aplus = _one(doc, "#aplus, #aplus_feature_div")
+    brand_story = _one(doc, "#brandStory_feature_div, .apm-brand-story")
+    store_href = _attr(brand_link, "href")
+    reviews_link = _first_attr(
+        doc, "href", "#acrCustomerReviewLink", "[data-hook='see-all-reviews-link-foot']"
+    )
+    seller_link = _one(
+        doc,
+        "#sellerProfileTriggerId, #merchant-info a[href*='seller='], a[href*='/sp?seller=']",
+    )
+    seller_href = _attr(seller_link, "href")
+    seller_id = (parse_qs(urlparse(seller_href or "").query).get("seller") or [None])[0]
+    # Fall back to the buy-box "Sold by" name when there is no seller profile link
+    # (Amazon-direct items and some link-less third-party merchants).
+    seller_name = _text(seller_link) or _first_text(
+        doc,
+        "#tabular-buybox [tabular-attribute-name='Sold by'] .tabular-buybox-text",
+        "#merchantInfoFeature_feature_div .offer-display-feature-text-message",
+    )
+    return {
+        "title": title,
+        "url": url,
+        "asin": asin,
+        "originalAsin": asin,
+        "brand": brand,
+        "author": _first_text(
+            doc, ".author", "#bylineInfo_feature_div .contributorNameID"
+        ),
+        "price": _price(price_text),
+        "listPrice": _price(list_price_text),
+        "shippingPrice": _price(
+            _first_text(doc, "#deliveryBlockMessage .a-color-secondary")
+        ),
+        "inStock": in_stock,
+        "inStockText": availability,
+        "delivery": _first_text(
+            doc, "#mir-layout-DELIVERY_BLOCK-slot-PRIMARY_DELIVERY_MESSAGE_LARGE"
+        ),
+        "fastestDelivery": _first_text(
+            doc, "#mir-layout-DELIVERY_BLOCK-slot-SECONDARY_DELIVERY_MESSAGE_LARGE"
+        ),
+        "condition": _first_text(
+            doc, "#buyNew_noncbb .a-color-price", "#usedAccordionRow"
+        ),
+        "stars": _float(
+            _first_text(doc, "#acrPopover", "[data-hook='rating-out-of-text']")
+        ),
+        "starsBreakdown": _stars_breakdown(doc),
+        "reviewsCount": _integer(_first_text(doc, "#acrCustomerReviewText")),
+        "answeredQuestions": _integer(_first_text(doc, "#askATFLink")),
+        "aiReviewsSummary": (
+            {
+                "text": _first_text(
+                    doc, "#product-summary, [data-hook='cr-insights-widget']"
+                )
+            }
+            if _first_text(doc, "#product-summary, [data-hook='cr-insights-widget']")
+            else None
+        ),
+        "monthlyPurchaseVolume": _first_text(
+            doc, "#social-proofing-faceout-title-tk_bought"
+        ),
+        "breadCrumbs": " > ".join(
+            _texts(doc, "#wayfinding-breadcrumbs_feature_div li a")
+        )
+        or None,
+        "description": _first_text(
+            doc, "#productDescription", "#bookDescription_feature_div"
+        ),
+        "features": _texts(doc, "#feature-bullets li span.a-list-item"),
+        "sustainabilityFeatures": [
+            {"text": value}
+            for value in _texts(doc, "#sustainability_feature_div .a-list-item")
+        ],
+        "videosCount": _integer(_first_text(doc, "#videoCount")),
+        "visitStoreLink": (
+            {"text": brand_text, "url": _absolute(domain, store_href)}
+            if store_href
+            else None
+        ),
+        "thumbnailImage": _first_attr(doc, "src", "#landingImage", "#imgBlkFront"),
+        "galleryThumbnails": gallery,
+        "highResolutionImages": high_res,
+        "aPlusContent": (
+            {
+                "text": _text(aplus),
+                "images": [
+                    _attr(img, "src") for img in aplus.css("img") if _attr(img, "src")
+                ],
+            }
+            if aplus is not None
+            else None
+        ),
+        "brandStory": {"text": _text(brand_story)} if brand_story is not None else None,
+        "returnPolicy": _first_text(
+            doc, "#returnsInfoFeature_feature_div", "#RETURNS_POLICY"
+        ),
+        "support": _first_text(doc, "#support_feature_div"),
+        "variantAsins": variant_asins,
+        "variantDetails": variant_details,
+        "variantAttributes": variant_attributes,
+        "attributes": attrs,
+        "attributesMapped": attrs_mapped or None,
+        "productOverview": [
+            {
+                "name": _first_text(row, "td:first-child"),
+                "value": _first_text(row, "td:last-child"),
+            }
+            for row in doc.css("#productOverview_feature_div tr")
+            if _first_text(row, "td:first-child")
+        ],
+        "seller": (
+            {
+                "id": seller_id,
+                "name": seller_name,
+                "url": _absolute(domain, seller_href),
+            }
+            if (seller_link is not None or seller_name)
+            else None
+        ),
+        "bestsellerRanks": ranks,
+        "isAmazonChoice": _one(doc, "#acBadge_feature_div, .ac-badge-wrapper")
+        is not None,
+        "amazonChoiceText": _first_text(doc, "#acBadge_feature_div, .ac-badge-wrapper"),
+        "reviewsLink": _absolute(domain, reviews_link),
+        "hasReviews": bool(local_reviews or foreign_reviews),
+        "productPageReviews": local_reviews,
+        "productPageReviewsFromOtherCountries": foreign_reviews,
+    }
+
+
+def parse_search_page(
+    html: str, *, page: int = 1, domain: str = "www.amazon.com"
+) -> list[dict[str, Any]]:
+    """Parse product cards from a search or category page."""
+    doc = Adaptor(html)
+    cards: list[dict[str, Any]] = []
+    for position, node in enumerate(
+        doc.css(
+            "[data-component-type='s-search-result'][data-asin], [data-asin].s-result-item"
+        ),
+        start=1,
+    ):
+        asin = (_attr(node, "data-asin") or "").upper()
+        if not _ASIN_RE.match(asin):
+            continue
+        href = _first_attr(node, "href", "h2 a", "a.a-link-normal")
+        title = _first_text(node, "h2", "h2 span", ".a-size-base-plus")
+        cards.append(
+            {
+                "asin": asin,
+                "originalAsin": asin,
+                "title": title,
+                "url": _absolute(domain, href) or f"https://{domain}/dp/{asin}",
+                "price": _price(_first_text(node, ".a-price .a-offscreen")),
+                "stars": _float(_first_text(node, ".a-icon-alt")),
+                "reviewsCount": _integer(
+                    _first_text(node, "[aria-label$='ratings']", ".s-underline-text")
+                ),
+                "thumbnailImage": _first_attr(node, "src", ".s-image"),
+                "categoryPageData": {
+                    "position": position,
+                    "page": page,
+                    "isSponsored": "sponsored" in (_text(node) or "").lower(),
+                    "isBestSeller": _one(node, ".a-badge-text, .s-badge-text")
+                    is not None,
+                },
+            }
+        )
+    return cards
+
+
+def parse_aod_offers(
+    html: str, *, domain: str = "www.amazon.com"
+) -> list[dict[str, Any]]:
+    """Parse rows from the public all-offers panel."""
+    doc = Adaptor(html)
+    offers: list[dict[str, Any]] = []
+    for position, node in enumerate(
+        doc.css("#aod-offer, .aod-information-block"), start=1
+    ):
+        seller_link = _one(
+            node, "#aod-offer-soldBy a[href*='seller='], a[href*='/sp?']"
+        )
+        href = _attr(seller_link, "href")
+        query = parse_qs(urlparse(href or "").query)
+        seller_id = (query.get("seller") or [None])[0]
+        offers.append(
+            {
+                "position": position,
+                "price": _price(_first_text(node, ".a-price .a-offscreen")),
+                "condition": _first_text(
+                    node, "#aod-offer-heading", ".aod-information-block"
+                ),
+                "delivery": _first_text(node, "#mir-layout-DELIVERY_BLOCK"),
+                "seller": {
+                    "id": seller_id,
+                    "name": _text(seller_link)
+                    or _first_text(node, "#aod-offer-soldBy .a-color-base"),
+                    "url": _absolute(domain, href),
+                },
+                "isPinnedOffer": _one(node, "#aod-pinned-offer, .aod-pinned-offer")
+                is not None,
+            }
+        )
+    return offers
+
+
+def parse_seller(
+    html: str, *, seller_id: str | None = None, domain: str = "www.amazon.com"
+) -> dict[str, Any]:
+    """Parse the public summary of a seller profile."""
+    doc = Adaptor(html)
+    return {
+        "id": seller_id,
+        "name": _first_text(doc, "#sellerName", "#seller-profile-container h1", "h1"),
+        "url": f"https://{domain}/sp?seller={seller_id}" if seller_id else None,
+        "reviewsCount": _integer(
+            _first_text(doc, "#seller-feedback-summary", "#feedback-summary-table")
+        ),
+        "averageRating": _float(
+            _first_text(doc, "#feedback-summary-table .a-icon-alt", ".a-icon-star")
+        ),
+    }
+
+
+def parse_bestsellers_page(
+    html: str, *, page: int = 1, domain: str = "www.amazon.com"
+) -> list[dict[str, Any]]:
+    """Parse ranked products from a best-sellers page."""
+    doc = Adaptor(html)
+    items: list[dict[str, Any]] = []
+    nodes = doc.css("#gridItemRoot, .zg-grid-general-faceout, [id^='p13n-asin-index-']")
+    for fallback_rank, node in enumerate(nodes, start=1):
+        href = _first_attr(node, "href", "a[href*='/dp/']", "a[href*='/gp/product/']")
+        asin = _asin_from_href(href) or (_attr(node, "data-asin") or None)
+        if not asin or not _ASIN_RE.match(asin):
+            continue
+        rank = _integer(_first_text(node, ".zg-bdg-text")) or fallback_rank
+        items.append(
+            {
+                "asin": asin,
+                "originalAsin": asin,
+                "title": _first_text(
+                    node, "._cDEzb_p13n-sc-css-line-clamp-3_g3dy1", "img"
+                ),
+                "url": _absolute(domain, href) or f"https://{domain}/dp/{asin}",
+                "price": _price(_first_text(node, ".a-price .a-offscreen")),
+                "stars": _float(_first_text(node, ".a-icon-alt")),
+                "thumbnailImage": _first_attr(node, "src", "img"),
+                "bestsellerPageData": {"rank": rank, "page": page},
+            }
+        )
+    return items

surfsense_backend/app/proprietary/platforms/amazon/schemas.py

@@ -0,0 +1,245 @@
+# ruff: noqa: N815
+"""Input/output models for the Amazon Product Scraper.
+
+The skeleton accepts the full input surface; output fields the implementation
+does not source yet are emitted as ``None``/``[]`` so parity is additive —
+exactly like the YouTube / Maps / Google Search models.
+
+Outputs use ``extra="allow"`` on purpose: it lets the output shape grow without
+breaking existing consumers. Only a small set of stable, always-populated nested
+objects (``price``, ``starsBreakdown``, ``seller``, ``visitStoreLink``) are
+typed; the sprawling, layout-volatile sections (A+ content, brand story,
+attribute tables, offers, on-page reviews, ...) stay loose ``dict``/``list`` and
+lean on ``extra="allow"`` rather than pinning a shape the parsers haven't
+verified against live HTML yet.
+
+Public, anonymous data only: there is no auth/login field on the input surface.
+Deep review pagination is login-gated on today's Amazon and is out of scope; the
+on-page ``productPageReviews`` are the only reviews modeled here.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+# --------------------------------------------------------------------------- #
+# Typed nested objects (the stable, always-populated ones).                   #
+# --------------------------------------------------------------------------- #
+
+
+class Price(BaseModel):
+    """A monetary amount as Amazon renders it (``price``, ``listPrice``,
+    ``shippingPrice``, and the ``min``/``max`` of ``priceRange``)."""
+
+    model_config = ConfigDict(extra="allow")
+
+    value: float | None = None
+    currency: str | None = None
+
+
+class StarsBreakdown(BaseModel):
+    """Per-star rating distribution (fractions summing to ~1.0).
+
+    Amazon's keys start with a digit (``5star`` ... ``1star``), which is not a
+    valid Python identifier, so each field is aliased. ``populate_by_name`` lets
+    tests construct it either way; ``to_output`` serializes ``by_alias`` so the
+    wire shape retains the digit-prefixed keys.
+    """
+
+    model_config = ConfigDict(extra="allow", populate_by_name=True)
+
+    five_star: float | None = Field(default=None, alias="5star")
+    four_star: float | None = Field(default=None, alias="4star")
+    three_star: float | None = Field(default=None, alias="3star")
+    two_star: float | None = Field(default=None, alias="2star")
+    one_star: float | None = Field(default=None, alias="1star")
+
+
+class Seller(BaseModel):
+    """The featured-offer seller stamped on the product item."""
+
+    model_config = ConfigDict(extra="allow")
+
+    id: str | None = None
+    name: str | None = None
+    url: str | None = None
+    reviewsCount: int | None = None
+    averageRating: float | None = None
+
+
+class VisitStoreLink(BaseModel):
+    """The brand-store link (``visitStoreLink``)."""
+
+    model_config = ConfigDict(extra="allow")
+
+    text: str | None = None
+    url: str | None = None
+
+
+# --------------------------------------------------------------------------- #
+# Input surface.                                                              #
+# --------------------------------------------------------------------------- #
+
+
+class AmazonScrapeInput(BaseModel):
+    """Full input surface for the Amazon Product Scraper.
+
+    ``categoryOrProductUrls`` mixes category/search, product, bestsellers, and
+    shortened URLs; ``proxyCountry`` defaults to ``AUTO_SELECT_PROXY_COUNTRY``
+    (derived from each URL's domain); ``scrapeProductDetails`` defaults on (deep
+    scrape). ``extra="allow"`` keeps a verbatim payload valid even for add-ons
+    this scraper does not model.
+    """
+
+    model_config = ConfigDict(extra="allow")
+
+    # Discovery (required)
+    categoryOrProductUrls: list[dict] = Field(min_length=1)
+
+    # Result caps
+    maxItemsPerStartUrl: int | None = Field(default=None, ge=0)
+    maxSearchPagesPerStartUrl: int = Field(default=9999, ge=1)
+    maxProductVariantsAsSeparateResults: int = Field(default=0, ge=0)
+    maxOffers: int = Field(default=0, ge=0)
+
+    # Localization
+    language: str | None = None
+    proxyCountry: str = "AUTO_SELECT_PROXY_COUNTRY"
+    countryCode: str | None = None
+    zipCode: str | None = None
+    locationDeliverableRoutes: list[str] | None = Field(
+        default_factory=lambda: ["PRODUCT", "SEARCH", "OFFERS"]
+    )
+
+    # Feature toggles
+    scrapeSellers: bool = False
+    useCaptchaSolver: bool = False
+    scrapeProductVariantPrices: bool = False
+    scrapeProductDetails: bool | None = True
+
+
+# --------------------------------------------------------------------------- #
+# Output item.                                                                #
+# --------------------------------------------------------------------------- #
+
+
+class ProductItem(BaseModel):
+    """Amazon Product Scraper output item (one per product).
+
+    Unsourced fields default to ``None``/``[]``; ``extra="allow"`` keeps the
+    contract open so later milestones add fields without breaking consumers.
+    """
+
+    model_config = ConfigDict(extra="allow", populate_by_name=True)
+
+    # Identity / core
+    title: str | None = None
+    url: str | None = None
+    asin: str | None = None
+    originalAsin: str | None = None
+    brand: str | None = None
+    author: str | None = None
+    price: Price | None = None
+    listPrice: Price | None = None
+    shippingPrice: Price | None = None
+    inStock: bool | None = None
+    inStockText: str | None = None
+    delivery: str | None = None
+    fastestDelivery: str | None = None
+    condition: str | None = None
+
+    # Ratings / social
+    stars: float | None = None
+    starsBreakdown: StarsBreakdown | None = None
+    reviewsCount: int | None = None
+    answeredQuestions: int | None = None
+    aiReviewsSummary: dict | None = None
+    monthlyPurchaseVolume: str | None = None
+
+    # Content
+    breadCrumbs: str | None = None
+    description: str | None = None
+    features: list[str] = Field(default_factory=list)
+    sustainabilityFeatures: list[dict] = Field(default_factory=list)
+    videosCount: int | None = None
+    visitStoreLink: VisitStoreLink | None = None
+    thumbnailImage: str | None = None
+    galleryThumbnails: list[str] = Field(default_factory=list)
+    highResolutionImages: list[str] = Field(default_factory=list)
+    importantInformation: dict | None = None
+    bookDescription: str | None = None
+    aPlusContent: dict | None = None
+    brandStory: dict | None = None
+    productComparison: dict | None = None
+
+    # Commerce
+    returnPolicy: str | None = None
+    support: str | None = None
+    priceRange: dict | None = None
+    variantAsins: list[str] = Field(default_factory=list)
+    variantDetails: list[dict] = Field(default_factory=list)
+    variantAttributes: list[dict] = Field(default_factory=list)
+    attributes: list[dict] = Field(default_factory=list)
+    attributesMapped: dict | None = None
+    productOverview: list[dict] = Field(default_factory=list)
+    manufacturerAttributes: list[dict] = Field(default_factory=list)
+    seller: Seller | None = None
+    bestsellerRanks: list[dict] = Field(default_factory=list)
+    isAmazonChoice: bool | None = None
+    amazonChoiceText: str | None = None
+    offers: list[dict] = Field(default_factory=list)
+
+    # Reviews (public, on-page only)
+    reviewsLink: str | None = None
+    hasReviews: bool | None = None
+    productPageReviews: list[dict] = Field(default_factory=list)
+    productPageReviewsFromOtherCountries: list[dict] = Field(default_factory=list)
+
+    # Provenance / meta
+    locationText: str | None = None
+    unNormalizedProductUrl: str | None = None
+    loadedCountryCode: str | None = None
+    categoryPageData: dict | None = None
+    bestsellerPageData: dict | None = None
+    input: str | None = None
+
+    def to_output(self) -> dict[str, Any]:
+        """Serialize to the flat dict output shape (keeps extras, aliases).
+
+        ``by_alias`` restores digit-prefixed keys like ``starsBreakdown.5star``;
+        ``exclude_none=False`` keeps unsourced keys present so consumers never
+        break on a missing field.
+        """
+        return self.model_dump(by_alias=True, exclude_none=False)
+
+
+# --------------------------------------------------------------------------- #
+# Error item (the failure model — pushed into the stream, not raised).        #
+# --------------------------------------------------------------------------- #
+
+ErrorCode = Literal[
+    "invalid_url",
+    "invalid_input",
+    "product_not_found",
+    "shortened_url_invalid",
+    "bestsellers_category_not_found",
+    "no_results_found",
+]
+
+
+class ErrorItem(BaseModel):
+    """A per-input failure emitted into the dataset instead of a normal item.
+
+    Consumers tell error items apart from products by the presence of ``error``.
+    ``invalid_input`` additionally terminates the run (enforced in later
+    milestones); all other codes are per-input and non-fatal.
+    """
+
+    model_config = ConfigDict(extra="allow")
+
+    error: ErrorCode
+    errorDescription: str | None = None
+    input: str | None = None
+    url: str | None = None

surfsense_backend/app/proprietary/platforms/amazon/scraper.py

@@ -0,0 +1,541 @@
+"""Orchestrate public Amazon product discovery and enrichment.
+
+The streaming core dispatches each start URL to a product, search, best-sellers,
+or shortened-link flow. Network and parsing concerns remain isolated in their
+own modules so markup changes and retry policy can be tested independently.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import AsyncIterator
+from typing import Any
+from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
+
+from .fetch import (
+    fetch_aod_html,
+    fetch_page,
+    fetch_seller_html,
+    gather_bounded,
+    get_location_session,
+    resolve_shortlink,
+    should_localize,
+)
+from .locale import accept_language_for, proxy_country_for
+from .parsers import (
+    parse_aod_offers,
+    parse_bestsellers_page,
+    parse_product,
+    parse_search_page,
+    parse_seller,
+)
+from .schemas import AmazonScrapeInput, ErrorItem, ProductItem
+from .url_resolver import ResolvedUrl, resolve_url
+
+__all__ = ["iter_products", "scrape_products"]
+
+_DETAIL_CONCURRENCY = 8
+_SEARCH_PAGE_LIMIT = 7
+_DEFAULT_ITEMS_PER_START_URL = 100
+
+
+def _error(
+    code: str, description: str, *, input_url: str | None, url: str | None = None
+) -> dict[str, Any]:
+    return ErrorItem(
+        error=code,
+        errorDescription=description,
+        input=input_url,
+        url=url or input_url,
+    ).model_dump()
+
+
+def _buybox_offer(fields: dict[str, Any]) -> dict[str, Any] | None:
+    """Synthesize a single offer from the PDP buy box.
+
+    Amazon serves the All-Offers-Display panel as a JS-only modal for many
+    (especially US) ASINs, so the AOD ajax endpoint 404s. Fall back to the
+    featured buy-box winner already parsed from the product page.
+    """
+    price = fields.get("price")
+    seller = fields.get("seller")
+    seller = seller if isinstance(seller, dict) else None
+    if not price and not (seller and seller.get("name")):
+        return None
+    return {
+        "position": 1,
+        "price": price,
+        "condition": fields.get("condition") or "New",
+        "delivery": fields.get("delivery"),
+        "seller": seller,
+        "isPinnedOffer": True,
+    }
+
+
+def _page_url(url: str, page: int) -> str:
+    parsed = urlparse(url)
+    query = dict(parse_qsl(parsed.query, keep_blank_values=True))
+    query["page"] = str(page)
+    return urlunparse(parsed._replace(query=urlencode(query)))
+
+
+async def _location_context(
+    resolved: ResolvedUrl, input_model: AmazonScrapeInput, route: str
+) -> tuple[dict[str, str] | None, str | None, str | None, str | None]:
+    if (
+        not input_model.zipCode
+        or not resolved.domain
+        or not should_localize(route, input_model.locationDeliverableRoutes)
+    ):
+        return None, None, None, None
+    session = await get_location_session(
+        resolved.domain,
+        zip_code=input_model.zipCode,
+        country_code=input_model.countryCode,
+        country=proxy_country_for(resolved.marketplace),
+        accept_language=accept_language_for(resolved.marketplace),
+    )
+    if session is None:
+        return None, None, None, None
+    return (
+        session.cookies,
+        session.proxy,
+        session.location_text,
+        session.country_code,
+    )
+
+
+async def _product_flow(
+    resolved: ResolvedUrl,
+    input_model: AmazonScrapeInput,
+    *,
+    provenance: dict[str, Any] | None = None,
+    allow_variants: bool = True,
+) -> AsyncIterator[dict[str, Any]]:
+    """Fetch and enrich one product detail page."""
+    asin = resolved.asin
+    if not asin or not resolved.domain:
+        yield _error(
+            "product_not_found",
+            "The product URL did not contain a valid ASIN.",
+            input_url=resolved.url,
+        )
+        return
+
+    cookies, proxy, location_text, loaded_country = await _location_context(
+        resolved, input_model, "PRODUCT"
+    )
+    country = proxy_country_for(resolved.marketplace)
+    accept_language = accept_language_for(resolved.marketplace)
+    response = await fetch_page(
+        resolved.url,
+        cookies=cookies,
+        proxy=proxy,
+        country=country,
+        accept_language=accept_language,
+    )
+    if response is None:
+        yield _error(
+            "product_not_found",
+            "The product page could not be loaded after retrying available proxy exits.",
+            input_url=resolved.url,
+        )
+        return
+    if response.status in {404, 410}:
+        yield _error(
+            "product_not_found",
+            "The product page was not found.",
+            input_url=resolved.url,
+        )
+        return
+    if response.status != 200:
+        yield _error(
+            "product_not_found",
+            f"The product page returned HTTP {response.status}.",
+            input_url=resolved.url,
+        )
+        return
+
+    fields = await asyncio.to_thread(
+        parse_product,
+        response.html,
+        asin=asin,
+        url=response.url,
+        domain=resolved.domain,
+    )
+    if not fields.get("title"):
+        yield _error(
+            "product_not_found",
+            "The response did not contain a recognizable product.",
+            input_url=resolved.url,
+            url=response.url,
+        )
+        return
+    fields.update(provenance or {})
+    fields["input"] = resolved.url
+    fields["unNormalizedProductUrl"] = resolved.url
+    fields["locationText"] = location_text
+    fields["loadedCountryCode"] = loaded_country
+
+    if input_model.maxOffers > 0:
+        offer_cookies, offer_proxy, _, _ = await _location_context(
+            resolved, input_model, "OFFERS"
+        )
+        offers_html = await fetch_aod_html(
+            asin,
+            resolved.domain,
+            cookies=offer_cookies or cookies,
+            proxy=offer_proxy or proxy,
+            country=country,
+            accept_language=accept_language,
+        )
+        offers = (
+            parse_aod_offers(offers_html, domain=resolved.domain) if offers_html else []
+        )
+        if not offers:
+            buybox = _buybox_offer(fields)
+            offers = [buybox] if buybox else []
+        if offers:
+            fields["offers"] = offers[: input_model.maxOffers]
+
+    if input_model.scrapeSellers:
+        seller_ids: list[str] = []
+        featured = fields.get("seller")
+        if isinstance(featured, dict) and featured.get("id"):
+            seller_ids.append(featured["id"])
+        for offer in fields.get("offers") or []:
+            seller = offer.get("seller") if isinstance(offer, dict) else None
+            if isinstance(seller, dict) and seller.get("id"):
+                seller_ids.append(seller["id"])
+        seller_ids = list(dict.fromkeys(seller_ids))
+
+        async def load_seller(seller_id: str) -> dict[str, Any] | None:
+            html = await fetch_seller_html(
+                seller_id,
+                resolved.domain,
+                cookies=cookies,
+                proxy=proxy,
+                country=country,
+                accept_language=accept_language,
+            )
+            return (
+                parse_seller(html, seller_id=seller_id, domain=resolved.domain)
+                if html
+                else None
+            )
+
+        sellers = await gather_bounded(
+            [
+                lambda seller_id=seller_id: load_seller(seller_id)
+                for seller_id in seller_ids
+            ],
+            concurrency=_DETAIL_CONCURRENCY,
+        )
+        by_id = {
+            seller["id"]: seller for seller in sellers if seller and seller.get("id")
+        }
+        if featured and featured.get("id") in by_id:
+            fields["seller"] = by_id[featured["id"]]
+        for offer in fields.get("offers") or []:
+            seller = offer.get("seller")
+            if isinstance(seller, dict) and seller.get("id") in by_id:
+                offer["seller"] = by_id[seller["id"]]
+
+    variant_asins = [
+        value for value in fields.get("variantAsins") or [] if value != asin
+    ]
+    variant_cap = (
+        min(len(variant_asins), input_model.maxProductVariantsAsSeparateResults)
+        if allow_variants
+        else 0
+    )
+    price_cap = (
+        len(variant_asins)
+        if allow_variants and input_model.scrapeProductVariantPrices
+        else 0
+    )
+    fetch_count = max(variant_cap, price_cap)
+    variant_items: list[dict[str, Any]] = []
+    if fetch_count:
+
+        async def load_variant(variant_asin: str) -> list[dict[str, Any]]:
+            variant_url = f"https://{resolved.domain}/dp/{variant_asin}"
+            variant_resolved = resolve_url(variant_url)
+            if variant_resolved is None:
+                return []
+            return [
+                item
+                async for item in _product_flow(
+                    variant_resolved, input_model, allow_variants=False
+                )
+            ]
+
+        batches = await gather_bounded(
+            [
+                lambda variant_asin=variant_asin: load_variant(variant_asin)
+                for variant_asin in variant_asins[:fetch_count]
+            ],
+            concurrency=_DETAIL_CONCURRENCY,
+        )
+        variant_items = [
+            item for batch in batches for item in batch if "error" not in item
+        ]
+        if input_model.scrapeProductVariantPrices:
+            prices = {
+                item.get("asin"): item.get("price")
+                for item in variant_items
+                if item.get("asin")
+            }
+            details = fields.get("variantDetails") or [
+                {"asin": variant_asin} for variant_asin in variant_asins
+            ]
+            for detail in details:
+                if detail.get("asin") in prices:
+                    detail["price"] = prices[detail["asin"]]
+            fields["variantDetails"] = details
+
+    yield ProductItem(**fields).to_output()
+    for item in variant_items[:variant_cap]:
+        item["originalAsin"] = asin
+        yield item
+
+
+async def _search_flow(
+    resolved: ResolvedUrl, input_model: AmazonScrapeInput
+) -> AsyncIterator[dict[str, Any]]:
+    """Page through search results and optionally fetch product details."""
+    if not resolved.domain:
+        yield _error(
+            "no_results_found", "The search domain was invalid.", input_url=resolved.url
+        )
+        return
+    cap = (
+        input_model.maxItemsPerStartUrl
+        if input_model.maxItemsPerStartUrl is not None
+        else _DEFAULT_ITEMS_PER_START_URL
+    )
+    max_pages = min(input_model.maxSearchPagesPerStartUrl, _SEARCH_PAGE_LIMIT)
+    cookies, proxy, _, _ = await _location_context(resolved, input_model, "SEARCH")
+    country = proxy_country_for(resolved.marketplace)
+    accept_language = accept_language_for(resolved.marketplace)
+    seen: set[str] = set()
+    emitted = 0
+    for page in range(1, max_pages + 1):
+        html_response = await fetch_page(
+            _page_url(resolved.url, page),
+            cookies=cookies,
+            proxy=proxy,
+            country=country,
+            accept_language=accept_language,
+        )
+        cards = (
+            await asyncio.to_thread(
+                parse_search_page,
+                html_response.html,
+                page=page,
+                domain=resolved.domain,
+            )
+            if html_response is not None and html_response.status == 200
+            else []
+        )
+        cards = [
+            card for card in cards if card.get("asin") and card["asin"] not in seen
+        ]
+        for card in cards:
+            seen.add(card["asin"])
+        if not cards:
+            if page == 1:
+                yield _error(
+                    "no_results_found",
+                    "The search page did not contain any products.",
+                    input_url=resolved.url,
+                )
+            return
+        cards = cards[: max(0, cap - emitted)]
+        if input_model.scrapeProductDetails is False:
+            for card in cards:
+                card["input"] = resolved.url
+                yield ProductItem(**card).to_output()
+                emitted += 1
+        else:
+
+            async def load_card(card: dict[str, Any]) -> list[dict[str, Any]]:
+                product = resolve_url(card["url"])
+                if product is None:
+                    return []
+                provenance = {"categoryPageData": card["categoryPageData"]}
+                return [
+                    item
+                    async for item in _product_flow(
+                        product, input_model, provenance=provenance
+                    )
+                ]
+
+            batches = await gather_bounded(
+                [lambda card=card: load_card(card) for card in cards],
+                concurrency=_DETAIL_CONCURRENCY,
+            )
+            for batch in batches:
+                for item in batch:
+                    if "error" not in item and emitted >= cap:
+                        return
+                    yield item
+                    if "error" not in item:
+                        emitted += 1
+        if emitted >= cap:
+            return
+
+
+async def _bestsellers_flow(
+    resolved: ResolvedUrl, input_model: AmazonScrapeInput
+) -> AsyncIterator[dict[str, Any]]:
+    """Page through a best-sellers category and emit ranked products."""
+    if not resolved.domain:
+        yield _error(
+            "bestsellers_category_not_found",
+            "The best-sellers domain was invalid.",
+            input_url=resolved.url,
+        )
+        return
+    cap = (
+        input_model.maxItemsPerStartUrl
+        if input_model.maxItemsPerStartUrl is not None
+        else _DEFAULT_ITEMS_PER_START_URL
+    )
+    country = proxy_country_for(resolved.marketplace)
+    accept_language = accept_language_for(resolved.marketplace)
+    seen: set[str] = set()
+    emitted = 0
+    for page in range(1, min(input_model.maxSearchPagesPerStartUrl, 2) + 1):
+        response = await fetch_page(
+            _page_url(resolved.url, page),
+            country=country,
+            accept_language=accept_language,
+        )
+        cards = (
+            await asyncio.to_thread(
+                parse_bestsellers_page,
+                response.html,
+                page=page,
+                domain=resolved.domain,
+            )
+            if response is not None and response.status == 200
+            else []
+        )
+        cards = [
+            card for card in cards if card.get("asin") and card["asin"] not in seen
+        ]
+        for card in cards:
+            seen.add(card["asin"])
+        if not cards:
+            if page == 1:
+                yield _error(
+                    "bestsellers_category_not_found",
+                    "The page did not contain a recognizable best-sellers category.",
+                    input_url=resolved.url,
+                )
+            return
+        cards = cards[: max(0, cap - emitted)]
+        if input_model.scrapeProductDetails is False:
+            for card in cards:
+                card["input"] = resolved.url
+                yield ProductItem(**card).to_output()
+                emitted += 1
+        else:
+
+            async def load_card(card: dict[str, Any]) -> list[dict[str, Any]]:
+                product = resolve_url(card["url"])
+                if product is None:
+                    return []
+                return [
+                    item
+                    async for item in _product_flow(
+                        product,
+                        input_model,
+                        provenance={"bestsellerPageData": card["bestsellerPageData"]},
+                    )
+                ]
+
+            for batch in await gather_bounded(
+                [lambda card=card: load_card(card) for card in cards],
+                concurrency=_DETAIL_CONCURRENCY,
+            ):
+                for item in batch:
+                    if "error" not in item and emitted >= cap:
+                        return
+                    yield item
+                    if "error" not in item:
+                        emitted += 1
+        if emitted >= cap:
+            return
+
+
+async def _shortened_flow(
+    resolved: ResolvedUrl, input_model: AmazonScrapeInput
+) -> AsyncIterator[dict[str, Any]]:
+    """Resolve a shortened link and dispatch its final Amazon URL."""
+    target = await resolve_shortlink(resolved.url)
+    final = resolve_url(target) if target else None
+    if final is None or final.kind == "shortened":
+        yield _error(
+            "shortened_url_invalid",
+            "The shortened link did not resolve to a supported Amazon page.",
+            input_url=resolved.url,
+            url=target,
+        )
+        return
+    async for item in _FLOWS[final.kind](final, input_model):
+        if "input" in item:
+            item["input"] = resolved.url
+        yield item
+
+
+_FLOWS = {
+    "product": _product_flow,
+    "search": _search_flow,
+    "bestsellers": _bestsellers_flow,
+    "shortened": _shortened_flow,
+}
+
+
+async def iter_products(
+    input_model: AmazonScrapeInput,
+) -> AsyncIterator[dict[str, Any]]:
+    """Yield product items for every start URL.
+
+    Each ``categoryOrProductUrls`` entry is classified and dispatched to its
+    per-kind flow. A URL that cannot be recognized as an Amazon product /
+    search / bestsellers / shortened link yields an ``invalid_url`` error item
+    (the failure model is in-stream error items, not exceptions).
+    """
+    for entry in input_model.categoryOrProductUrls:
+        url = entry.get("url") if isinstance(entry, dict) else None
+        resolved = resolve_url(url) if url else None
+        if resolved is None:
+            yield _error(
+                "invalid_url",
+                "Start URL was malformed or not a recognized Amazon URL.",
+                input_url=url,
+            )
+            continue
+        async for item in _FLOWS[resolved.kind](resolved, input_model):
+            yield item
+
+
+async def scrape_products(
+    input_model: AmazonScrapeInput, *, limit: int | None = None
+) -> list[dict[str, Any]]:
+    """Collect :func:`iter_products` into a list, honoring an optional ``limit``.
+
+    ``limit`` is a request-time policy guard (used by the route/capability), NOT
+    a ceiling in the streaming core.
+    """
+    from app.capabilities.core.progress import emit_progress
+
+    results: list[dict[str, Any]] = []
+    async for item in iter_products(input_model):
+        results.append(item)
+        emit_progress("scraping", current=len(results), total=limit, unit="product")
+        if limit is not None and len(results) >= limit:
+            break
+    return results

surfsense_backend/app/proprietary/platforms/amazon/url_resolver.py

@@ -0,0 +1,98 @@
+"""Classify an Amazon start URL and extract its identifiers.
+
+Covers the ``categoryOrProductUrls`` shapes accepted on input: product
+pages (``/dp/ASIN``, ``/gp/product/ASIN``), search / category pages (``/s?k=``,
+``/s?bbn=``, ``/s?rh=``), bestsellers pages (``/zgbs/`` / ``/gp/bestsellers/``),
+and shortened links (``a.co`` / ``amzn.to`` / ``amzn.eu``, which need a network
+redirect to resolve — classified here, resolved later in the fetch layer).
+
+Pure, no I/O. The ``marketplace`` suffix (``com``, ``de``, ``co.uk``, ...) is
+derived from the host so the fetch layer can default the proxy country and UI
+language without a separate geocoding step.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Literal
+from urllib.parse import parse_qs, urlparse
+
+ResolvedKind = Literal["product", "search", "bestsellers", "shortened"]
+
+_SHORTLINK_HOSTS = ("a.co", "amzn.to", "amzn.eu")
+
+# ASIN: a 10-char uppercase alphanumeric id, carried in the canonical product
+# path forms (``/dp/``, ``/gp/product/``, ``/product/``) or a ``?asin=`` query.
+_ASIN_PATH_RE = re.compile(r"/(?:dp|gp/product|product|d)/([A-Z0-9]{10})(?:[/?]|$)")
+_ASIN_QUERY_RE = re.compile(r"[?&]asin=([A-Z0-9]{10})", re.IGNORECASE)
+
+# Search/category pages carry their query in these params even without a /s path.
+_SEARCH_QUERY_KEYS = ("k", "bbn", "rh")
+
+
+@dataclass(frozen=True)
+class ResolvedUrl:
+    kind: ResolvedKind
+    url: str
+    asin: str | None = None
+    domain: str | None = None
+    marketplace: str | None = None  # TLD suffix, e.g. "com", "de", "co.uk"
+
+
+def extract_asin(url: str) -> str | None:
+    """Pull the 10-char ASIN out of an Amazon product URL, if present."""
+    match = _ASIN_PATH_RE.search(url)
+    if match:
+        return match.group(1)
+    match = _ASIN_QUERY_RE.search(url)
+    return match.group(1).upper() if match else None
+
+
+def _marketplace_from_host(host: str) -> str | None:
+    """The Amazon TLD suffix (``com``, ``de``, ``co.uk``, ...), or ``None``.
+
+    Strips common subdomain prefixes and the ``amazon`` label, returning the
+    remainder (e.g. ``www.amazon.co.jp`` -> ``co.jp``).
+    """
+    host = host.lower()
+    for prefix in ("www.", "smile."):
+        if host.startswith(prefix):
+            host = host[len(prefix) :]
+    if not host.startswith("amazon."):
+        return None
+    return host[len("amazon.") :] or None
+
+
+def resolve_url(url: str) -> ResolvedUrl | None:
+    """Classify an Amazon start URL into a scrape job, or ``None`` if unrecognized."""
+    parsed = urlparse(url)
+    host = (parsed.hostname or "").lower()
+    path = parsed.path or ""
+
+    # Shortened links resolve to their real URL later, in the fetch layer.
+    if host in _SHORTLINK_HOSTS:
+        return ResolvedUrl("shortened", url)
+
+    if "amazon." not in host:
+        return None
+
+    marketplace = _marketplace_from_host(host)
+
+    # Bestsellers before search: a /zgbs page has no /s path but is its own kind.
+    if "/zgbs/" in path or path.startswith("/gp/bestsellers"):
+        return ResolvedUrl("bestsellers", url, domain=host, marketplace=marketplace)
+
+    # Product: canonical /dp/ (etc.) path with an ASIN.
+    asin = extract_asin(url)
+    if asin is not None:
+        return ResolvedUrl(
+            "product", url, asin=asin, domain=host, marketplace=marketplace
+        )
+
+    # Search / category: the /s path, or any search-carrying query param.
+    query = parse_qs(parsed.query)
+    if path.startswith("/s") or any(k in query for k in _SEARCH_QUERY_KEYS):
+        return ResolvedUrl("search", url, domain=host, marketplace=marketplace)
+
+    return None

surfsense_backend/app/routes/__init__.py

@@ -1,6 +1,7 @@
 from fastapi import APIRouter, Depends
 
 # Import verb namespaces for their registration side effects before the door builds.
+import app.capabilities.amazon
 import app.capabilities.google_maps
 import app.capabilities.google_search
 import app.capabilities.instagram

surfsense_backend/app/utils/proxy/__init__.py

@@ -15,6 +15,16 @@ def get_proxy_url() -> str | None:
     return get_active_provider().get_proxy_url()
 
 
+def get_geo_proxy_url(country: str | None = None) -> str | None:
+    """Proxy URL pinned to an exit country when the provider supports it."""
+    return get_active_provider().get_geo_proxy_url(country)
+
+
+def get_sticky_proxy_url(session_id: str, country: str | None = None) -> str | None:
+    """Proxy URL pinned to a stable vendor session when supported."""
+    return get_active_provider().get_sticky_proxy_url(session_id, country)
+
+
 def get_playwright_proxy() -> dict[str, str] | None:
     """Playwright-style proxy dict, or ``None`` when not configured."""
     return get_active_provider().get_playwright_proxy()
@@ -41,9 +51,11 @@ def get_residential_proxy_url() -> str | None:
 __all__ = [
     "ProxyProvider",
     "get_active_provider",
+    "get_geo_proxy_url",
     "get_playwright_proxy",
     "get_proxy_url",
     "get_requests_proxies",
     "get_residential_proxy_url",
+    "get_sticky_proxy_url",
     "is_pool_backed",
 ]

surfsense_backend/app/utils/proxy/base.py

@@ -67,6 +67,24 @@ class ProxyProvider(ABC):
             return None
         return {"http": proxy_url, "https": proxy_url}
 
+    def get_geo_proxy_url(self, country: str | None = None) -> str | None:
+        """Return a proxy URL pinned to ``country`` when supported.
+
+        Providers without vendor-specific country routing safely fall back to
+        their ordinary proxy URL.
+        """
+        return self.get_proxy_url()
+
+    def get_sticky_proxy_url(
+        self, session_id: str, country: str | None = None
+    ) -> str | None:
+        """Return a proxy URL pinned to ``session_id`` when supported.
+
+        Providers without vendor-specific session routing safely fall back to
+        their ordinary proxy URL.
+        """
+        return self.get_geo_proxy_url(country)
+
     def get_location(self) -> str:
         """Return the proxy's configured exit region (e.g. ``"us"``), or ``""``.
 

surfsense_backend/app/utils/proxy/providers/dataimpulse.py

@@ -16,15 +16,13 @@ Example URL::
 
     http://<token>__cr.us:<password>@gw.dataimpulse.com:823
 
-ponytail: sticky sessions (a stable exit IP across requests) are another
-username suffix (``__sid.<id>``) — the lever the Reddit scraper's README flags as
-a TODO for its ``loid``-per-IP flow. Not built yet: Reddit isn't wired to a
-route, so there's no caller to thread a session id through. Add a
-``get_sticky_proxy_url(session_id)`` here (rewriting the username) when it lands.
+Sticky sessions use the vendor's ``__sid.<id>`` username suffix, allowing a
+caller to keep cookies and requests on the same exit IP.
 """
 
 import logging
-from urllib.parse import urlsplit
+import re
+from urllib.parse import quote, urlsplit, urlunsplit
 
 from app.config import Config
 from app.utils.proxy.base import ProxyProvider
@@ -34,6 +32,22 @@ logger = logging.getLogger(__name__)
 # DataImpulse encodes country routing as a "__cr.<country>" username suffix; the
 # country token runs until the next "__" param (e.g. "__sid") or the end.
 _COUNTRY_MARKER = "__cr."
+_COUNTRY_RE = re.compile(r"__cr\.[A-Za-z]{2,}")
+_SESSION_RE = re.compile(r"__sid\.[A-Za-z0-9_-]+")
+
+
+def _safe_session_id(session_id: str) -> str:
+    safe_id = re.sub(r"[^A-Za-z0-9_-]", "-", session_id).strip("-")
+    if not safe_id:
+        raise ValueError("session_id must contain at least one letter or digit")
+    return safe_id
+
+
+def _safe_country(country: str | None) -> str | None:
+    if country is None:
+        return None
+    safe_country = re.sub(r"[^a-z]", "", country.lower())
+    return safe_country or None
 
 
 class DataImpulseProvider(ProxyProvider):
@@ -54,3 +68,41 @@ class DataImpulseProvider(ProxyProvider):
         if _COUNTRY_MARKER not in username:
             return ""
         return username.split(_COUNTRY_MARKER, 1)[1].split("__", 1)[0].lower()
+
+    def _rewrite_proxy_url(
+        self, *, country: str | None = None, session_id: str | None = None
+    ) -> str | None:
+        url = self.get_proxy_url()
+        if not url:
+            return None
+        parts = urlsplit(url)
+        username = _SESSION_RE.sub("", _COUNTRY_RE.sub("", parts.username or ""))
+        safe_country = _safe_country(country)
+        if safe_country is not None:
+            username += f"__cr.{safe_country}"
+        elif _COUNTRY_MARKER in (parts.username or ""):
+            username += f"__cr.{self.get_location()}"
+        if session_id is not None:
+            username += f"__sid.{_safe_session_id(session_id)}"
+        userinfo = quote(username, safe="%")
+        if parts.password is not None:
+            userinfo += f":{quote(parts.password, safe='%')}"
+        host = parts.hostname or ""
+        netloc = f"{userinfo}@{host}"
+        if parts.port:
+            netloc += f":{parts.port}"
+        return urlunsplit(
+            (parts.scheme, netloc, parts.path, parts.query, parts.fragment)
+        )
+
+    def get_geo_proxy_url(self, country: str | None = None) -> str | None:
+        """Return the configured URL with a country-routing suffix when requested."""
+        if not _safe_country(country):
+            return self.get_proxy_url()
+        return self._rewrite_proxy_url(country=country)
+
+    def get_sticky_proxy_url(
+        self, session_id: str, country: str | None = None
+    ) -> str | None:
+        """Return the configured URL with deterministic country/session suffixes."""
+        return self._rewrite_proxy_url(country=country, session_id=session_id)

surfsense_backend/scripts/e2e_amazon_scraper.py

@@ -0,0 +1,84 @@
+"""Manual end-to-end check for the public Amazon scraper.
+
+Run from the backend directory:
+
+    uv run python scripts/e2e_amazon_scraper.py
+    uv run python scripts/e2e_amazon_scraper.py --refresh-fixtures
+
+The script requires live network access and the configured residential proxy.
+The optional flag replaces the product and search parser fixtures with current
+live responses. The script is intentionally excluded from pytest.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import sys
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+_BACKEND_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(_BACKEND_ROOT))
+for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"):
+    if _candidate.exists():
+        load_dotenv(_candidate)
+        break
+
+from app.proprietary.platforms.amazon import (  # noqa: E402
+    AmazonScrapeInput,
+    scrape_products,
+)
+from app.proprietary.platforms.amazon.fetch import fetch_page  # noqa: E402
+
+_PRODUCT_URL = "https://www.amazon.com/dp/B09V3KXJPB"
+_SEARCH_URL = "https://www.amazon.com/s?k=wireless+mouse"
+_FIXTURE_DIR = _BACKEND_ROOT / "tests" / "unit" / "platforms" / "amazon" / "fixtures"
+
+
+def _check(label: str, passed: bool) -> bool:
+    print(f"[{'PASS' if passed else 'FAIL'}] {label}")
+    return passed
+
+
+async def main() -> int:
+    product_items = await scrape_products(
+        AmazonScrapeInput(categoryOrProductUrls=[{"url": _PRODUCT_URL}]),
+        limit=1,
+    )
+    product = product_items[0] if product_items else {}
+    print(json.dumps(product, indent=2, ensure_ascii=False)[:3000])
+    product_ok = _check(
+        "product detail has identity and title",
+        bool(product.get("asin") and product.get("title")),
+    )
+
+    search_items = await scrape_products(
+        AmazonScrapeInput(
+            categoryOrProductUrls=[{"url": _SEARCH_URL}],
+            maxItemsPerStartUrl=3,
+            scrapeProductDetails=False,
+        )
+    )
+    search_ok = _check(
+        "search returns product cards",
+        bool(search_items)
+        and all(
+            item.get("asin") and item.get("categoryPageData") for item in search_items
+        ),
+    )
+    fixture_ok = True
+    if "--refresh-fixtures" in sys.argv:
+        _FIXTURE_DIR.mkdir(parents=True, exist_ok=True)
+        for name, url in (("product.html", _PRODUCT_URL), ("search.html", _SEARCH_URL)):
+            response = await fetch_page(url)
+            saved = response is not None and response.status == 200
+            if saved:
+                (_FIXTURE_DIR / name).write_text(response.html, encoding="utf-8")
+            fixture_ok &= _check(f"refreshed {name}", saved)
+    return 0 if product_ok and search_ok and fixture_ok else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(asyncio.run(main()))

surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py

@@ -27,6 +27,7 @@ pytestmark = pytest.mark.unit
 # specialist is a deliberate product change and must be reflected here.
 _EXPECTED_SUBAGENTS = frozenset(
     {
+        "amazon",
         "deliverables",
         "dropbox",
         "google_drive",

surfsense_backend/tests/unit/capabilities/amazon/__init__.py

@@ -0,0 +1 @@
+

surfsense_backend/tests/unit/capabilities/amazon/scrape/__init__.py

@@ -0,0 +1 @@
+

surfsense_backend/tests/unit/capabilities/amazon/scrape/test_executor.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+from app.capabilities.amazon.scrape.executor import build_scrape_executor
+from app.capabilities.amazon.scrape.schemas import MAX_AMAZON_RESULTS, ScrapeInput
+from app.proprietary.platforms.amazon import AmazonScrapeInput
+
+
+class _FakeScraper:
+    def __init__(self) -> None:
+        self.calls: list[tuple[AmazonScrapeInput, int | None]] = []
+
+    async def __call__(
+        self, input_model: AmazonScrapeInput, *, limit: int | None = None
+    ) -> list[dict]:
+        self.calls.append((input_model, limit))
+        return [{"asin": "B09V3KXJPB", "title": "Product"}]
+
+
+async def test_executor_maps_agent_input_to_scraper_input():
+    scraper = _FakeScraper()
+    execute = build_scrape_executor(scraper)
+
+    output = await execute(
+        ScrapeInput(
+            search_terms=["wireless mouse"],
+            urls=["https://www.amazon.com/dp/B09V3KXJPB"],
+            max_items=5,
+            max_offers=2,
+            include_sellers=True,
+            zip_code="10001",
+            country_code="US",
+        )
+    )
+
+    assert output.items[0].asin == "B09V3KXJPB"
+    input_model, limit = scraper.calls[0]
+    assert input_model.categoryOrProductUrls == [
+        {"url": "https://www.amazon.com/dp/B09V3KXJPB"},
+        {"url": "https://www.amazon.com/s?k=wireless+mouse"},
+    ]
+    assert input_model.maxItemsPerStartUrl == 5
+    assert input_model.maxOffers == 2
+    assert input_model.scrapeSellers is True
+    assert input_model.zipCode == "10001"
+    assert limit == MAX_AMAZON_RESULTS

surfsense_backend/tests/unit/capabilities/amazon/scrape/test_schemas.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+import pytest
+from pydantic import ValidationError
+
+from app.capabilities.amazon.scrape.schemas import (
+    MAX_AMAZON_RESULTS,
+    ScrapeInput,
+    ScrapeOutput,
+)
+
+
+def test_estimated_units_cover_search_and_direct_variant_fanout():
+    payload = ScrapeInput(
+        search_terms=["mouse", "keyboard"],
+        urls=["https://www.amazon.com/dp/B09V3KXJPB"],
+        max_items=20,
+        max_variants=3,
+    )
+
+    assert payload.estimated_units == 44
+
+
+def test_estimated_units_respect_hard_run_ceiling():
+    payload = ScrapeInput(search_terms=["x"] * 20, max_items=100)
+    assert payload.estimated_units == MAX_AMAZON_RESULTS
+
+
+def test_at_least_one_source_is_required():
+    with pytest.raises(ValidationError):
+        ScrapeInput()
+
+
+def test_error_items_are_not_billable():
+    output = ScrapeOutput(
+        items=[
+            {"asin": "B09V3KXJPB", "title": "Product"},
+            {"error": "product_not_found", "errorDescription": "Missing"},
+        ]
+    )
+
+    assert output.billable_units == 1

surfsense_backend/tests/unit/capabilities/amazon/test_registry.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+from app.capabilities.amazon.scrape.schemas import ScrapeInput, ScrapeOutput
+from app.capabilities.core import BillingUnit
+from app.capabilities.core.store import get_capability
+
+
+def test_amazon_scrape_is_registered_and_metered():
+    capability = get_capability("amazon.scrape")
+
+    assert capability.input_schema is ScrapeInput
+    assert capability.output_schema is ScrapeOutput
+    assert capability.billing_unit is BillingUnit.AMAZON_PRODUCT

surfsense_backend/tests/unit/capabilities/google_maps/test_registry.py

@@ -7,6 +7,7 @@ import pytest
 from app.capabilities import (
     google_maps,  # noqa: F401  — importing the namespace registers its verbs
 )
+from app.capabilities.core import BillingUnit
 from app.capabilities.core.store import get_capability
 from app.capabilities.google_maps.reviews.schemas import ReviewsInput, ReviewsOutput
 from app.capabilities.google_maps.scrape.schemas import ScrapeInput, ScrapeOutput
@@ -14,19 +15,19 @@ from app.capabilities.google_maps.scrape.schemas import ScrapeInput, ScrapeOutpu
 pytestmark = pytest.mark.unit
 
 
-def test_google_maps_scrape_is_registered_and_free():
+def test_google_maps_scrape_is_registered_and_billable():
     cap = get_capability("google_maps.scrape")
 
     assert cap.name == "google_maps.scrape"
     assert cap.input_schema is ScrapeInput
     assert cap.output_schema is ScrapeOutput
-    assert cap.billing_unit is None
+    assert cap.billing_unit is BillingUnit.GOOGLE_MAPS_PLACE
 
 
-def test_google_maps_reviews_is_registered_and_free():
+def test_google_maps_reviews_is_registered_and_billable():
     cap = get_capability("google_maps.reviews")
 
     assert cap.name == "google_maps.reviews"
     assert cap.input_schema is ReviewsInput
     assert cap.output_schema is ReviewsOutput
-    assert cap.billing_unit is None
+    assert cap.billing_unit is BillingUnit.GOOGLE_MAPS_REVIEW

surfsense_backend/tests/unit/capabilities/reddit/test_registry.py

@@ -7,16 +7,17 @@ import pytest
 from app.capabilities import (
     reddit,  # noqa: F401  — importing the namespace registers its verbs
 )
+from app.capabilities.core import BillingUnit
 from app.capabilities.core.store import get_capability
 from app.capabilities.reddit.scrape.schemas import ScrapeInput, ScrapeOutput
 
 pytestmark = pytest.mark.unit
 
 
-def test_reddit_scrape_is_registered_and_free():
+def test_reddit_scrape_is_registered_and_billable():
     cap = get_capability("reddit.scrape")
 
     assert cap.name == "reddit.scrape"
     assert cap.input_schema is ScrapeInput
     assert cap.output_schema is ScrapeOutput
-    assert cap.billing_unit is None
+    assert cap.billing_unit is BillingUnit.REDDIT_ITEM

surfsense_backend/tests/unit/capabilities/youtube/test_registry.py

@@ -7,6 +7,7 @@ import pytest
 from app.capabilities import (
     youtube,  # noqa: F401  — importing the namespace registers its verbs
 )
+from app.capabilities.core import BillingUnit
 from app.capabilities.core.store import get_capability
 from app.capabilities.youtube.comments.schemas import CommentsInput, CommentsOutput
 from app.capabilities.youtube.scrape.schemas import ScrapeInput, ScrapeOutput
@@ -14,19 +15,19 @@ from app.capabilities.youtube.scrape.schemas import ScrapeInput, ScrapeOutput
 pytestmark = pytest.mark.unit
 
 
-def test_youtube_scrape_is_registered_and_free():
+def test_youtube_scrape_is_registered_and_billable():
     cap = get_capability("youtube.scrape")
 
     assert cap.name == "youtube.scrape"
     assert cap.input_schema is ScrapeInput
     assert cap.output_schema is ScrapeOutput
-    assert cap.billing_unit is None
+    assert cap.billing_unit is BillingUnit.YOUTUBE_VIDEO
 
 
-def test_youtube_comments_is_registered_and_free():
+def test_youtube_comments_is_registered_and_billable():
     cap = get_capability("youtube.comments")
 
     assert cap.name == "youtube.comments"
     assert cap.input_schema is CommentsInput
     assert cap.output_schema is CommentsOutput
-    assert cap.billing_unit is None
+    assert cap.billing_unit is BillingUnit.YOUTUBE_COMMENT

surfsense_backend/tests/unit/platforms/amazon/fixtures/bestsellers.html

@@ -0,0 +1,17 @@
+<!doctype html>
+<html lang="en">
+  <body>
+    <div id="gridItemRoot">
+      <span class="zg-bdg-text">#1</span>
+      <a href="/dp/B09V3KXJPB"><span class="_cDEzb_p13n-sc-css-line-clamp-3_g3dy1">Example Wireless Headphones</span></a>
+      <span class="a-price"><span class="a-offscreen">$49.99</span></span>
+      <span class="a-icon-alt">4.6 out of 5 stars</span>
+      <img src="https://images.example/headphones.jpg" alt="Example Wireless Headphones">
+    </div>
+    <div id="gridItemRoot">
+      <span class="zg-bdg-text">#2</span>
+      <a href="/dp/B09V3KXJP1"><span class="_cDEzb_p13n-sc-css-line-clamp-3_g3dy1">Example Earbuds</span></a>
+      <span class="a-price"><span class="a-offscreen">$29.99</span></span>
+    </div>
+  </body>
+</html>

surfsense_backend/tests/unit/platforms/amazon/fixtures/blocked.html

@@ -0,0 +1,9 @@
+<!doctype html>
+<html lang="en">
+  <head><title>Robot Check</title></head>
+  <body>
+    <form action="/errors/validateCaptcha">
+      <p>Enter the characters you see below</p>
+    </form>
+  </body>
+</html>

surfsense_backend/tests/unit/platforms/amazon/fixtures/offers.html

@@ -0,0 +1,17 @@
+<!doctype html>
+<html lang="en">
+  <body>
+    <div id="aod-offer">
+      <div id="aod-pinned-offer"></div>
+      <div id="aod-offer-heading">New</div>
+      <span class="a-price"><span class="a-offscreen">$47.50</span></span>
+      <div id="aod-offer-soldBy"><a href="/sp?seller=SELLER123">Example Retailer</a></div>
+      <div id="mir-layout-DELIVERY_BLOCK">FREE delivery tomorrow</div>
+    </div>
+    <div id="aod-offer">
+      <div id="aod-offer-heading">Used - Very Good</div>
+      <span class="a-price"><span class="a-offscreen">$39.00</span></span>
+      <div id="aod-offer-soldBy"><a href="/sp?seller=SELLER456">Second Seller</a></div>
+    </div>
+  </body>
+</html>

surfsense_backend/tests/unit/platforms/amazon/fixtures/product.html

@@ -0,0 +1,51 @@
+<!doctype html>
+<html lang="en">
+  <body>
+    <div id="wayfinding-breadcrumbs_feature_div"><ul><li><a href="/b?node=172282">Electronics</a></li><li><a href="/b?node=667846011">Audio</a></li></ul></div>
+    <span id="productTitle">Example Wireless Headphones</span>
+    <a id="bylineInfo" href="/stores/Example">Visit the Example Store</a>
+    <a id="sellerProfileTriggerId" href="/sp?seller=SELLER123">Example Retailer</a>
+    <div id="corePrice_feature_div">
+      <span class="priceToPay"><span class="a-offscreen">$49.99</span></span>
+      <span class="basisPrice"><span class="a-offscreen">$79.99</span></span>
+    </div>
+    <div id="averageCustomerReviews">
+      <span id="acrPopover">4.6 out of 5 stars</span>
+      <span id="acrCustomerReviewText">1,234 ratings</span>
+      <a id="acrCustomerReviewLink" href="/product-reviews/B09V3KXJPB">Reviews</a>
+    </div>
+    <table id="histogramTable">
+      <tr><td class="a-text-left">5 star</td><td class="a-text-right">80%</td></tr>
+      <tr><td class="a-text-left">4 star</td><td class="a-text-right">12%</td></tr>
+    </table>
+    <div id="availability">In Stock</div>
+    <div id="social-proofing-faceout-title-tk_bought">2K+ bought in past month</div>
+    <div id="feature-bullets"><ul><li><span class="a-list-item">Long battery life</span></li><li><span class="a-list-item">Noise cancelling</span></li></ul></div>
+    <div id="productDescription">Detailed product description.</div>
+    <img id="landingImage" src="https://images.example/thumb.jpg" data-old-hires="https://images.example/full.jpg" alt="Example Wireless Headphones">
+    <div id="aplus"><p>Premium materials and comfort.</p><img src="https://images.example/aplus.jpg" alt="Premium materials and comfort"></div>
+    <div id="brandStory_feature_div">Designed for everyday listening.</div>
+    <div id="detailBulletsWrapper_feature_div"><ul><li>Best Sellers Rank: #12 in Electronics #3 in Headphones</li></ul></div>
+    <table id="productDetails_techSpec_section_1"><tr><th>Model</th><td>WH-100</td></tr></table>
+    <div id="productOverview_feature_div"><table><tr><td>Color</td><td>Black</td></tr></table></div>
+    <div id="acBadge_feature_div">Amazon's Choice</div>
+    <script>
+      window.twister = {
+        "dimensionValuesDisplayData": {
+          "B09V3KXJPB": ["Black"],
+          "B09V3KXJP1": ["Blue"]
+        },
+        "variationValues": {"color_name": ["Black", "Blue"]}
+      };
+    </script>
+    <div data-hook="review" id="R1">
+      <span class="a-profile-name">Alex</span>
+      <a data-hook="review-title" href="/gp/customer-reviews/R1">Excellent sound</a>
+      <i data-hook="review-star-rating">5.0 out of 5 stars</i>
+      <span data-hook="review-date">Reviewed on July 1, 2026</span>
+      <span data-hook="review-body">Clear and comfortable.</span>
+      <span data-hook="avp-badge">Verified Purchase</span>
+      <span data-hook="helpful-vote-statement">12 people found this helpful</span>
+    </div>
+  </body>
+</html>

surfsense_backend/tests/unit/platforms/amazon/fixtures/search.html

@@ -0,0 +1,19 @@
+<!doctype html>
+<html lang="en">
+  <body>
+    <div data-component-type="s-search-result" data-asin="B09V3KXJPB" class="s-result-item">
+      <h2><a href="/dp/B09V3KXJPB"><span>Example Wireless Headphones</span></a></h2>
+      <span class="a-price"><span class="a-offscreen">$49.99</span></span>
+      <span class="a-icon-alt">4.6 out of 5 stars</span>
+      <span class="s-underline-text">1,234</span>
+      <img class="s-image" src="https://images.example/headphones.jpg" alt="Example Wireless Headphones">
+      <span class="a-badge-text">Best Seller</span>
+    </div>
+    <div data-component-type="s-search-result" data-asin="B09V3KXJP1" class="s-result-item">
+      <span>Sponsored</span>
+      <h2><a href="/dp/B09V3KXJP1"><span>Example Earbuds</span></a></h2>
+      <span class="a-price"><span class="a-offscreen">$29.99</span></span>
+      <span class="a-icon-alt">4.2 out of 5 stars</span>
+    </div>
+  </body>
+</html>

surfsense_backend/tests/unit/platforms/amazon/fixtures/seller.html

@@ -0,0 +1,11 @@
+<!doctype html>
+<html lang="en">
+  <body>
+    <h1 id="sellerName">Example Retailer</h1>
+    <div id="seller-feedback-summary">1,875 ratings</div>
+    <table id="feedback-summary-table">
+      <tr><td><span class="a-icon-alt">4.8 out of 5 stars</span></td></tr>
+      <tr><td>1,875 ratings</td></tr>
+    </table>
+  </body>
+</html>

surfsense_backend/tests/unit/platforms/amazon/test_flows.py

@@ -0,0 +1,220 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from app.proprietary.platforms.amazon import AmazonScrapeInput, scrape_products, scraper
+from app.proprietary.platforms.amazon.fetch import FetchResult
+from app.proprietary.platforms.amazon.url_resolver import resolve_url
+
+_FIXTURES = Path(__file__).parent / "fixtures"
+
+
+def _fixture(name: str) -> str:
+    return (_FIXTURES / name).read_text(encoding="utf-8")
+
+
+def _response(url: str, html: str, status: int = 200) -> FetchResult:
+    return FetchResult(status=status, html=html, url=url, cookies={})
+
+
+async def test_product_flow_emits_parsed_item(monkeypatch):
+    async def fetch_page(url: str, **_kwargs):
+        return _response(url, _fixture("product.html"))
+
+    monkeypatch.setattr(scraper, "fetch_page", fetch_page)
+    items = await scrape_products(
+        AmazonScrapeInput(
+            categoryOrProductUrls=[{"url": "https://www.amazon.com/dp/B09V3KXJPB"}]
+        )
+    )
+
+    assert len(items) == 1
+    assert items[0]["asin"] == "B09V3KXJPB"
+    assert items[0]["title"] == "Example Wireless Headphones"
+
+
+async def test_product_flow_maps_not_found_to_error_item(monkeypatch):
+    async def fetch_page(url: str, **_kwargs):
+        return _response(url, "", status=404)
+
+    monkeypatch.setattr(scraper, "fetch_page", fetch_page)
+    items = await scrape_products(
+        AmazonScrapeInput(
+            categoryOrProductUrls=[{"url": "https://www.amazon.com/dp/B09V3KXJPB"}]
+        )
+    )
+
+    assert items[0]["error"] == "product_not_found"
+
+
+async def test_search_flow_honors_cap_and_stops_on_empty_page(monkeypatch):
+    calls: list[str] = []
+
+    async def fetch_page(url: str, **_kwargs):
+        calls.append(url)
+        html = _fixture("search.html") if "page=1" in url else "<html></html>"
+        return _response(url, html)
+
+    monkeypatch.setattr(scraper, "fetch_page", fetch_page)
+    items = await scrape_products(
+        AmazonScrapeInput(
+            categoryOrProductUrls=[{"url": "https://www.amazon.com/s?k=headphones"}],
+            maxItemsPerStartUrl=1,
+            maxSearchPagesPerStartUrl=5,
+            scrapeProductDetails=False,
+        )
+    )
+
+    assert len(items) == 1
+    assert items[0]["categoryPageData"]["position"] == 1
+    assert len(calls) == 1
+
+
+async def test_search_flow_threads_marketplace_locale_to_fetch(monkeypatch):
+    calls: list[dict[str, object]] = []
+
+    async def fetch_page(url: str, **kwargs):
+        calls.append({"url": url, **kwargs})
+        return _response(url, _fixture("search.html"))
+
+    monkeypatch.setattr(scraper, "fetch_page", fetch_page)
+    items = await scrape_products(
+        AmazonScrapeInput(
+            categoryOrProductUrls=[{"url": "https://www.amazon.co.uk/s?k=headphones"}],
+            maxItemsPerStartUrl=1,
+            scrapeProductDetails=False,
+        )
+    )
+
+    assert len(items) == 1
+    assert calls[0]["country"] == "gb"
+    assert calls[0]["accept_language"] == "en-GB"
+
+
+async def test_search_flow_returns_no_results_error(monkeypatch):
+    async def fetch_page(url: str, **_kwargs):
+        return _response(url, "<html></html>")
+
+    monkeypatch.setattr(scraper, "fetch_page", fetch_page)
+    items = await scrape_products(
+        AmazonScrapeInput(
+            categoryOrProductUrls=[{"url": "https://www.amazon.com/s?k=missing"}],
+            scrapeProductDetails=False,
+        )
+    )
+
+    assert items[0]["error"] == "no_results_found"
+
+
+async def test_product_flow_enriches_offers_and_seller(monkeypatch):
+    async def fetch_page(url: str, **_kwargs):
+        return _response(url, _fixture("product.html"))
+
+    async def fetch_aod_html(*_args, **_kwargs):
+        return _fixture("offers.html")
+
+    async def fetch_seller_html(*_args, **_kwargs):
+        return _fixture("seller.html")
+
+    monkeypatch.setattr(scraper, "fetch_page", fetch_page)
+    monkeypatch.setattr(scraper, "fetch_aod_html", fetch_aod_html)
+    monkeypatch.setattr(scraper, "fetch_seller_html", fetch_seller_html)
+    items = await scrape_products(
+        AmazonScrapeInput(
+            categoryOrProductUrls=[{"url": "https://www.amazon.com/dp/B09V3KXJPB"}],
+            maxOffers=1,
+            scrapeSellers=True,
+        )
+    )
+
+    assert len(items[0]["offers"]) == 1
+    assert items[0]["offers"][0]["seller"]["averageRating"] == 4.8
+    assert items[0]["seller"]["reviewsCount"] == 1875
+
+
+async def test_bestsellers_card_only_flow(monkeypatch):
+    async def fetch_page(url: str, **_kwargs):
+        return _response(url, _fixture("bestsellers.html"))
+
+    monkeypatch.setattr(scraper, "fetch_page", fetch_page)
+    items = await scrape_products(
+        AmazonScrapeInput(
+            categoryOrProductUrls=[
+                {
+                    "url": (
+                        "https://www.amazon.com/Best-Sellers-Electronics/"
+                        "zgbs/electronics"
+                    )
+                }
+            ],
+            maxItemsPerStartUrl=2,
+            scrapeProductDetails=False,
+        )
+    )
+
+    assert [item["bestsellerPageData"]["rank"] for item in items] == [1, 2]
+
+
+async def test_shortlink_resolves_and_dispatches(monkeypatch):
+    async def resolve(_url: str):
+        return "https://www.amazon.com/dp/B09V3KXJPB"
+
+    async def fetch_page(url: str, **_kwargs):
+        return _response(url, _fixture("product.html"))
+
+    monkeypatch.setattr(scraper, "resolve_shortlink", resolve)
+    monkeypatch.setattr(scraper, "fetch_page", fetch_page)
+    items = await scrape_products(
+        AmazonScrapeInput(categoryOrProductUrls=[{"url": "https://a.co/d/example"}])
+    )
+
+    assert items[0]["asin"] == "B09V3KXJPB"
+    assert items[0]["input"] == "https://a.co/d/example"
+
+
+async def test_product_flow_expands_variants_and_attaches_prices(monkeypatch):
+    async def fetch_page(url: str, **_kwargs):
+        return _response(url, _fixture("product.html"))
+
+    monkeypatch.setattr(scraper, "fetch_page", fetch_page)
+    items = await scrape_products(
+        AmazonScrapeInput(
+            categoryOrProductUrls=[{"url": "https://www.amazon.com/dp/B09V3KXJPB"}],
+            maxProductVariantsAsSeparateResults=1,
+            scrapeProductVariantPrices=True,
+        )
+    )
+
+    assert [item["asin"] for item in items] == ["B09V3KXJPB", "B09V3KXJP1"]
+    assert items[1]["originalAsin"] == "B09V3KXJPB"
+    variant = next(
+        detail
+        for detail in items[0]["variantDetails"]
+        if detail["asin"] == "B09V3KXJP1"
+    )
+    assert variant["price"]["value"] == 49.99
+
+
+async def test_route_outside_location_allowlist_skips_session_mint(monkeypatch):
+    calls = 0
+
+    async def get_location_session(*_args, **_kwargs):
+        nonlocal calls
+        calls += 1
+        return None
+
+    monkeypatch.setattr(scraper, "get_location_session", get_location_session)
+    resolved = resolve_url("https://www.amazon.com/dp/B09V3KXJPB")
+    assert resolved is not None
+    context = await scraper._location_context(
+        resolved,
+        AmazonScrapeInput(
+            categoryOrProductUrls=[{"url": resolved.url}],
+            zipCode="10001",
+            locationDeliverableRoutes=["OFFERS"],
+        ),
+        "PRODUCT",
+    )
+
+    assert context == (None, None, None, None)
+    assert calls == 0

surfsense_backend/tests/unit/platforms/amazon/test_locale.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from app.proprietary.platforms.amazon.locale import (
+    accept_language_for,
+    proxy_country_for,
+)
+
+
+def test_proxy_country_for_confirmed_marketplaces():
+    assert proxy_country_for("com") == "us"
+    assert proxy_country_for("co.uk") == "gb"
+    assert proxy_country_for("de") == "de"
+    assert proxy_country_for("fr") == "fr"
+    assert proxy_country_for("it") == "it"
+    assert proxy_country_for("es") == "es"
+
+
+def test_proxy_country_for_unknown_marketplace_falls_back():
+    assert proxy_country_for("co.jp") is None
+    assert proxy_country_for(None) is None
+
+
+def test_accept_language_for_marketplaces():
+    assert accept_language_for("co.uk") == "en-GB"
+    assert accept_language_for("de") == "de-DE"
+    assert accept_language_for("fr") == "fr-FR"
+    assert accept_language_for("it") == "it-IT"
+    assert accept_language_for("es") == "es-ES"
+
+
+def test_accept_language_for_unknown_marketplace_defaults_to_us_english():
+    assert accept_language_for("co.jp") == "en-US"
+    assert accept_language_for(None) == "en-US"

surfsense_backend/tests/unit/platforms/amazon/test_parsers.py

@@ -0,0 +1,176 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from app.proprietary.platforms.amazon import fetch
+from app.proprietary.platforms.amazon.fetch import (
+    FetchResult,
+    get_location_session,
+    is_blocked,
+    should_localize,
+)
+from app.proprietary.platforms.amazon.parsers import (
+    _float,
+    parse_aod_offers,
+    parse_bestsellers_page,
+    parse_product,
+    parse_search_page,
+    parse_seller,
+)
+
+_FIXTURES = Path(__file__).parent / "fixtures"
+
+
+def _fixture(name: str) -> str:
+    return (_FIXTURES / name).read_text(encoding="utf-8")
+
+
+def test_product_parser_extracts_core_public_fields():
+    item = parse_product(
+        _fixture("product.html"),
+        asin="B09V3KXJPB",
+        url="https://www.amazon.com/dp/B09V3KXJPB",
+    )
+
+    assert item["title"] == "Example Wireless Headphones"
+    assert item["price"] == {"value": 49.99, "currency": "USD"}
+    assert item["listPrice"] == {"value": 79.99, "currency": "USD"}
+    assert item["stars"] == 4.6
+    assert item["reviewsCount"] == 1234
+    assert item["inStock"] is True
+    assert item["features"] == ["Long battery life", "Noise cancelling"]
+    assert item["starsBreakdown"]["5star"] == 0.8
+    assert "B09V3KXJP1" in item["variantAsins"]
+    assert item["variantAttributes"][0]["name"] == "color_name"
+    assert item["productPageReviews"][0]["id"] == "R1"
+
+
+def test_block_detection_handles_status_and_markup():
+    blocked = _fixture("blocked.html")
+    assert is_blocked(blocked, 200)
+    assert is_blocked("", 503)
+    assert not is_blocked(_fixture("product.html"), 200)
+
+
+def test_block_detection_handles_waf_header_and_body_markers():
+    assert is_blocked(
+        "<html>ordinary status</html>",
+        200,
+        {"X-Amzn-Waf-Action": "challenge"},
+    )
+    assert is_blocked(
+        "<script src='https://token.awswaf.com/challenge.js'></script>", 200
+    )
+    assert is_blocked(
+        '<meta http-equiv="refresh" content="5; URL=/s?bm-verify=x">', 200
+    )
+
+
+def test_float_handles_us_and_eu_price_formats():
+    assert _float("1.234,56 €") == 1234.56
+    assert _float("12,99 €") == 12.99
+    assert _float("$1,234.56") == 1234.56
+    assert _float("1234") == 1234.0
+
+
+def test_block_retry_proxy_uses_fresh_country_session(monkeypatch):
+    geo_calls: list[str | None] = []
+    sticky_calls: list[tuple[str, str | None]] = []
+
+    def get_geo_proxy_url(country: str | None = None):
+        geo_calls.append(country)
+        return f"http://geo-{country}"
+
+    def get_sticky_proxy_url(session_id: str, country: str | None = None):
+        sticky_calls.append((session_id, country))
+        return f"http://sticky-{country}-{session_id}"
+
+    monkeypatch.setattr(fetch, "get_geo_proxy_url", get_geo_proxy_url)
+    monkeypatch.setattr(fetch, "get_sticky_proxy_url", get_sticky_proxy_url)
+
+    first = fetch._selected_proxy(None, "fr", 1, "https://www.amazon.fr/s?k=x")
+    second = fetch._selected_proxy(None, "fr", 2, "https://www.amazon.fr/s?k=x")
+
+    assert first == "http://geo-fr"
+    assert second.startswith("http://sticky-fr-amazon-fr-2-")
+    assert geo_calls == ["fr"]
+    assert sticky_calls[0][1] == "fr"
+
+
+def test_search_parser_extracts_cards_and_provenance():
+    cards = parse_search_page(_fixture("search.html"), page=2)
+
+    assert [card["asin"] for card in cards] == ["B09V3KXJPB", "B09V3KXJP1"]
+    assert cards[0]["categoryPageData"] == {
+        "position": 1,
+        "page": 2,
+        "isSponsored": False,
+        "isBestSeller": True,
+    }
+    assert cards[1]["categoryPageData"]["isSponsored"] is True
+
+
+def test_offer_and_seller_parsers():
+    offers = parse_aod_offers(_fixture("offers.html"))
+    seller = parse_seller(_fixture("seller.html"), seller_id="SELLER123")
+
+    assert len(offers) == 2
+    assert offers[0]["price"]["value"] == 47.5
+    assert offers[0]["seller"]["id"] == "SELLER123"
+    assert offers[0]["isPinnedOffer"] is True
+    assert seller["name"] == "Example Retailer"
+    assert seller["averageRating"] == 4.8
+    assert seller["reviewsCount"] == 1875
+
+
+def test_bestsellers_parser_extracts_ranked_products():
+    items = parse_bestsellers_page(_fixture("bestsellers.html"))
+
+    assert [item["asin"] for item in items] == ["B09V3KXJPB", "B09V3KXJP1"]
+    assert [item["bestsellerPageData"]["rank"] for item in items] == [1, 2]
+
+
+def test_location_route_filter_is_explicit():
+    routes = ["PRODUCT", "OFFERS"]
+    assert should_localize("product", routes)
+    assert not should_localize("search", routes)
+
+
+async def test_location_session_mints_once_and_reuses_cache(monkeypatch):
+    fetch._location_sessions.clear()
+    calls: list[str] = []
+
+    async def sticky_proxy(_session_id: str, _country: str | None = None):
+        return "http://sticky-proxy"
+
+    async def fetch_page(url: str, **_kwargs):
+        calls.append(url)
+        if len(calls) == 1:
+            return FetchResult(
+                status=200,
+                html='<input name="anti-csrftoken-a2z" value="token-123">',
+                url=url,
+                cookies={"session-id": "session", "ubid-main": "ubid"},
+            )
+        return FetchResult(
+            status=200,
+            html='{"isValidAddress":1}',
+            url=url,
+            cookies={"session-id": "session"},
+        )
+
+    monkeypatch.setattr(fetch, "_sticky_proxy", sticky_proxy)
+    monkeypatch.setattr(fetch, "fetch_page", fetch_page)
+
+    first = await get_location_session(
+        "www.amazon.com", zip_code="10001", country_code="US"
+    )
+    second = await get_location_session(
+        "www.amazon.com", zip_code="10001", country_code="US"
+    )
+
+    assert first is second
+    assert first is not None
+    assert first.proxy == "http://sticky-proxy"
+    assert first.country_code == "US"
+    assert len(calls) == 2

surfsense_backend/tests/unit/platforms/amazon/test_proxy.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+import pytest
+
+from app.config import Config
+from app.utils.proxy.providers.dataimpulse import DataImpulseProvider
+
+
+def test_dataimpulse_sticky_url_is_deterministic(monkeypatch):
+    monkeypatch.setattr(
+        Config,
+        "PROXY_URL",
+        "http://token__cr.us:secret@gw.dataimpulse.com:823",
+    )
+    provider = DataImpulseProvider()
+
+    first = provider.get_sticky_proxy_url("location-123")
+    second = provider.get_sticky_proxy_url("location-123")
+
+    assert first == second
+    assert "token__cr.us__sid.location-123" in first
+    assert first.endswith("@gw.dataimpulse.com:823")
+
+
+def test_dataimpulse_sticky_url_replaces_existing_session(monkeypatch):
+    monkeypatch.setattr(
+        Config,
+        "PROXY_URL",
+        "http://token__cr.us__sid.old:secret@gw.dataimpulse.com:823",
+    )
+
+    result = DataImpulseProvider().get_sticky_proxy_url("new")
+
+    assert "__sid.old" not in result
+    assert result.count("__sid.new") == 1
+
+
+def test_dataimpulse_sticky_url_rewrites_country(monkeypatch):
+    monkeypatch.setattr(
+        Config,
+        "PROXY_URL",
+        "http://token__cr.us:secret@gw.dataimpulse.com:823",
+    )
+
+    result = DataImpulseProvider().get_sticky_proxy_url("new", "gb")
+
+    assert "token__cr.gb__sid.new" in result
+    assert "__cr.us" not in result
+
+
+def test_dataimpulse_sticky_url_rejects_empty_session(monkeypatch):
+    monkeypatch.setattr(
+        Config,
+        "PROXY_URL",
+        "http://token:secret@gw.dataimpulse.com:823",
+    )
+
+    with pytest.raises(ValueError):
+        DataImpulseProvider().get_sticky_proxy_url("...")

surfsense_backend/tests/unit/platforms/amazon/test_skeleton.py

@@ -0,0 +1,177 @@
+"""Offline contract tests for the Amazon Product Scraper.
+
+Deterministic (no network, no live Amazon HTML): asserts the input defaults, the
+full output-item serialization contract, the error-item shape, and URL
+classification. The live fetch / parse flows are exercised by the e2e script in
+later milestones, not here.
+"""
+
+from __future__ import annotations
+
+from app.proprietary.platforms.amazon import (
+    AmazonScrapeInput,
+    ErrorItem,
+    ProductItem,
+    scrape_products,
+)
+from app.proprietary.platforms.amazon.url_resolver import extract_asin, resolve_url
+
+# A complete input payload should validate without removing optional fields.
+_SPEC_INPUT = {
+    "categoryOrProductUrls": [{"url": "https://www.amazon.com/s?k=keyboard"}],
+    "maxItemsPerStartUrl": 100,
+    "language": "en",
+    "proxyCountry": "AUTO_SELECT_PROXY_COUNTRY",
+    "maxSearchPagesPerStartUrl": 9999,
+    "maxOffers": 0,
+    "scrapeSellers": False,
+    "useCaptchaSolver": False,
+    "scrapeProductVariantPrices": False,
+    "scrapeProductDetails": True,
+    "countryCode": "US",
+    "zipCode": "10001",
+    "locationDeliverableRoutes": ["PRODUCT", "SEARCH", "OFFERS"],
+}
+
+
+def test_input_has_no_auth_fields():
+    # Public, anonymous only: no auth-shaped field may exist on the input surface.
+    forbidden = {"username", "password", "token", "login", "auth", "credentials"}
+    assert forbidden.isdisjoint(AmazonScrapeInput.model_fields)
+
+
+def test_scrape_input_defaults_match_contract():
+    inp = AmazonScrapeInput(
+        categoryOrProductUrls=[{"url": "https://www.amazon.com/dp/B0"}]
+    )
+    assert inp.maxItemsPerStartUrl is None
+    assert inp.maxSearchPagesPerStartUrl == 9999
+    assert inp.maxProductVariantsAsSeparateResults == 0
+    assert inp.maxOffers == 0
+    assert inp.proxyCountry == "AUTO_SELECT_PROXY_COUNTRY"
+    assert inp.language is None
+    assert inp.countryCode is None
+    assert inp.zipCode is None
+    assert inp.locationDeliverableRoutes == ["PRODUCT", "SEARCH", "OFFERS"]
+    assert inp.scrapeSellers is False
+    assert inp.useCaptchaSolver is False
+    assert inp.scrapeProductVariantPrices is False
+    assert inp.scrapeProductDetails is True
+
+
+def test_complete_payload_validates():
+    inp = AmazonScrapeInput(**_SPEC_INPUT)
+    assert inp.maxItemsPerStartUrl == 100
+    assert inp.countryCode == "US"
+
+
+def test_input_allows_extra_inert_fields():
+    # extra="allow": unknown add-ons (e.g. upstream proxy config) are accepted.
+    inp = AmazonScrapeInput(
+        categoryOrProductUrls=[{"url": "https://www.amazon.com/dp/B0"}],
+        proxyConfiguration={"useResidentialProxy": True},
+        someFutureAddon=123,
+    )
+    assert inp.model_dump().get("someFutureAddon") == 123
+
+
+def test_output_item_serializes_full_shape():
+    item = ProductItem(asin="B08EXAMPLE01").to_output()
+    assert item["asin"] == "B08EXAMPLE01"
+    # Unsourced scalars are still present as None (consumers never KeyError).
+    assert item["title"] is None
+    assert item["monthlyPurchaseVolume"] is None
+    # List fields default to [].
+    assert item["features"] == []
+    assert item["offers"] == []
+    assert item["productPageReviews"] == []
+    # Typed-but-unset nested objects are None.
+    assert item["price"] is None
+    assert item["seller"] is None
+
+
+def test_stars_breakdown_round_trips_digit_keys():
+    item = ProductItem(
+        stars=4.8, starsBreakdown={"5star": 0.86, "1star": 0.01}
+    ).to_output()
+    # by_alias serialization restores Amazon's digit-prefixed keys.
+    assert item["starsBreakdown"]["5star"] == 0.86
+    assert item["starsBreakdown"]["1star"] == 0.01
+    assert item["starsBreakdown"]["4star"] is None
+
+
+def test_error_item_shape():
+    err = ErrorItem(
+        error="product_not_found",
+        errorDescription="Loaded a 404 page.",
+        input="https://www.amazon.com/dp/B0XXXXXXXX",
+        url="https://www.amazon.com/dp/B0XXXXXXXX",
+    ).model_dump()
+    assert set(err) >= {"error", "errorDescription", "input", "url"}
+    assert err["error"] == "product_not_found"
+
+
+def test_extract_asin():
+    assert extract_asin("https://www.amazon.com/dp/B09X7MPX8L") == "B09X7MPX8L"
+    assert extract_asin("https://www.amazon.com/gp/product/B09X7MPX8L/ref=x") == (
+        "B09X7MPX8L"
+    )
+    assert extract_asin("https://www.amazon.com/s?k=keyboard") is None
+
+
+def test_resolve_url_classifies_all_kinds():
+    product = resolve_url("https://www.amazon.com/dp/B0EXAMPLE1")
+    assert product is not None
+    assert product.kind == "product"
+    assert product.asin == "B0EXAMPLE1"
+    assert product.marketplace == "com"
+
+    search = resolve_url("https://www.amazon.com/s?k=keyboard")
+    assert search is not None and search.kind == "search"
+
+    # A category URL with no /s path but a bbn/rh query still classifies as search.
+    category = resolve_url(
+        "https://www.amazon.de/s?i=specialty-aps&bbn=16225007011&rh=n%3A16225007011"
+    )
+    assert category is not None
+    assert category.kind == "search"
+    assert category.marketplace == "de"
+
+    bestsellers = resolve_url(
+        "https://www.amazon.com/Best-Sellers-Electronics/zgbs/electronics"
+    )
+    assert bestsellers is not None and bestsellers.kind == "bestsellers"
+
+    shortened = resolve_url("https://a.co/d/abcd123")
+    assert shortened is not None and shortened.kind == "shortened"
+
+
+def test_resolve_url_rejects_non_amazon_and_unrecognized():
+    assert resolve_url("https://example.com/dp/B08EXAMPLE1") is None
+    # An Amazon host but an unrecognized path (e.g. the homepage) is unrecognized.
+    assert resolve_url("https://www.amazon.com/") is None
+
+
+async def test_iter_products_unrecognized_yields_error():
+    # A junk / non-Amazon URL yields exactly one invalid_url error item.
+    items = await scrape_products(
+        AmazonScrapeInput(categoryOrProductUrls=[{"url": "https://example.com/foo"}])
+    )
+    assert len(items) == 1
+    assert items[0]["error"] == "invalid_url"
+    assert items[0]["input"] == "https://example.com/foo"
+
+
+def test_valid_product_url_is_ready_for_dispatch():
+    resolved = resolve_url("https://www.amazon.com/dp/B0EXAMPLE1")
+    assert resolved is not None
+    assert resolved.kind == "product"
+
+
+async def test_iter_products_missing_url_key_yields_error():
+    # An entry without a "url" key is treated as an invalid start URL, not a crash.
+    items = await scrape_products(
+        AmazonScrapeInput(categoryOrProductUrls=[{"noturl": "x"}])
+    )
+    assert len(items) == 1
+    assert items[0]["error"] == "invalid_url"

surfsense_backend/tests/unit/tasks/celery_tasks/test_schedule_checker_task.py

@@ -59,7 +59,7 @@ def _due_connector(connector_type: SearchSourceConnectorType) -> SimpleNamespace
     return SimpleNamespace(
         id=42,
         connector_type=connector_type,
-        search_space_id=7,
+        workspace_id=7,
         user_id="00000000-0000-0000-0000-000000000001",
         config={},
         periodic_indexing_enabled=True,
@@ -86,13 +86,8 @@ async def _run_checker(monkeypatch: pytest.MonkeyPatch, connector: SimpleNamespa
 
 
 @pytest.mark.asyncio
-async def test_due_bookstack_connector_dispatches_indexing_task(monkeypatch):
-    """A due BookStack connector must dispatch index_bookstack_pages_task.
-
-    Regression test for the connector type missing from the scheduler's
-    task_map, which made periodic BookStack syncs silently no-op with only a
-    "No task found" warning.
-    """
+async def test_due_bookstack_connector_disables_deprecated_indexing(monkeypatch):
+    """A due BookStack connector is retired from periodic indexing, not dispatched."""
     from app.tasks.celery_tasks import connector_tasks
 
     task_mock = MagicMock()
@@ -101,16 +96,9 @@ async def test_due_bookstack_connector_dispatches_indexing_task(monkeypatch):
     connector = _due_connector(SearchSourceConnectorType.BOOKSTACK_CONNECTOR)
     session = await _run_checker(monkeypatch, connector)
 
-    task_mock.delay.assert_called_once_with(
-        connector.id,
-        connector.search_space_id,
-        str(connector.user_id),
-        None,
-        None,
-    )
-    # The next run must be rescheduled, otherwise the connector stays "due"
-    # and is re-examined every minute.
-    assert connector.next_scheduled_at > datetime.now(UTC)
+    task_mock.delay.assert_not_called()
+    assert connector.periodic_indexing_enabled is False
+    assert connector.next_scheduled_at is None
     assert session.commits == 1
 
 

surfsense_backend/tests/unit/utils/proxy/test_dataimpulse_provider.py

@@ -44,6 +44,51 @@ def test_location_stops_at_next_param(monkeypatch: pytest.MonkeyPatch) -> None:
     assert DataImpulseProvider().get_location() == "de"
 
 
+def test_geo_proxy_url_rewrites_country_suffix(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(Config, "PROXY_URL", _URL)
+
+    result = DataImpulseProvider().get_geo_proxy_url("gb")
+
+    assert result == "http://tok123__cr.gb:secret@gw.dataimpulse.com:823"
+
+
+def test_geo_proxy_url_replaces_existing_country_once(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(
+        Config,
+        "PROXY_URL",
+        "http://tok123__cr.us__sid.old:secret@gw.dataimpulse.com:823",
+    )
+
+    result = DataImpulseProvider().get_geo_proxy_url("de")
+
+    assert "__cr.us" not in result
+    assert result.count("__cr.de") == 1
+    assert "__sid.old" not in result
+
+
+def test_geo_proxy_url_without_country_keeps_base_url(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(Config, "PROXY_URL", _URL)
+
+    assert DataImpulseProvider().get_geo_proxy_url(None) == _URL
+    assert DataImpulseProvider().get_geo_proxy_url("") == _URL
+
+
+def test_sticky_proxy_url_composes_country_and_session(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(Config, "PROXY_URL", _URL)
+
+    result = DataImpulseProvider().get_sticky_proxy_url("location-123", "gb")
+
+    assert result == (
+        "http://tok123__cr.gb__sid.location-123:secret@gw.dataimpulse.com:823"
+    )
+
+
 def test_no_country_suffix_yields_empty_location(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:

surfsense_mcp/mcp_server/features/scrapers/__init__.py

@@ -14,6 +14,7 @@ from ...core.client import SurfSenseClient
 from ...core.workspace_context import WorkspaceContext
 from . import run_history
 from .platforms import (
+    amazon,
     google_maps,
     google_search,
     instagram,
@@ -31,6 +32,7 @@ _REGISTRARS = (
     instagram,
     tiktok,
     google_maps,
+    amazon,
     run_history,
 )
 

surfsense_mcp/mcp_server/features/scrapers/platforms/amazon.py

@@ -0,0 +1,145 @@
+"""Amazon product scraper tool."""
+
+from __future__ import annotations
+
+from typing import Annotated
+
+from mcp.server.fastmcp import FastMCP
+from pydantic import Field
+
+from ....core.client import SurfSenseClient
+from ....core.rendering import ResponseFormatParam
+from ....core.workspace_context import WorkspaceContext, WorkspaceParam
+from ..annotations import SCRAPE
+from ..capability import run_scraper
+
+
+def register(mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext) -> None:
+    """Register the Amazon product tool."""
+
+    @mcp.tool(
+        name="surfsense_amazon_scrape",
+        title="Scrape Amazon products",
+        annotations=SCRAPE,
+        structured_output=False,
+    )
+    async def amazon_scrape(
+        urls: Annotated[
+            list[str] | None,
+            Field(
+                description="Amazon product, search, category, best-seller, or short "
+                "(a.co / amzn.to) URLs. Provide urls OR search_terms."
+            ),
+        ] = None,
+        search_terms: Annotated[
+            list[str] | None,
+            Field(
+                description="Search phrases run on the Amazon domain, e.g. "
+                "['wireless earbuds']. Provide search_terms OR urls."
+            ),
+        ] = None,
+        max_items: Annotated[
+            int,
+            Field(
+                ge=1,
+                le=100,
+                description="Max products per search term or category/best-seller URL.",
+            ),
+        ] = 10,
+        domain: Annotated[
+            str,
+            Field(
+                description="Amazon marketplace domain, e.g. 'www.amazon.com', "
+                "'www.amazon.co.uk'."
+            ),
+        ] = "www.amazon.com",
+        include_details: Annotated[
+            bool,
+            Field(
+                description="Fetch full product detail pages. False returns faster "
+                "card-only results from search/category listings."
+            ),
+        ] = True,
+        max_offers: Annotated[
+            int,
+            Field(
+                ge=0,
+                le=100,
+                description="Extra marketplace offers to fetch per product. "
+                "0 returns the featured offer only.",
+            ),
+        ] = 0,
+        include_sellers: Annotated[
+            bool,
+            Field(
+                description="Enrich the featured product and each offer with the "
+                "seller's public profile summary."
+            ),
+        ] = False,
+        max_variants: Annotated[
+            int,
+            Field(
+                ge=0,
+                le=100,
+                description="Product variants (e.g. colors, sizes) to return as "
+                "separate results. 0 disables variant expansion.",
+            ),
+        ] = 0,
+        include_variant_prices: Annotated[
+            bool,
+            Field(
+                description="Attach per-variant prices (one extra request per variant)."
+            ),
+        ] = False,
+        country_code: Annotated[
+            str | None,
+            Field(description="Two-letter delivery country for localized pricing, "
+            "e.g. 'us'."),
+        ] = None,
+        zip_code: Annotated[
+            str | None,
+            Field(
+                description="Delivery ZIP/postal code for localized availability and "
+                "pricing, e.g. '10001'."
+            ),
+        ] = None,
+        language: Annotated[
+            str | None,
+            Field(description="Content language for the domain, e.g. 'en'."),
+        ] = None,
+        workspace: WorkspaceParam = None,
+        response_format: ResponseFormatParam = "markdown",
+    ) -> str:
+        """Scrape public Amazon product data by URL or search term.
+
+        Use this for product research: title, price, list price, rating and
+        review breakdown, availability, images, features, best-seller ranks,
+        marketplace offers, sellers, and on-page reviews. Only public,
+        anonymous data — no login. Provide search_terms to discover products
+        or urls to target specific products, searches, categories, or
+        best-seller pages.
+        Example: search_terms=['mechanical keyboard'], domain='www.amazon.com',
+        max_items=5, max_offers=3.
+        """
+        return await run_scraper(
+            client,
+            context,
+            platform="amazon",
+            verb="scrape",
+            payload={
+                "urls": urls,
+                "search_terms": search_terms,
+                "max_items": max_items,
+                "domain": domain,
+                "include_details": include_details,
+                "max_offers": max_offers,
+                "include_sellers": include_sellers,
+                "max_variants": max_variants,
+                "include_variant_prices": include_variant_prices,
+                "country_code": country_code,
+                "zip_code": zip_code,
+                "language": language,
+            },
+            workspace=workspace,
+            response_format=response_format,
+        )

surfsense_mcp/mcp_server/selfcheck.py

@@ -29,6 +29,7 @@ EXPECTED_TOOLS = {
     "surfsense_tiktok_trending",
     "surfsense_google_maps_scrape",
     "surfsense_google_maps_reviews",
+    "surfsense_amazon_scrape",
     "surfsense_instagram_scrape",
     "surfsense_instagram_details",
     "surfsense_list_scraper_runs",

surfsense_web/app/dashboard/[workspace_id]/playground/components/playground-runner.tsx

@@ -14,6 +14,11 @@ import { AppError } from "@/lib/error";
 import { findVerb } from "@/lib/playground/catalog";
 import { formatCost, formatDuration, formatPricing } from "@/lib/playground/format";
 import { buildPayload, initialFormValues, parseSchemaFields } from "@/lib/playground/json-schema";
+import {
+	AmazonMarketplaceHint,
+	getAmazonFieldOptions,
+	hasAmazonFranceValue,
+} from "@/lib/playground/platform-overrides/amazon";
 import { ApiReference } from "./api-reference";
 import { OutputViewer } from "./output-viewer";
 import { RunProgressPanel } from "./run-progress-panel";
@@ -171,6 +176,8 @@ export function PlaygroundRunner({ workspaceId, platform, verb }: PlaygroundRunn
 		[run.detail]
 	);
 	const endpoint = `POST /workspaces/${workspaceId}/scrapers/${platform}/${verb}`;
+	const isAmazonScrape = platform === "amazon" && verb === "scrape";
+	const hasAmazonFranceUrl = useMemo(() => hasAmazonFranceValue(values), [values]);
 
 	useEffect(() => {
 		const previousStatus = previousStatusRef.current;
@@ -260,7 +267,9 @@ export function PlaygroundRunner({ workspaceId, platform, verb }: PlaygroundRunn
 						values={values}
 						onChange={handleChange}
 						disabled={isRunning}
+						getFieldOptions={isAmazonScrape ? getAmazonFieldOptions : undefined}
 					/>
+					{isAmazonScrape ? <AmazonMarketplaceHint showFranceWarning={hasAmazonFranceUrl} /> : null}
 
 					<div className="flex items-center gap-2">
 						<Button type="button" onClick={handleRun} disabled={isRunning} className="relative">

surfsense_web/app/dashboard/[workspace_id]/playground/components/schema-form.tsx

@@ -17,11 +17,19 @@ import { Textarea } from "@/components/ui/textarea";
 import type { FormField } from "@/lib/playground/json-schema";
 import { cn } from "@/lib/utils";
 
+export interface FieldOption {
+	label: string;
+	value: string;
+}
+
+type FieldOptionsResolver = (field: FormField) => FieldOption[] | undefined;
+
 interface SchemaFormProps {
 	fields: FormField[];
 	values: Record<string, unknown>;
 	onChange: (name: string, value: unknown) => void;
 	disabled?: boolean;
+	getFieldOptions?: FieldOptionsResolver;
 	/** Field names flagged by a 422 response, shown with error styling. */
 	fieldErrors?: Record<string, string>;
 }
@@ -32,12 +40,14 @@ function FieldControl({
 	onChange,
 	disabled,
 	invalid,
+	options,
 }: {
 	field: FormField;
 	value: unknown;
 	onChange: (value: unknown) => void;
 	disabled?: boolean;
 	invalid?: boolean;
+	options?: FieldOption[];
 }) {
 	const id = `field-${field.name}`;
 
@@ -68,6 +78,27 @@ function FieldControl({
 		);
 	}
 
+	if (options) {
+		return (
+			<Select
+				value={value ? String(value) : options[0]?.value}
+				onValueChange={onChange}
+				disabled={disabled}
+			>
+				<SelectTrigger id={id} className={cn("w-full", invalid && "border-destructive")}>
+					<SelectValue placeholder="Select…" />
+				</SelectTrigger>
+				<SelectContent>
+					{options.map((option) => (
+						<SelectItem key={option.value} value={option.value}>
+							{option.label} ({option.value})
+						</SelectItem>
+					))}
+				</SelectContent>
+			</Select>
+		);
+	}
+
 	if (field.kind === "string_array") {
 		return (
 			<Textarea
@@ -116,12 +147,14 @@ function FieldRow({
 	onChange,
 	disabled,
 	error,
+	options,
 }: {
 	field: FormField;
 	value: unknown;
 	onChange: (value: unknown) => void;
 	disabled?: boolean;
 	error?: string;
+	options?: FieldOption[];
 }) {
 	return (
 		<div className="space-y-1.5">
@@ -143,13 +176,21 @@ function FieldRow({
 				onChange={onChange}
 				disabled={disabled}
 				invalid={!!error}
+				options={options}
 			/>
 			{error && <p className="text-xs text-destructive">{error}</p>}
 		</div>
 	);
 }
 
-export function SchemaForm({ fields, values, onChange, disabled, fieldErrors }: SchemaFormProps) {
+export function SchemaForm({
+	fields,
+	values,
+	onChange,
+	disabled,
+	getFieldOptions,
+	fieldErrors,
+}: SchemaFormProps) {
 	const [showAdvanced, setShowAdvanced] = useState(false);
 
 	const { primary, advanced } = useMemo(() => {
@@ -168,6 +209,7 @@ export function SchemaForm({ fields, values, onChange, disabled, fieldErrors }:
 					onChange={(value) => onChange(field.name, value)}
 					disabled={disabled}
 					error={fieldErrors?.[field.name]}
+					options={getFieldOptions?.(field)}
 				/>
 			))}
 
@@ -197,6 +239,7 @@ export function SchemaForm({ fields, values, onChange, disabled, fieldErrors }:
 									onChange={(value) => onChange(field.name, value)}
 									disabled={disabled}
 									error={fieldErrors?.[field.name]}
+									options={getFieldOptions?.(field)}
 								/>
 							))}
 						</div>

surfsense_web/components/assistant-ui/image.tsx

@@ -233,4 +233,4 @@ Image.Preview = ImagePreview;
 Image.Filename = ImageFilename;
 Image.Zoom = ImageZoom;
 
-export { Image, ImageRoot, ImagePreview, ImageFilename, ImageZoom, imageVariants };
+export { Image, ImageFilename, ImagePreview, ImageRoot, ImageZoom, imageVariants };

surfsense_web/components/new-chat/composer-suggestion-popup.tsx

@@ -181,13 +181,13 @@ function ComposerSuggestionSkeleton({
 }
 
 export {
-	ComposerSuggestionPopoverContent,
-	ComposerSuggestionList,
 	ComposerSuggestionGroup,
 	ComposerSuggestionGroupHeading,
 	ComposerSuggestionHeader,
 	ComposerSuggestionItem,
-	ComposerSuggestionSeparator,
+	ComposerSuggestionList,
 	ComposerSuggestionMessage,
+	ComposerSuggestionPopoverContent,
+	ComposerSuggestionSeparator,
 	ComposerSuggestionSkeleton,
 };

surfsense_web/components/tool-ui/generate-image.tsx

@@ -135,8 +135,8 @@ export const GenerateImageToolUI = ({
 };
 
 export {
-	GenerateImageArgsSchema,
-	GenerateImageResultSchema,
 	type GenerateImageArgs,
+	GenerateImageArgsSchema,
 	type GenerateImageResult,
+	GenerateImageResultSchema,
 };

surfsense_web/components/tool-ui/sandbox-execute.tsx

@@ -418,4 +418,4 @@ export const SandboxExecuteToolUI = ({
 	return <ExecuteCompleted command={command} parsed={parsed} threadId={threadId} />;
 };
 
-export { ExecuteArgsSchema, ExecuteResultSchema, type ExecuteArgs, type ExecuteResult };
+export { type ExecuteArgs, ExecuteArgsSchema, type ExecuteResult, ExecuteResultSchema };

surfsense_web/components/tool-ui/user-memory.tsx

@@ -90,8 +90,8 @@ export const UpdateMemoryToolUI = ({
 // ============================================================================
 
 export {
-	UpdateMemoryArgsSchema,
-	UpdateMemoryResultSchema,
 	type UpdateMemoryArgs,
+	UpdateMemoryArgsSchema,
 	type UpdateMemoryResult,
+	UpdateMemoryResultSchema,
 };

surfsense_web/components/tool-ui/write-todos.tsx

@@ -157,4 +157,4 @@ export const WriteTodosToolUI = ({
 	);
 };
 
-export { WriteTodosSchema, type WriteTodosData };
+export { type WriteTodosData, WriteTodosSchema };

surfsense_web/components/ui/accordion.tsx

@@ -61,4 +61,4 @@ function AccordionContent({
 	);
 }
 
-export { Accordion, AccordionItem, AccordionTrigger, AccordionContent };
+export { Accordion, AccordionContent, AccordionItem, AccordionTrigger };

surfsense_web/components/ui/alert-dialog.tsx

@@ -121,14 +121,14 @@ function AlertDialogCancel({
 
 export {
 	AlertDialog,
-	AlertDialogPortal,
-	AlertDialogOverlay,
-	AlertDialogTrigger,
-	AlertDialogContent,
-	AlertDialogHeader,
-	AlertDialogFooter,
-	AlertDialogTitle,
-	AlertDialogDescription,
 	AlertDialogAction,
 	AlertDialogCancel,
+	AlertDialogContent,
+	AlertDialogDescription,
+	AlertDialogFooter,
+	AlertDialogHeader,
+	AlertDialogOverlay,
+	AlertDialogPortal,
+	AlertDialogTitle,
+	AlertDialogTrigger,
 };

surfsense_web/components/ui/alert.tsx

@@ -59,4 +59,4 @@ function AlertDescription({ className, ...props }: React.ComponentProps<"div">)
 	);
 }
 
-export { Alert, AlertTitle, AlertDescription };
+export { Alert, AlertDescription, AlertTitle };

surfsense_web/components/ui/animated-tabs.tsx

@@ -556,4 +556,4 @@ const TabsContent = forwardRef<
 });
 TabsContent.displayName = "TabsContent";
 
-export { Tabs, TabsList, TabsTrigger, TabsContent };
+export { Tabs, TabsContent, TabsList, TabsTrigger };

surfsense_web/components/ui/avatar.tsx

@@ -55,4 +55,4 @@ function AvatarGroupCount({ className, ...props }: React.ComponentProps<"span">)
 	);
 }
 
-export { Avatar, AvatarImage, AvatarFallback, AvatarGroup, AvatarGroupCount };
+export { Avatar, AvatarFallback, AvatarGroup, AvatarGroupCount, AvatarImage };

surfsense_web/components/ui/card.tsx

@@ -52,4 +52,4 @@ const CardFooter = React.forwardRef<HTMLDivElement, React.HTMLAttributes<HTMLDiv
 );
 CardFooter.displayName = "CardFooter";
 
-export { Card, CardHeader, CardTitle, CardDescription, CardContent, CardFooter };
+export { Card, CardContent, CardDescription, CardFooter, CardHeader, CardTitle };

surfsense_web/components/ui/collapsible.tsx

@@ -18,4 +18,4 @@ function CollapsibleContent({
 	return <CollapsiblePrimitive.CollapsibleContent data-slot="collapsible-content" {...props} />;
 }
 
-export { Collapsible, CollapsibleTrigger, CollapsibleContent };
+export { Collapsible, CollapsibleContent, CollapsibleTrigger };

surfsense_web/components/ui/command.tsx

@@ -150,11 +150,11 @@ function CommandShortcut({ className, ...props }: React.ComponentProps<"span">)
 export {
 	Command,
 	CommandDialog,
-	CommandInput,
-	CommandList,
 	CommandEmpty,
 	CommandGroup,
+	CommandInput,
 	CommandItem,
-	CommandShortcut,
+	CommandList,
 	CommandSeparator,
+	CommandShortcut,
 };

surfsense_web/components/ui/context-menu.tsx

@@ -207,18 +207,18 @@ function ContextMenuShortcut({ className, ...props }: React.ComponentProps<"span
 
 export {
 	ContextMenu,
-	ContextMenuTrigger,
-	ContextMenuContent,
-	ContextMenuItem,
 	ContextMenuCheckboxItem,
-	ContextMenuRadioItem,
+	ContextMenuContent,
+	ContextMenuGroup,
+	ContextMenuItem,
 	ContextMenuLabel,
+	ContextMenuPortal,
+	ContextMenuRadioGroup,
+	ContextMenuRadioItem,
 	ContextMenuSeparator,
 	ContextMenuShortcut,
-	ContextMenuGroup,
-	ContextMenuPortal,
 	ContextMenuSub,
 	ContextMenuSubContent,
 	ContextMenuSubTrigger,
-	ContextMenuRadioGroup,
+	ContextMenuTrigger,
 };

surfsense_web/components/ui/dialog.tsx

@@ -92,13 +92,13 @@ DialogDescription.displayName = DialogPrimitive.Description.displayName;
 
 export {
 	Dialog,
-	DialogPortal,
-	DialogOverlay,
 	DialogClose,
-	DialogTrigger,
 	DialogContent,
-	DialogHeader,
-	DialogFooter,
-	DialogTitle,
 	DialogDescription,
+	DialogFooter,
+	DialogHeader,
+	DialogOverlay,
+	DialogPortal,
+	DialogTitle,
+	DialogTrigger,
 };

surfsense_web/components/ui/drawer.tsx

@@ -102,14 +102,14 @@ DrawerHandle.displayName = "DrawerHandle";
 
 export {
 	Drawer,
-	DrawerPortal,
-	DrawerOverlay,
-	DrawerTrigger,
 	DrawerClose,
 	DrawerContent,
-	DrawerHeader,
-	DrawerFooter,
-	DrawerTitle,
 	DrawerDescription,
+	DrawerFooter,
 	DrawerHandle,
+	DrawerHeader,
+	DrawerOverlay,
+	DrawerPortal,
+	DrawerTitle,
+	DrawerTrigger,
 };

surfsense_web/components/ui/dropdown-menu.tsx

@@ -211,18 +211,18 @@ function DropdownMenuSubContent({
 
 export {
 	DropdownMenu,
-	DropdownMenuPortal,
-	DropdownMenuTrigger,
+	DropdownMenuCheckboxItem,
 	DropdownMenuContent,
 	DropdownMenuGroup,
-	DropdownMenuLabel,
 	DropdownMenuItem,
-	DropdownMenuCheckboxItem,
+	DropdownMenuLabel,
+	DropdownMenuPortal,
 	DropdownMenuRadioGroup,
 	DropdownMenuRadioItem,
 	DropdownMenuSeparator,
 	DropdownMenuShortcut,
 	DropdownMenuSub,
-	DropdownMenuSubTrigger,
 	DropdownMenuSubContent,
+	DropdownMenuSubTrigger,
+	DropdownMenuTrigger,
 };

surfsense_web/components/ui/empty.tsx

@@ -91,4 +91,4 @@ function EmptyContent({ className, ...props }: React.ComponentProps<"div">) {
 	);
 }
 
-export { Empty, EmptyHeader, EmptyTitle, EmptyDescription, EmptyContent, EmptyMedia };
+export { Empty, EmptyContent, EmptyDescription, EmptyHeader, EmptyMedia, EmptyTitle };

surfsense_web/components/ui/expanded-gif-overlay.tsx

@@ -91,4 +91,4 @@ const ExpandedGifOverlay = ExpandedMediaOverlay;
 /** @deprecated Use useExpandedMedia instead */
 const useExpandedGif = useExpandedMedia;
 
-export { ExpandedMediaOverlay, useExpandedMedia, ExpandedGifOverlay, useExpandedGif };
+export { ExpandedGifOverlay, ExpandedMediaOverlay, useExpandedGif, useExpandedMedia };

surfsense_web/components/ui/form.tsx

@@ -140,12 +140,12 @@ function FormMessage({ className, ...props }: React.ComponentProps<"p">) {
 }
 
 export {
-	useFormField,
 	Form,
-	FormItem,
-	FormLabel,
 	FormControl,
 	FormDescription,
-	FormMessage,
 	FormField,
+	FormItem,
+	FormLabel,
+	FormMessage,
+	useFormField,
 };

surfsense_web/components/ui/pagination.tsx

@@ -96,9 +96,9 @@ function PaginationEllipsis({ className, ...props }: React.ComponentProps<"span"
 export {
 	Pagination,
 	PaginationContent,
-	PaginationLink,
-	PaginationItem,
-	PaginationPrevious,
-	PaginationNext,
 	PaginationEllipsis,
+	PaginationItem,
+	PaginationLink,
+	PaginationNext,
+	PaginationPrevious,
 };

surfsense_web/components/ui/popover.tsx

@@ -39,4 +39,4 @@ function PopoverAnchor({ ...props }: React.ComponentProps<typeof PopoverPrimitiv
 	return <PopoverPrimitive.Anchor data-slot="popover-anchor" {...props} />;
 }
 
-export { Popover, PopoverTrigger, PopoverContent, PopoverAnchor };
+export { Popover, PopoverAnchor, PopoverContent, PopoverTrigger };

surfsense_web/components/ui/sheet.tsx

@@ -122,11 +122,11 @@ function SheetDescription({
 
 export {
 	Sheet,
-	SheetTrigger,
 	SheetClose,
 	SheetContent,
-	SheetHeader,
-	SheetFooter,
-	SheetTitle,
 	SheetDescription,
+	SheetFooter,
+	SheetHeader,
+	SheetTitle,
+	SheetTrigger,
 };

surfsense_web/components/ui/table.tsx

@@ -89,4 +89,4 @@ function TableCaption({ className, ...props }: React.ComponentProps<"caption">)
 	);
 }
 
-export { Table, TableHeader, TableBody, TableFooter, TableHead, TableRow, TableCell, TableCaption };
+export { Table, TableBody, TableCaption, TableCell, TableFooter, TableHead, TableHeader, TableRow };

surfsense_web/components/ui/tabs.tsx

@@ -52,4 +52,4 @@ const TabsContent = React.forwardRef<
 ));
 TabsContent.displayName = TabsPrimitive.Content.displayName;
 
-export { Tabs, TabsList, TabsTrigger, TabsContent };
+export { Tabs, TabsContent, TabsList, TabsTrigger };

surfsense_web/components/ui/tooltip.tsx

@@ -82,4 +82,4 @@ function TooltipContent({
 	);
 }
 
-export { Tooltip, TooltipTrigger, TooltipContent, TooltipProvider };
+export { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger };

surfsense_web/content/docs/connectors/native/amazon.mdx

@@ -0,0 +1,67 @@
+---
+title: Amazon
+description: Scrape public Amazon product data as structured JSON
+---
+
+The Amazon scraper returns public product data as structured JSON: price and list price, rating and review breakdown, availability, marketplace offers, sellers, best-seller ranks, product variants, and on-page reviews. It only uses public, anonymous data — no login or seller account. Give it search terms or Amazon product, search, category, or best-seller URLs.
+
+Marketplace support is inferred from the Amazon URL or `domain` you provide. Supported public marketplaces include US (`amazon.com`), UK (`amazon.co.uk`), Germany (`amazon.de`), Italy (`amazon.it`), and Spain (`amazon.es`). France (`amazon.fr`) is supported in the scraper but remains best-effort because Amazon France is more sensitive to proxy-pool WAF challenges.
+
+## Endpoint
+
+```bash
+POST /api/v1/workspaces/{workspace_id}/scrapers/amazon/scrape
+```
+
+## Inputs
+
+| Field | Default | Description |
+|-------|---------|-------------|
+| `urls` | — | Amazon product, search, category, best-seller, or short (`a.co` / `amzn.to`) URLs. Marketplace is inferred from the URL, e.g. `amazon.co.uk`, `amazon.de`, `amazon.it`, or `amazon.es`. Provide `urls` or `search_terms` |
+| `search_terms` | — | Search phrases run on the selected `domain`, e.g. `wireless earbuds`. Provide `search_terms` or `urls` |
+| `max_items` | `10` | Max products per search term or category/best-seller URL (1–100) |
+| `domain` | `www.amazon.com` | Amazon marketplace domain for `search_terms`, e.g. `www.amazon.co.uk`, `www.amazon.de`, `www.amazon.it`, or `www.amazon.es`. `www.amazon.fr` is best-effort due to Amazon WAF sensitivity |
+| `include_details` | `true` | Fetch full product detail pages; `false` returns faster card-only results |
+| `max_offers` | `0` | Extra marketplace offers per product (0–100); `0` = featured offer only |
+| `include_sellers` | `false` | Enrich the product and each offer with the seller's public profile |
+| `max_variants` | `0` | Product variants to return as separate results (0–100) |
+| `include_variant_prices` | `false` | Attach per-variant prices (one extra request per variant) |
+| `country_code` | — | Two-letter delivery country for localized pricing, e.g. `us` |
+| `zip_code` | — | Delivery ZIP/postal code for localized availability, e.g. `10001` |
+| `language` | — | Content language for the domain, e.g. `en` |
+
+At least one of `urls` or `search_terms` is required, with up to 20 combined sources per call.
+
+## Example
+
+```bash
+curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/amazon/scrape" \
+  -H "Authorization: Bearer $SURFSENSE_API_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "search_terms": ["mechanical keyboard"],
+    "max_items": 5,
+    "max_offers": 3
+  }'
+```
+
+## Marketplace examples
+
+```json
+{
+  "urls": [
+    "https://www.amazon.co.uk/s?k=usb+c+cable",
+    "https://www.amazon.de/s?k=mechanische+tastatur",
+    "https://www.amazon.it/s?k=cavo+usb+c",
+    "https://www.amazon.es/s?k=cable+usb+c"
+  ],
+  "max_items": 5,
+  "include_details": false
+}
+```
+
+For `search_terms`, set `domain` to the marketplace you want to search. For direct URLs, the scraper infers the marketplace from each URL.
+
+The response is `{ "items": [...] }` — one item per product, plus structured error items for any input that could not be resolved. Billing is per returned product; error items are never billed.
+
+For the full input and output JSON schemas and generated code snippets in your language, open **API Playground → Amazon → Scrape** in your workspace.

surfsense_web/content/docs/connectors/native/meta.json

@@ -7,6 +7,7 @@
 		"tiktok",
 		"google-maps",
 		"google-search",
+		"amazon",
 		"web-crawl"
 	],
 	"defaultOpen": false

surfsense_web/lib/apis/connectors-api.service.ts

@@ -426,6 +426,6 @@ export interface ObsidianStats {
 	last_sync_at: string | null;
 }
 
-export type { SlackChannel, DiscordChannel };
+export type { DiscordChannel, SlackChannel };
 
 export const connectorsApiService = new ConnectorsApiService();

surfsense_web/lib/connectors-marketing/amazon.tsx

@@ -0,0 +1,320 @@
+import { IconBrandAmazon } from "@tabler/icons-react";
+import type { ConnectorPageContent } from "./types";
+
+export const amazon: ConnectorPageContent = {
+	slug: "amazon",
+	name: "Amazon",
+	cardTitle: "Amazon Product API",
+	icon: IconBrandAmazon,
+
+	metaTitle: "Amazon Product Scraper API for Price & Review Data | SurfSense",
+	metaDescription:
+		"Scrape public Amazon product data as structured JSON across US, UK, Germany, Italy, and Spain: prices, ratings, offers, sellers, and ranks. Start free.",
+	keywords: [
+		"amazon product api",
+		"amazon scraper api",
+		"amazon price scraper",
+		"scrape amazon product data",
+		"amazon product data api",
+		"amazon review scraper",
+		"amazon best sellers api",
+		"amazon asin lookup",
+		"amazon price tracking api",
+		"amazon competitor monitoring",
+		"amazon offers scraper",
+		"ecommerce product api",
+	],
+
+	h1: "Amazon Product Scraper API for Price, Review, and Offer Data",
+	heroLede:
+		"The SurfSense Amazon Product API scrapes public Amazon listings as structured JSON: price and list price, rating and review breakdown, availability, marketplace offers, sellers, best-seller ranks, and on-page reviews. Point your AI agents at a search term or product URL and track prices across US, UK, German, Italian, and Spanish marketplaces — no login, only public data.",
+
+	transcript: {
+		prompt: "Track the price and rating of the top mechanical keyboards on Amazon",
+		toolCall:
+			'amazon.scrape({ search_terms: ["mechanical keyboard"],\n  max_items: 5, max_offers: 3 })',
+		rows: [
+			{
+				primary: "Keychron K8 Pro — $89.99 (was $99.99)",
+				secondary: "4.7 stars · 12,431 ratings · In Stock",
+				tag: "-10%",
+			},
+			{
+				primary: "3 marketplace offers, cheapest $84.50 used",
+				secondary: "sold by 2 third-party sellers",
+				tag: "offers",
+			},
+			{
+				primary: "#2 in Computer Keyboards best-sellers",
+				secondary: "Electronics › Accessories › Keyboards",
+				tag: "rank",
+			},
+		],
+		resultSummary: "5 products · 14 offers · 5 best-seller ranks · surfaced in 3.1s",
+	},
+
+	extractIntro:
+		"Give the API a list of product/search/category/best-seller URLs or search terms and a domain. Marketplace is inferred from each Amazon URL, or from the domain used with search terms. Supported marketplaces include US, UK, Germany, Italy, and Spain; France is best-effort because Amazon France is more WAF-sensitive.",
+	extractFields: [
+		{
+			label: "Product core",
+			description:
+				"Title, ASIN, brand, price, list price, availability, condition, and canonical URL for every product.",
+		},
+		{
+			label: "Ratings & reviews",
+			description:
+				"Star rating, review count, the 5-to-1-star histogram, and on-page customer reviews with text, author, and date.",
+		},
+		{
+			label: "Marketplace offers",
+			description:
+				"Additional buy-box offers with price, condition, delivery, and the third-party seller behind each.",
+		},
+		{
+			label: "Sellers",
+			description:
+				"Public seller profile summaries — name, rating, and feedback count — for the featured and offer sellers.",
+		},
+		{
+			label: "Best-seller ranks",
+			description:
+				"Category rank positions and best-seller list placements, for demand and category monitoring.",
+		},
+		{
+			label: "Variants & media",
+			description:
+				"Variant ASINs and attributes (color, size), per-variant prices, gallery and high-resolution images.",
+		},
+	],
+
+	useCasesHeading: "What teams do with the Amazon Product API",
+	useCases: [
+		{
+			title: "Price and buy-box tracking",
+			description:
+				"Watch your own and competitors' prices, list prices, and marketplace offers across US, UK, German, Italian, and Spanish marketplaces. Feed each run to an agent that diffs prices and alerts you when a competitor undercuts or the buy box changes hands.",
+		},
+		{
+			title: "Review mining and product research",
+			description:
+				"Pull ratings, the star histogram, and on-page reviews to understand what customers love and complain about, then brief an agent to summarize themes and inform your roadmap or listing copy.",
+		},
+		{
+			title: "Best-seller and category monitoring",
+			description:
+				"Track best-seller ranks in the categories you care about to spot rising products and demand shifts before they show up anywhere else.",
+		},
+		{
+			title: "Catalog and seller intelligence",
+			description:
+				"Enrich a catalog by ASIN with structured attributes, variants, and images, and see which third-party sellers are winning offers on the products that matter to you.",
+		},
+	],
+
+	comparison: {
+		heading: "An Amazon scraper API built for agents",
+		intro:
+			"Most Amazon data APIs bill per request, meter add-ons separately, and leave the discovery-to-detail logic to you. Here is how SurfSense compares.",
+		columnLabel: "Typical Amazon data API",
+		rows: [
+			{
+				feature: "Pricing",
+				official: "Per-request pricing that climbs fast at scale",
+				surfsense: "Pay per product returned, with a free tier to start",
+			},
+			{
+				feature: "Discovery",
+				official: "Separate search and detail endpoints you stitch together",
+				surfsense: "One verb takes search terms, product, category, or best-seller URLs",
+			},
+			{
+				feature: "Offers & sellers",
+				official: "Often separate paid endpoints",
+				surfsense: "Offers and seller profiles enriched inline on the product",
+			},
+			{
+				feature: "Data access",
+				official: "Requires accounts, keys, or approval for many providers",
+				surfsense: "Public, anonymous data only — no login or seller account",
+			},
+			{
+				feature: "Agent-ready",
+				official: "No; you wire the harness yourself",
+				surfsense: "MCP server exposes amazon.scrape as a native tool",
+			},
+		],
+	},
+
+	api: {
+		platform: "amazon",
+		verb: "scrape",
+		mcpTool: "amazon.scrape",
+		requestBody: {
+			search_terms: ["mechanical keyboard"],
+			max_items: 5,
+			max_offers: 3,
+		},
+	},
+
+	schema: {
+		requestNote:
+			"Provide urls or search_terms (at least one). Marketplace is inferred from Amazon URLs, or from domain for search terms. Up to 20 combined sources per call.",
+		request: [
+			{
+				name: "urls",
+				type: "string[]",
+				description:
+					"Amazon product, search, category, best-seller, or short (a.co / amzn.to) URLs. Supports amazon.com, amazon.co.uk, amazon.de, amazon.it, and amazon.es; amazon.fr is best-effort. Provide urls or search_terms.",
+			},
+			{
+				name: "search_terms",
+				type: "string[]",
+				description:
+					"Search phrases run on the Amazon domain, e.g. 'wireless earbuds'. Provide search_terms or urls.",
+			},
+			{
+				name: "max_items",
+				type: "integer",
+				defaultValue: "10",
+				description: "Max products per search term or category/best-seller URL, 1 to 100.",
+			},
+			{
+				name: "domain",
+				type: "string",
+				defaultValue: '"www.amazon.com"',
+				description:
+					"Amazon marketplace domain for search_terms, e.g. 'www.amazon.co.uk', 'www.amazon.de', 'www.amazon.it', or 'www.amazon.es'. 'www.amazon.fr' is best-effort due to WAF sensitivity.",
+			},
+			{
+				name: "include_details",
+				type: "boolean",
+				defaultValue: "true",
+				description: "Fetch full product detail pages. false returns faster card-only results.",
+			},
+			{
+				name: "max_offers",
+				type: "integer",
+				defaultValue: "0",
+				description:
+					"Extra marketplace offers to fetch per product, 0 to 100. 0 returns the featured offer only.",
+			},
+			{
+				name: "include_sellers",
+				type: "boolean",
+				defaultValue: "false",
+				description: "Enrich the product and each offer with the seller's public profile.",
+			},
+			{
+				name: "max_variants",
+				type: "integer",
+				defaultValue: "0",
+				description: "Product variants to return as separate results, 0 to 100.",
+			},
+			{
+				name: "include_variant_prices",
+				type: "boolean",
+				defaultValue: "false",
+				description: "Attach per-variant prices (one extra request per variant).",
+			},
+			{
+				name: "country_code",
+				type: "string",
+				description: "Two-letter delivery country for localized pricing, e.g. 'us'.",
+			},
+			{
+				name: "zip_code",
+				type: "string",
+				description: "Delivery ZIP/postal code for localized availability, e.g. '10001'.",
+			},
+			{
+				name: "language",
+				type: "string",
+				description: "Content language for the domain, e.g. 'en'.",
+			},
+		],
+		responseNote:
+			"The response is { items: [...] } with one item per product. One returned product is one billable unit; error items are never billed.",
+		response: [
+			{
+				name: "title / asin / brand",
+				type: "string",
+				description: "Product identity: display title, ASIN, and brand.",
+			},
+			{
+				name: "price / listPrice",
+				type: "object",
+				description: "Current price and strike-through list price, each with value and currency.",
+			},
+			{
+				name: "stars / reviewsCount / starsBreakdown",
+				type: "object",
+				description:
+					"Average rating, total review count, and the 5-to-1-star distribution as fractions.",
+			},
+			{
+				name: "offers",
+				type: "object[]",
+				description:
+					"Marketplace offers with price, condition, delivery, seller, and pinned-offer flag.",
+			},
+			{
+				name: "seller",
+				type: "object",
+				description: "Featured seller profile: id, name, url, rating, and feedback count.",
+			},
+			{
+				name: "bestsellerRanks",
+				type: "object[]",
+				description: "Category rank positions and best-seller list placements.",
+			},
+			{
+				name: "variantAsins / variantDetails / variantAttributes",
+				type: "object[]",
+				description: "Related variant ASINs and their attributes and per-variant prices.",
+			},
+			{
+				name: "productPageReviews",
+				type: "object[]",
+				description: "On-page customer reviews with text, author, rating, and date.",
+			},
+		],
+	},
+
+	faq: [
+		{
+			question: "What is an Amazon product API?",
+			answer:
+				"An Amazon product API returns a listing's data as structured JSON instead of raw HTML. The SurfSense Amazon API scrapes public product pages and gives you price, rating, review breakdown, offers, sellers, and best-seller ranks as clean JSON your agents can read.",
+		},
+		{
+			question: "Can I track Amazon prices with it?",
+			answer:
+				"Yes. Each product returns its current price, list price, and marketplace offers. Run the same URLs or search terms on a schedule and diff the prices to build price and buy-box tracking, without maintaining scrapers or proxies yourself.",
+		},
+		{
+			question: "Does it only use public data?",
+			answer:
+				"It does. The scraper collects public, anonymous product data only — no login, seller account, or authenticated APIs. That covers product details, ratings, on-page reviews, offers, public seller profiles, and best-seller rankings.",
+		},
+		{
+			question: "How do I look up a product by ASIN or search term?",
+			answer:
+				"Pass a product URL (which contains the ASIN) in urls, or a phrase in search_terms to discover products on the domain. You can also pass search, category, and best-seller URLs. One verb, amazon.scrape, handles all of them.",
+		},
+		{
+			question: "Which Amazon marketplaces are supported?",
+			answer:
+				"Marketplace routing is inferred from the Amazon URL or domain. SurfSense supports public Amazon data from US, UK, Germany, Italy, and Spain. France is wired in and retried like the others, but remains best-effort because Amazon France is more sensitive to proxy-pool WAF challenges.",
+		},
+	],
+
+	related: [
+		{ label: "Google Search API", href: "/google-search" },
+		{ label: "Google Maps API", href: "/google-maps" },
+		{ label: "Web Crawl API", href: "/web-crawl" },
+		{ label: "Reddit API", href: "/reddit" },
+		{ label: "SurfSense MCP Server", href: "/mcp-server" },
+		{ label: "Read the docs", href: "/docs" },
+	],
+};

surfsense_web/lib/connectors-marketing/index.ts

@@ -1,3 +1,4 @@
+import { amazon } from "./amazon";
 import { googleMaps } from "./google-maps";
 import { googleSearch } from "./google-search";
 import { instagram } from "./instagram";
@@ -17,6 +18,7 @@ const CONNECTOR_LIST: ConnectorPageContent[] = [
 	tiktok,
 	googleMaps,
 	googleSearch,
+	amazon,
 	webCrawl,
 ];
 

surfsense_web/lib/playground/catalog.ts

@@ -1,5 +1,6 @@
 import type { ComponentType } from "react";
 import {
+	AmazonIcon,
 	GoogleMapsIcon,
 	GoogleSearchIcon,
 	InstagramIcon,
@@ -89,6 +90,12 @@ export const PLAYGROUND_PLATFORMS: PlaygroundPlatform[] = [
 		icon: GoogleSearchIcon,
 		verbs: [{ name: "google_search.scrape", verb: "scrape", label: "Scrape" }],
 	},
+	{
+		id: "amazon",
+		label: "Amazon",
+		icon: AmazonIcon,
+		verbs: [{ name: "amazon.scrape", verb: "scrape", label: "Scrape" }],
+	},
 	{
 		id: "web",
 		label: "Web",

surfsense_web/lib/playground/platform-icons.tsx

@@ -22,6 +22,7 @@ function brandIcon(src: string, alt: string) {
 	};
 }
 
+export const AmazonIcon = brandIcon("/connectors/amazon.svg", "Amazon");
 export const RedditIcon = brandIcon("/connectors/reddit.svg", "Reddit");
 export const YouTubeIcon = brandIcon("/connectors/youtube.svg", "YouTube");
 export const InstagramIcon = brandIcon("/connectors/instagram.svg", "Instagram");

surfsense_web/lib/playground/platform-overrides/amazon.tsx

@@ -0,0 +1,38 @@
+import { Info } from "lucide-react";
+import type { FieldOption } from "@/app/dashboard/[workspace_id]/playground/components/schema-form";
+import { Alert, AlertDescription } from "@/components/ui/alert";
+import type { FormField } from "@/lib/playground/json-schema";
+
+export const AMAZON_DOMAIN_OPTIONS: FieldOption[] = [
+	{ label: "US", value: "www.amazon.com" },
+	{ label: "UK", value: "www.amazon.co.uk" },
+	{ label: "Germany", value: "www.amazon.de" },
+	{ label: "Italy", value: "www.amazon.it" },
+	{ label: "Spain", value: "www.amazon.es" },
+	{ label: "France, best effort", value: "www.amazon.fr" },
+];
+
+const AMAZON_SUPPORTED_COUNTRIES = "US, UK, Germany, Italy, Spain, France";
+
+export function getAmazonFieldOptions(field: FormField): FieldOption[] | undefined {
+	return field.name === "domain" ? AMAZON_DOMAIN_OPTIONS : undefined;
+}
+
+export function hasAmazonFranceValue(values: Record<string, unknown>): boolean {
+	return JSON.stringify(values).toLowerCase().includes("amazon.fr");
+}
+
+export function AmazonMarketplaceHint({ showFranceWarning }: { showFranceWarning: boolean }) {
+	return (
+		<Alert>
+			<Info />
+			<AlertDescription className="flex flex-wrap items-baseline gap-x-1">
+				<span className="font-medium text-foreground">Supported countries: </span>
+				<span>{AMAZON_SUPPORTED_COUNTRIES}</span>
+				{showFranceWarning
+					? " France is more WAF-sensitive. If this run returns no results, retry later or use another marketplace."
+					: null}
+			</AlertDescription>
+		</Alert>
+	);
+}

surfsense_web/public/connectors/amazon.svg

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
+<svg width="800px" height="800px" viewBox="0 0 48 48" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+    
+    <title>Amazon-color</title>
+    <desc>Created with Sketch.</desc>
+    <defs>
+
+</defs>
+    <g id="Icons" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
+        <g id="Color-" transform="translate(-601.000000, -560.000000)">
+            <g id="Amazon" transform="translate(601.000000, 560.000000)">
+                <path d="M25.4026553,25.9595294 C24.660417,27.4418824 23.3876054,28.3962353 22.0103725,28.7181176 C21.8015298,28.7181176 21.4826213,28.8225882 21.1637129,28.8225882 C18.835399,28.8225882 17.458166,27.0211765 17.458166,24.3727059 C17.458166,20.9788235 19.4703937,19.392 22.0103725,18.6465882 C23.3876054,18.3303529 24.9793255,18.2230588 26.5682233,18.2230588 L26.5682233,19.4964706 C26.5682233,21.9331765 26.6726447,23.8390588 25.4026553,25.9595294 L25.4026553,25.9595294 Z M26.5682233,13.3524706 C25.1909904,13.4569412 23.5992703,13.5614118 22.0103725,13.7703529 C19.574815,14.0922353 17.1392576,14.5157647 15.1298521,15.4701176 C11.2098182,17.0597647 8.55977364,20.4508235 8.55977364,25.4287059 C8.55977364,31.6856471 12.5842289,34.8621176 17.6726531,34.8621176 C19.3659723,34.8621176 20.7432053,34.6475294 22.0103725,34.3341176 C24.0282445,33.696 25.7187415,32.5298824 27.7309692,30.4094118 C28.8965372,31.9990588 29.2182679,32.7444706 31.2276733,34.4385882 C31.7582467,34.6475294 32.28882,34.6475294 32.7093276,34.3341176 C33.9821392,33.2724706 36.208854,31.3637647 37.3715998,30.3049412 C37.9021732,29.8814118 37.7977518,29.2432941 37.4760212,28.7181176 C36.3132753,27.2329412 35.1448851,25.9595294 35.1448851,23.0992941 L35.1448851,13.5614118 C35.1448851,9.53505882 35.4666157,5.82494118 32.5004849,3.072 C30.0649275,0.849882353 26.2493149,0 23.2831841,0 L22.0103725,0 C16.6115064,0.313411765 10.8937319,2.64564706 9.61809814,9.32329412 C9.40643324,10.1731765 10.0442501,10.4894118 10.4675799,10.5938824 L16.3998415,11.3364706 C17.0348362,11.2291765 17.3537447,10.6983529 17.458166,10.1731765 C17.9859172,7.84094118 19.8937235,6.67482353 22.0103725,6.46023529 L22.4365245,6.46023529 C23.7093361,6.46023529 25.086569,6.99105882 25.8259851,8.05270588 C26.6726447,9.32329412 26.5682233,11.0202353 26.5682233,12.5054118 L26.5682233,13.3524706 L26.5682233,13.3524706 Z" fill="#343B45">
+
+</path>
+                <path d="M47.9943556,35.9463529 L47.9943556,35.9435294 C47.971778,35.4437647 47.8673567,35.0625882 47.658514,34.7463529 L47.6359364,34.7152941 L47.6105366,34.6842353 C47.3988717,34.4527059 47.1956734,34.3651765 46.9755419,34.2691765 C46.3179696,34.0150588 45.3612442,33.8795294 44.2097872,33.8767059 C43.382883,33.8767059 42.4713128,33.9557647 41.5540982,34.1562353 L41.551276,34.0941176 L40.6284171,34.4018824 L40.6114839,34.4103529 L40.0893771,34.5797647 L40.0893771,34.6023529 C39.47696,34.8564706 38.9209869,35.1727059 38.4045245,35.5482353 C38.0827939,35.7882353 37.8175072,36.1072941 37.8033962,36.5957647 C37.7949296,36.8611765 37.9303952,37.1661176 38.1533489,37.3468235 C38.3763025,37.5275294 38.6359448,37.5896471 38.8645429,37.5896471 C38.9181647,37.5896471 38.9689643,37.5868235 39.0141194,37.5783529 L39.0592746,37.5755294 L39.093141,37.5698824 C39.5446928,37.4738824 40.2022651,37.4089412 40.9727253,37.3016471 C41.6331198,37.2282353 42.3330251,37.1745882 42.9397978,37.1745882 C43.368772,37.1717647 43.7554132,37.2028235 44.0206999,37.2592941 C44.1533432,37.2875294 44.2521202,37.3214118 44.3057419,37.3496471 C44.3254973,37.3552941 44.3396083,37.3637647 44.3480749,37.3694118 C44.3593637,37.4061176 44.3762969,37.5021176 44.3734747,37.6348235 C44.3791191,38.1430588 44.164632,39.0861176 43.8683012,40.0065882 C43.5804369,40.9270588 43.2304843,41.8503529 42.999064,42.4630588 C42.94262,42.6042353 42.9059314,42.7595294 42.9059314,42.9289412 C42.900287,43.1745882 43.0018862,43.4738824 43.2163733,43.6715294 C43.425216,43.8691765 43.696147,43.9482353 43.9219229,43.9482353 L43.9332117,43.9482353 C44.2718756,43.9454118 44.5597398,43.8098824 44.8080933,43.6150588 C47.1505182,41.5087059 47.9661336,38.1430588 48,36.2484706 L47.9943556,35.9463529 Z M41.0489247,38.8658824 C40.8090378,38.8630588 40.5635065,38.9195294 40.3349084,39.0268235 C40.0780883,39.1284706 39.8156239,39.2470588 39.5672704,39.3515294 L39.2032068,39.504 L38.7290774,39.6931765 L38.7290774,39.6988235 C33.5785648,41.7882353 28.16841,43.0136471 23.1618295,43.1209412 C22.9783866,43.1265882 22.7921215,43.1265882 22.614323,43.1265882 C14.7403887,43.1322353 8.31706456,39.4785882 1.83729642,35.8785882 C1.61152053,35.76 1.37727804,35.6978824 1.15150215,35.6978824 C0.860815683,35.6978824 0.561662624,35.808 0.344353327,36.0112941 C0.12704403,36.2174118 -0.00277710907,36.5138824 4.50895989e-05,36.816 C-0.00277710907,37.2084706 0.208887791,37.5698824 0.505218651,37.8042353 C6.58705678,43.0870588 13.25309,47.9943529 22.2192152,48 C22.3941915,48 22.57199,47.9943529 22.7497885,47.9915294 C28.453452,47.8644706 34.902176,45.936 39.9087564,42.7905882 L39.9398006,42.7708235 C40.5945507,42.3783529 41.2493008,41.9322353 41.8673623,41.4381176 C42.2511813,41.1529412 42.516468,40.7068235 42.516468,40.2437647 C42.4995348,39.4221176 41.8024517,38.8658824 41.0489247,38.8658824 Z" id="Fill-237" fill="#FF9A00">
+
+</path>
+            </g>
+        </g>
+    </g>
+</svg>

surfsense_web/public/connectors/google-calendar.svg

@@ -1 +1 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="256" height="256" viewBox="0 0 256 256"><path fill="#fff" d="M195.368 60.632H60.632v134.736h134.736z"/><path fill="#ea4335" d="M195.368 256L256 195.368l-30.316-5.172l-30.316 5.172l-5.533 27.73z"/><path fill="#188038" d="M0 195.368v40.421C0 246.956 9.044 256 20.21 256h40.422l6.225-30.316l-6.225-30.316l-33.033-5.172z"/><path fill="#1967d2" d="M256 60.632V20.21C256 9.044 246.956 0 235.79 0h-40.422q-5.532 22.554-5.533 33.196q0 10.641 5.533 27.436q20.115 5.76 30.316 5.76T256 60.631"/><path fill="#fbbc04" d="M256 60.632h-60.632v134.736H256z"/><path fill="#34a853" d="M195.368 195.368H60.632V256h134.736z"/><path fill="#4285f4" d="M195.368 0H20.211C9.044 0 0 9.044 0 20.21v175.158h60.632V60.632h134.736z"/><path fill="#4285f4" d="M88.27 165.154c-5.036-3.402-8.523-8.37-10.426-14.94l11.689-4.816q1.59 6.063 5.558 9.398c2.627 2.223 5.827 3.318 9.566 3.318q5.734 0 9.852-3.487c2.746-2.324 4.127-5.288 4.127-8.875q0-5.508-4.345-8.994c-2.897-2.324-6.535-3.486-10.88-3.486h-6.754v-11.57h6.063q5.608 0 9.448-3.033c2.56-2.02 3.84-4.783 3.84-8.303c0-3.132-1.145-5.625-3.435-7.494c-2.29-1.87-5.188-2.813-8.708-2.813c-3.436 0-6.164.91-8.185 2.745a16.1 16.1 0 0 0-4.413 6.754l-11.57-4.817c1.532-4.345 4.345-8.185 8.471-11.503s9.398-4.985 15.798-4.985c4.733 0 8.994.91 12.767 2.745c3.772 1.836 6.736 4.379 8.875 7.613c2.14 3.25 3.2 6.888 3.2 10.93c0 4.126-.993 7.613-2.98 10.476s-4.43 5.052-7.327 6.585v.69a22.25 22.25 0 0 1 9.398 7.327c2.442 3.284 3.672 7.208 3.672 11.79c0 4.58-1.163 8.673-3.487 12.26c-2.324 3.588-5.54 6.417-9.617 8.472c-4.092 2.055-8.69 3.1-13.793 3.1c-5.912.016-11.369-1.685-16.405-5.087m71.797-58.005l-12.833 9.28l-6.417-9.734l23.023-16.607h8.825v78.333h-12.598z"/></svg>
+<svg xmlns="http://www.w3.org/2000/svg" width="192" height="192" fill="none"><path fill="#bbe2ff" d="M32 36.8C32 20.894 44.894 8 60.8 8h70.4C147.106 8 160 20.894 160 36.8v30.4c0 15.906-12.894 28.8-28.8 28.8H60.8C44.894 96 32 83.106 32 67.2z"/><path fill="#3c90ff" d="M19.867 49.392C17.818 33.82 29.94 20 45.645 20h100.71c15.706 0 27.827 13.82 25.778 29.392L166 96l6.133 46.608C174.182 158.18 162.061 172 146.355 172H45.645c-15.706 0-27.827-13.82-25.778-29.392L26 96z"/><mask id="a" width="154" height="152" x="19" y="20" maskUnits="userSpaceOnUse" style="mask-type:alpha"><path fill="#3c90ff" d="M19.867 49.392C17.818 33.82 29.94 20 45.645 20h100.71c15.706 0 27.827 13.82 25.778 29.392L166 96l6.133 46.608C174.182 158.18 162.061 172 146.355 172H45.645c-15.706 0-27.827-13.82-25.778-29.392L26 96z"/></mask><g mask="url(#a)"><path fill="url(#b)" d="M0 0h166v76H0z" transform="matrix(1 0 0 -1 13 172)"/></g><mask id="c" width="154" height="152" x="19" y="20" maskUnits="userSpaceOnUse" style="mask-type:alpha"><path fill="#3186ff" d="M19.867 49.392C17.818 33.82 29.94 20 45.645 20h100.71c15.706 0 27.827 13.82 25.778 29.392L166 96l6.133 46.608C174.182 158.18 162.061 172 146.355 172H45.645c-15.706 0-27.827-13.82-25.778-29.392L26 96z"/></mask><g mask="url(#c)"><path fill="url(#d)" d="M32 27.2C32 16.596 40.596 8 51.2 8h89.6c10.604 0 19.2 8.596 19.2 19.2V96H32z" filter="url(#e)"/></g><path fill="#fff" d="M75.353 133.336q-6.282 0-10.777-2.043t-7.61-5.465q-3.065-3.474-4.342-6.793T51.603 115a2.07 2.07 0 0 1 1.021-1.124l5.67-2.247q.714-.357 1.43-.102.714.204 1.685 2.349 1.022 2.145 2.86 4.546a14.3 14.3 0 0 0 4.495 3.728q2.606 1.328 6.435 1.328 6.18 0 9.807-3.575 3.677-3.575 3.677-9.091 0-5.976-3.882-9.194-3.881-3.269-10.266-3.269h-5.362a1.9 1.9 0 0 1-1.328-.51q-.51-.562-.511-1.277v-5.465q0-.767.51-1.277a1.82 1.82 0 0 1 1.329-.562h4.647q5.721 0 9.194-3.116t3.473-8.07q0-4.902-3.116-7.916t-8.58-3.014q-3.065 0-5.312 1.022a11.5 11.5 0 0 0-3.882 2.86 22.7 22.7 0 0 0-2.809 3.78q-1.174 1.941-1.89 2.145-.714.153-1.379-.255l-5.363-2.605q-.664-.358-.868-1.124t1.226-3.575q1.481-2.86 4.494-5.823a21 21 0 0 1 7.049-4.597q4.035-1.635 9.398-1.634 9.96 0 15.782 5.26 5.823 5.21 5.823 13.791 0 5.925-2.86 10.266-2.81 4.34-7.968 6.13v.204q6.231 1.838 9.806 6.741 3.627 4.853 3.626 11.594 0 9.654-6.742 15.834-6.74 6.18-17.57 6.18zm51.25-1.175q-.868 0-1.533-.664a2.25 2.25 0 0 1-.612-1.583V73.118l-11.492 8.274q-.614.46-1.431.307a1.96 1.96 0 0 1-1.225-.766l-3.32-4.7a1.98 1.98 0 0 1-.358-1.43q.153-.816.817-1.276l20.379-14.557q.256-.204.562-.306.307-.153.715-.153h4.291q.868 0 1.379.613.562.56.562 1.43v69.36q0 .92-.664 1.583a2 2 0 0 1-1.533.664z"/><defs><linearGradient id="b" x1="83" x2="83" y1="76" gradientUnits="userSpaceOnUse"><stop stop-color="#4fa0ff"/><stop offset="1" stop-color="#3186ff"/></linearGradient><linearGradient id="d" x1="89.06" x2="89.06" y1="21.75" y2="96.39" gradientUnits="userSpaceOnUse"><stop stop-color="#a9a8ff"/><stop offset=".8" stop-color="#3c90ff"/></linearGradient><filter id="e" width="152" height="112" x="20" y="-4" color-interpolation-filters="sRGB" filterUnits="userSpaceOnUse"><feFlood flood-opacity="0" result="BackgroundImageFix"/><feBlend in="SourceGraphic" in2="BackgroundImageFix" result="shape"/><feGaussianBlur result="effect1_foregroundBlur_37330_7673" stdDeviation="6"/></filter></defs></svg>

surfsense_web/public/connectors/google-drive.svg

@@ -1 +1 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="256" height="229" viewBox="0 0 256 229"><path fill="#0066da" d="m19.354 196.034l11.29 19.5c2.346 4.106 5.718 7.332 9.677 9.678q17.009-21.591 23.68-33.137q6.77-11.717 16.641-36.655q-26.604-3.502-40.32-3.502q-13.165 0-40.322 3.502c0 4.545 1.173 9.09 3.519 13.196z"/><path fill="#ea4335" d="M215.681 225.212c3.96-2.346 7.332-5.572 9.677-9.677l4.692-8.064l22.434-38.855a26.57 26.57 0 0 0 3.518-13.196q-27.315-3.502-40.247-3.502q-13.899 0-40.248 3.502q9.754 25.075 16.422 36.655q6.724 11.683 23.752 33.137"/><path fill="#00832d" d="M128.001 73.311q19.68-23.768 27.125-36.655q5.996-10.377 13.196-33.137C164.363 1.173 159.818 0 155.126 0h-54.25C96.184 0 91.64 1.32 87.68 3.519q9.16 26.103 15.544 37.154q7.056 12.213 24.777 32.638"/><path fill="#2684fc" d="M175.36 155.42H80.642l-40.32 69.792c3.958 2.346 8.503 3.519 13.195 3.519h148.968c4.692 0 9.238-1.32 13.196-3.52z"/><path fill="#00ac47" d="M128.001 73.311L87.681 3.52c-3.96 2.346-7.332 5.571-9.678 9.677L3.519 142.224A26.57 26.57 0 0 0 0 155.42h80.642z"/><path fill="#ffba00" d="m215.242 77.71l-37.243-64.514c-2.345-4.106-5.718-7.331-9.677-9.677l-40.32 69.792l47.358 82.109h80.496c0-4.546-1.173-9.09-3.519-13.196z"/></svg>
+<svg xmlns="http://www.w3.org/2000/svg" width="192" height="192" fill="none"><mask id="a" width="168" height="154" x="12" y="18" maskUnits="userSpaceOnUse" style="mask-type:alpha"><path fill="#b43333" d="M63.09 37c14.626-25.333 51.193-25.334 65.819 0l45.033 78c14.626 25.334-3.657 57.001-32.91 57.001H50.967c-29.253 0-47.536-31.667-32.91-57.001z"/></mask><g mask="url(#a)"><path fill="url(#b)" d="M206.905 172.02h-91.888l-19.015-32.934 45.944-79.578z"/><path fill="url(#c)" d="M-14.919 172.006 50.04 59.494v.002L31.032 92.422h38.02L115 172.004l-129.918.001z"/><path fill="url(#d)" d="M96.007-20.085 141.954 59.5l-19.011 32.928H31.048z"/></g><defs><linearGradient id="b" x1="193.6" x2="103.09" y1="165.6" y2="111.21" gradientUnits="userSpaceOnUse"><stop offset=".09" stop-color="#ffe921"/><stop offset="1" stop-color="#fec700"/></linearGradient><linearGradient id="c" x1="114.4" x2="15.53" y1="181.61" y2="121.8" gradientUnits="userSpaceOnUse"><stop offset=".15" stop-color="#a9a8ff"/><stop offset=".33" stop-color="#6d97ff"/><stop offset=".48" stop-color="#3186ff"/></linearGradient><linearGradient id="d" x1="128.88" x2="28.7" y1="37.88" y2="84.64" gradientUnits="userSpaceOnUse"><stop offset=".55" stop-color="#0ebc5f"/><stop offset=".85" stop-color="#78c9ff"/></linearGradient></defs></svg>

surfsense_web/public/connectors/google-gmail.svg

@@ -1 +1 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="256" height="193" viewBox="0 0 256 193"><path fill="#4285f4" d="M58.182 192.05V93.14L27.507 65.077L0 49.504v125.091c0 9.658 7.825 17.455 17.455 17.455z"/><path fill="#34a853" d="M197.818 192.05h40.727c9.659 0 17.455-7.826 17.455-17.455V49.505l-31.156 17.837l-27.026 25.798z"/><path fill="#ea4335" d="m58.182 93.14l-4.174-38.647l4.174-36.989L128 69.868l69.818-52.364l4.669 34.992l-4.669 40.644L128 145.504z"/><path fill="#fbbc04" d="M197.818 17.504V93.14L256 49.504V26.231c0-21.585-24.64-33.89-41.89-20.945z"/><path fill="#c5221f" d="m0 49.504l26.759 20.07L58.182 93.14V17.504L41.89 5.286C24.61-7.66 0 4.646 0 26.23z"/></svg>
+<svg xmlns="http://www.w3.org/2000/svg" width="192" height="192" fill="none" viewBox="0 0 192 192"><path fill="url(#a)" d="M146 44h38v110c0 6.627-5.373 12-12 12h-20a6 6 0 0 1-6-6z"/><path fill="#fc413d" d="M46 44H8v110c0 6.627 5.373 12 12 12h20a6 6 0 0 0 6-6z"/><path fill="url(#b)" d="M39.226 30.456c-8.033-6.752-20.018-5.714-26.77 2.319-6.752 8.032-5.714 20.017 2.319 26.77l76.078 63.949a8 8 0 0 0 10.295 0l76.078-63.95c8.032-6.752 9.07-18.737 2.318-26.77-6.752-8.032-18.737-9.07-26.769-2.318L96 78.18z"/><defs><linearGradient id="a" x1="165" x2="165" y1="44" y2="166" gradientUnits="userSpaceOnUse"><stop stop-color="#60d673"/><stop offset=".17" stop-color="#42c868"/><stop offset=".39" stop-color="#0ebc5f"/><stop offset=".62" stop-color="#00a9bb"/><stop offset=".86" stop-color="#3c90ff"/><stop offset="1" stop-color="#3186ff"/></linearGradient><linearGradient id="b" x1="8" x2="184" y1="46.13" y2="46.13" gradientUnits="userSpaceOnUse"><stop offset=".08" stop-color="#ff63a0"/><stop offset=".3" stop-color="#fc413d"/><stop offset=".5" stop-color="#fc413d"/><stop offset=".65" stop-color="#fc413d"/><stop offset=".72" stop-color="#fc5c30"/><stop offset=".86" stop-color="#feb10c"/><stop offset=".91" stop-color="#fec700"/><stop offset=".96" stop-color="#ffdb0f"/></linearGradient></defs></svg>