feat(amazon): add scraper orchestrator flows

surfsense_backend/app/proprietary/platforms/amazon/scraper.py

@@ -0,0 +1,485 @@
+"""Orchestrate public Amazon product discovery and enrichment.
+
+The streaming core dispatches each start URL to a product, search, best-sellers,
+or shortened-link flow. Network and parsing concerns remain isolated in their
+own modules so markup changes and retry policy can be tested independently.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import AsyncIterator
+from typing import Any
+from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
+
+from .fetch import (
+    fetch_aod_html,
+    fetch_page,
+    fetch_seller_html,
+    gather_bounded,
+    get_location_session,
+    resolve_shortlink,
+    should_localize,
+)
+from .parsers import (
+    parse_aod_offers,
+    parse_bestsellers_page,
+    parse_product,
+    parse_search_page,
+    parse_seller,
+)
+from .schemas import AmazonScrapeInput, ErrorItem, ProductItem
+from .url_resolver import ResolvedUrl, resolve_url
+
+__all__ = ["iter_products", "scrape_products"]
+
+_DETAIL_CONCURRENCY = 8
+_SEARCH_PAGE_LIMIT = 7
+_DEFAULT_ITEMS_PER_START_URL = 100
+
+
+def _error(
+    code: str, description: str, *, input_url: str | None, url: str | None = None
+) -> dict[str, Any]:
+    return ErrorItem(
+        error=code,
+        errorDescription=description,
+        input=input_url,
+        url=url or input_url,
+    ).model_dump()
+
+
+def _page_url(url: str, page: int) -> str:
+    parsed = urlparse(url)
+    query = dict(parse_qsl(parsed.query, keep_blank_values=True))
+    query["page"] = str(page)
+    return urlunparse(parsed._replace(query=urlencode(query)))
+
+
+async def _location_context(
+    resolved: ResolvedUrl, input_model: AmazonScrapeInput, route: str
+) -> tuple[dict[str, str] | None, str | None, str | None, str | None]:
+    if (
+        not input_model.zipCode
+        or not resolved.domain
+        or not should_localize(route, input_model.locationDeliverableRoutes)
+    ):
+        return None, None, None, None
+    session = await get_location_session(
+        resolved.domain,
+        zip_code=input_model.zipCode,
+        country_code=input_model.countryCode,
+    )
+    if session is None:
+        return None, None, None, None
+    return (
+        session.cookies,
+        session.proxy,
+        session.location_text,
+        session.country_code,
+    )
+
+
+async def _product_flow(
+    resolved: ResolvedUrl,
+    input_model: AmazonScrapeInput,
+    *,
+    provenance: dict[str, Any] | None = None,
+    allow_variants: bool = True,
+) -> AsyncIterator[dict[str, Any]]:
+    """Fetch and enrich one product detail page."""
+    asin = resolved.asin
+    if not asin or not resolved.domain:
+        yield _error(
+            "product_not_found",
+            "The product URL did not contain a valid ASIN.",
+            input_url=resolved.url,
+        )
+        return
+
+    cookies, proxy, location_text, loaded_country = await _location_context(
+        resolved, input_model, "PRODUCT"
+    )
+    response = await fetch_page(resolved.url, cookies=cookies, proxy=proxy)
+    if response is None:
+        yield _error(
+            "product_not_found",
+            "The product page could not be loaded after retrying available proxy exits.",
+            input_url=resolved.url,
+        )
+        return
+    if response.status in {404, 410}:
+        yield _error(
+            "product_not_found",
+            "The product page was not found.",
+            input_url=resolved.url,
+        )
+        return
+    if response.status != 200:
+        yield _error(
+            "product_not_found",
+            f"The product page returned HTTP {response.status}.",
+            input_url=resolved.url,
+        )
+        return
+
+    fields = await asyncio.to_thread(
+        parse_product,
+        response.html,
+        asin=asin,
+        url=response.url,
+        domain=resolved.domain,
+    )
+    if not fields.get("title"):
+        yield _error(
+            "product_not_found",
+            "The response did not contain a recognizable product.",
+            input_url=resolved.url,
+            url=response.url,
+        )
+        return
+    fields.update(provenance or {})
+    fields["input"] = resolved.url
+    fields["unNormalizedProductUrl"] = resolved.url
+    fields["locationText"] = location_text
+    fields["loadedCountryCode"] = loaded_country
+
+    if input_model.maxOffers > 0:
+        offer_cookies, offer_proxy, _, _ = await _location_context(
+            resolved, input_model, "OFFERS"
+        )
+        offers_html = await fetch_aod_html(
+            asin,
+            resolved.domain,
+            cookies=offer_cookies or cookies,
+            proxy=offer_proxy or proxy,
+        )
+        if offers_html:
+            fields["offers"] = parse_aod_offers(offers_html, domain=resolved.domain)[
+                : input_model.maxOffers
+            ]
+
+    if input_model.scrapeSellers:
+        seller_ids: list[str] = []
+        featured = fields.get("seller")
+        if isinstance(featured, dict) and featured.get("id"):
+            seller_ids.append(featured["id"])
+        for offer in fields.get("offers") or []:
+            seller = offer.get("seller") if isinstance(offer, dict) else None
+            if isinstance(seller, dict) and seller.get("id"):
+                seller_ids.append(seller["id"])
+        seller_ids = list(dict.fromkeys(seller_ids))
+
+        async def load_seller(seller_id: str) -> dict[str, Any] | None:
+            html = await fetch_seller_html(
+                seller_id, resolved.domain, cookies=cookies, proxy=proxy
+            )
+            return (
+                parse_seller(html, seller_id=seller_id, domain=resolved.domain)
+                if html
+                else None
+            )
+
+        sellers = await gather_bounded(
+            [
+                lambda seller_id=seller_id: load_seller(seller_id)
+                for seller_id in seller_ids
+            ],
+            concurrency=_DETAIL_CONCURRENCY,
+        )
+        by_id = {
+            seller["id"]: seller for seller in sellers if seller and seller.get("id")
+        }
+        if featured and featured.get("id") in by_id:
+            fields["seller"] = by_id[featured["id"]]
+        for offer in fields.get("offers") or []:
+            seller = offer.get("seller")
+            if isinstance(seller, dict) and seller.get("id") in by_id:
+                offer["seller"] = by_id[seller["id"]]
+
+    variant_asins = [
+        value for value in fields.get("variantAsins") or [] if value != asin
+    ]
+    variant_cap = (
+        min(len(variant_asins), input_model.maxProductVariantsAsSeparateResults)
+        if allow_variants
+        else 0
+    )
+    price_cap = (
+        len(variant_asins)
+        if allow_variants and input_model.scrapeProductVariantPrices
+        else 0
+    )
+    fetch_count = max(variant_cap, price_cap)
+    variant_items: list[dict[str, Any]] = []
+    if fetch_count:
+
+        async def load_variant(variant_asin: str) -> list[dict[str, Any]]:
+            variant_url = f"https://{resolved.domain}/dp/{variant_asin}"
+            variant_resolved = resolve_url(variant_url)
+            if variant_resolved is None:
+                return []
+            return [
+                item
+                async for item in _product_flow(
+                    variant_resolved, input_model, allow_variants=False
+                )
+            ]
+
+        batches = await gather_bounded(
+            [
+                lambda variant_asin=variant_asin: load_variant(variant_asin)
+                for variant_asin in variant_asins[:fetch_count]
+            ],
+            concurrency=_DETAIL_CONCURRENCY,
+        )
+        variant_items = [
+            item for batch in batches for item in batch if "error" not in item
+        ]
+        if input_model.scrapeProductVariantPrices:
+            prices = {
+                item.get("asin"): item.get("price")
+                for item in variant_items
+                if item.get("asin")
+            }
+            details = fields.get("variantDetails") or [
+                {"asin": variant_asin} for variant_asin in variant_asins
+            ]
+            for detail in details:
+                if detail.get("asin") in prices:
+                    detail["price"] = prices[detail["asin"]]
+            fields["variantDetails"] = details
+
+    yield ProductItem(**fields).to_output()
+    for item in variant_items[:variant_cap]:
+        item["originalAsin"] = asin
+        yield item
+
+
+async def _search_flow(
+    resolved: ResolvedUrl, input_model: AmazonScrapeInput
+) -> AsyncIterator[dict[str, Any]]:
+    """Page through search results and optionally fetch product details."""
+    if not resolved.domain:
+        yield _error(
+            "no_results_found", "The search domain was invalid.", input_url=resolved.url
+        )
+        return
+    cap = (
+        input_model.maxItemsPerStartUrl
+        if input_model.maxItemsPerStartUrl is not None
+        else _DEFAULT_ITEMS_PER_START_URL
+    )
+    max_pages = min(input_model.maxSearchPagesPerStartUrl, _SEARCH_PAGE_LIMIT)
+    cookies, proxy, _, _ = await _location_context(resolved, input_model, "SEARCH")
+    seen: set[str] = set()
+    emitted = 0
+    for page in range(1, max_pages + 1):
+        html_response = await fetch_page(
+            _page_url(resolved.url, page), cookies=cookies, proxy=proxy
+        )
+        cards = (
+            await asyncio.to_thread(
+                parse_search_page,
+                html_response.html,
+                page=page,
+                domain=resolved.domain,
+            )
+            if html_response is not None and html_response.status == 200
+            else []
+        )
+        cards = [
+            card for card in cards if card.get("asin") and card["asin"] not in seen
+        ]
+        for card in cards:
+            seen.add(card["asin"])
+        if not cards:
+            if page == 1:
+                yield _error(
+                    "no_results_found",
+                    "The search page did not contain any products.",
+                    input_url=resolved.url,
+                )
+            return
+        cards = cards[: max(0, cap - emitted)]
+        if input_model.scrapeProductDetails is False:
+            for card in cards:
+                card["input"] = resolved.url
+                yield ProductItem(**card).to_output()
+                emitted += 1
+        else:
+
+            async def load_card(card: dict[str, Any]) -> list[dict[str, Any]]:
+                product = resolve_url(card["url"])
+                if product is None:
+                    return []
+                provenance = {"categoryPageData": card["categoryPageData"]}
+                return [
+                    item
+                    async for item in _product_flow(
+                        product, input_model, provenance=provenance
+                    )
+                ]
+
+            batches = await gather_bounded(
+                [lambda card=card: load_card(card) for card in cards],
+                concurrency=_DETAIL_CONCURRENCY,
+            )
+            for batch in batches:
+                for item in batch:
+                    if "error" not in item and emitted >= cap:
+                        return
+                    yield item
+                    if "error" not in item:
+                        emitted += 1
+        if emitted >= cap:
+            return
+
+
+async def _bestsellers_flow(
+    resolved: ResolvedUrl, input_model: AmazonScrapeInput
+) -> AsyncIterator[dict[str, Any]]:
+    """Page through a best-sellers category and emit ranked products."""
+    if not resolved.domain:
+        yield _error(
+            "bestsellers_category_not_found",
+            "The best-sellers domain was invalid.",
+            input_url=resolved.url,
+        )
+        return
+    cap = (
+        input_model.maxItemsPerStartUrl
+        if input_model.maxItemsPerStartUrl is not None
+        else _DEFAULT_ITEMS_PER_START_URL
+    )
+    seen: set[str] = set()
+    emitted = 0
+    for page in range(1, min(input_model.maxSearchPagesPerStartUrl, 2) + 1):
+        response = await fetch_page(_page_url(resolved.url, page))
+        cards = (
+            await asyncio.to_thread(
+                parse_bestsellers_page,
+                response.html,
+                page=page,
+                domain=resolved.domain,
+            )
+            if response is not None and response.status == 200
+            else []
+        )
+        cards = [
+            card for card in cards if card.get("asin") and card["asin"] not in seen
+        ]
+        for card in cards:
+            seen.add(card["asin"])
+        if not cards:
+            if page == 1:
+                yield _error(
+                    "bestsellers_category_not_found",
+                    "The page did not contain a recognizable best-sellers category.",
+                    input_url=resolved.url,
+                )
+            return
+        cards = cards[: max(0, cap - emitted)]
+        if input_model.scrapeProductDetails is False:
+            for card in cards:
+                card["input"] = resolved.url
+                yield ProductItem(**card).to_output()
+                emitted += 1
+        else:
+
+            async def load_card(card: dict[str, Any]) -> list[dict[str, Any]]:
+                product = resolve_url(card["url"])
+                if product is None:
+                    return []
+                return [
+                    item
+                    async for item in _product_flow(
+                        product,
+                        input_model,
+                        provenance={"bestsellerPageData": card["bestsellerPageData"]},
+                    )
+                ]
+
+            for batch in await gather_bounded(
+                [lambda card=card: load_card(card) for card in cards],
+                concurrency=_DETAIL_CONCURRENCY,
+            ):
+                for item in batch:
+                    if "error" not in item and emitted >= cap:
+                        return
+                    yield item
+                    if "error" not in item:
+                        emitted += 1
+        if emitted >= cap:
+            return
+
+
+async def _shortened_flow(
+    resolved: ResolvedUrl, input_model: AmazonScrapeInput
+) -> AsyncIterator[dict[str, Any]]:
+    """Resolve a shortened link and dispatch its final Amazon URL."""
+    target = await resolve_shortlink(resolved.url)
+    final = resolve_url(target) if target else None
+    if final is None or final.kind == "shortened":
+        yield _error(
+            "shortened_url_invalid",
+            "The shortened link did not resolve to a supported Amazon page.",
+            input_url=resolved.url,
+            url=target,
+        )
+        return
+    async for item in _FLOWS[final.kind](final, input_model):
+        if "input" in item:
+            item["input"] = resolved.url
+        yield item
+
+
+_FLOWS = {
+    "product": _product_flow,
+    "search": _search_flow,
+    "bestsellers": _bestsellers_flow,
+    "shortened": _shortened_flow,
+}
+
+
+async def iter_products(
+    input_model: AmazonScrapeInput,
+) -> AsyncIterator[dict[str, Any]]:
+    """Yield product items for every start URL.
+
+    Each ``categoryOrProductUrls`` entry is classified and dispatched to its
+    per-kind flow. A URL that cannot be recognized as an Amazon product /
+    search / bestsellers / shortened link yields an ``invalid_url`` error item
+    (the failure model is in-stream error items, not exceptions).
+    """
+    for entry in input_model.categoryOrProductUrls:
+        url = entry.get("url") if isinstance(entry, dict) else None
+        resolved = resolve_url(url) if url else None
+        if resolved is None:
+            yield _error(
+                "invalid_url",
+                "Start URL was malformed or not a recognized Amazon URL.",
+                input_url=url,
+            )
+            continue
+        async for item in _FLOWS[resolved.kind](resolved, input_model):
+            yield item
+
+
+async def scrape_products(
+    input_model: AmazonScrapeInput, *, limit: int | None = None
+) -> list[dict[str, Any]]:
+    """Collect :func:`iter_products` into a list, honoring an optional ``limit``.
+
+    ``limit`` is a request-time policy guard (used by the route/capability), NOT
+    a ceiling in the streaming core.
+    """
+    from app.capabilities.core.progress import emit_progress
+
+    results: list[dict[str, Any]] = []
+    async for item in iter_products(input_model):
+        results.append(item)
+        emit_progress("scraping", current=len(results), total=limit, unit="product")
+        if limit is not None and len(results) >= limit:
+            break
+    return results