# ruff: noqa: N815 """Input/output models for the Amazon Product Scraper. The skeleton accepts the full input surface; output fields the implementation does not source yet are emitted as ``None``/``[]`` so parity is additive — exactly like the YouTube / Maps / Google Search models. Outputs use ``extra="allow"`` on purpose: it lets the output shape grow without breaking existing consumers. Only a small set of stable, always-populated nested objects (``price``, ``starsBreakdown``, ``seller``, ``visitStoreLink``) are typed; the sprawling, layout-volatile sections (A+ content, brand story, attribute tables, offers, on-page reviews, ...) stay loose ``dict``/``list`` and lean on ``extra="allow"`` rather than pinning a shape the parsers haven't verified against live HTML yet. Public, anonymous data only: there is no auth/login field on the input surface. Deep review pagination is login-gated on today's Amazon and is out of scope; the on-page ``productPageReviews`` are the only reviews modeled here. """ from __future__ import annotations from typing import Any, Literal from pydantic import BaseModel, ConfigDict, Field # --------------------------------------------------------------------------- # # Typed nested objects (the stable, always-populated ones). # # --------------------------------------------------------------------------- # class Price(BaseModel): """A monetary amount as Amazon renders it (``price``, ``listPrice``, ``shippingPrice``, and the ``min``/``max`` of ``priceRange``).""" model_config = ConfigDict(extra="allow") value: float | None = None currency: str | None = None class StarsBreakdown(BaseModel): """Per-star rating distribution (fractions summing to ~1.0). Amazon's keys start with a digit (``5star`` ... ``1star``), which is not a valid Python identifier, so each field is aliased. ``populate_by_name`` lets tests construct it either way; ``to_output`` serializes ``by_alias`` so the wire shape retains the digit-prefixed keys. """ model_config = ConfigDict(extra="allow", populate_by_name=True) five_star: float | None = Field(default=None, alias="5star") four_star: float | None = Field(default=None, alias="4star") three_star: float | None = Field(default=None, alias="3star") two_star: float | None = Field(default=None, alias="2star") one_star: float | None = Field(default=None, alias="1star") class Seller(BaseModel): """The featured-offer seller stamped on the product item.""" model_config = ConfigDict(extra="allow") id: str | None = None name: str | None = None url: str | None = None reviewsCount: int | None = None averageRating: float | None = None class VisitStoreLink(BaseModel): """The brand-store link (``visitStoreLink``).""" model_config = ConfigDict(extra="allow") text: str | None = None url: str | None = None # --------------------------------------------------------------------------- # # Input surface. # # --------------------------------------------------------------------------- # class AmazonScrapeInput(BaseModel): """Full input surface for the Amazon Product Scraper. ``categoryOrProductUrls`` mixes category/search, product, bestsellers, and shortened URLs; ``proxyCountry`` defaults to ``AUTO_SELECT_PROXY_COUNTRY`` (derived from each URL's domain); ``scrapeProductDetails`` defaults on (deep scrape). ``extra="allow"`` keeps a verbatim payload valid even for add-ons this scraper does not model. """ model_config = ConfigDict(extra="allow") # Discovery (required) categoryOrProductUrls: list[dict] = Field(min_length=1) # Result caps maxItemsPerStartUrl: int | None = Field(default=None, ge=0) maxSearchPagesPerStartUrl: int = Field(default=9999, ge=1) maxProductVariantsAsSeparateResults: int = Field(default=0, ge=0) maxOffers: int = Field(default=0, ge=0) # Localization language: str | None = None proxyCountry: str = "AUTO_SELECT_PROXY_COUNTRY" countryCode: str | None = None zipCode: str | None = None locationDeliverableRoutes: list[str] | None = Field( default_factory=lambda: ["PRODUCT", "SEARCH", "OFFERS"] ) # Feature toggles scrapeSellers: bool = False useCaptchaSolver: bool = False scrapeProductVariantPrices: bool = False scrapeProductDetails: bool | None = True # --------------------------------------------------------------------------- # # Output item. # # --------------------------------------------------------------------------- # class ProductItem(BaseModel): """Amazon Product Scraper output item (one per product). Unsourced fields default to ``None``/``[]``; ``extra="allow"`` keeps the contract open so later milestones add fields without breaking consumers. """ model_config = ConfigDict(extra="allow", populate_by_name=True) # Identity / core title: str | None = None url: str | None = None asin: str | None = None originalAsin: str | None = None brand: str | None = None author: str | None = None price: Price | None = None listPrice: Price | None = None shippingPrice: Price | None = None inStock: bool | None = None inStockText: str | None = None delivery: str | None = None fastestDelivery: str | None = None condition: str | None = None # Ratings / social stars: float | None = None starsBreakdown: StarsBreakdown | None = None reviewsCount: int | None = None answeredQuestions: int | None = None aiReviewsSummary: dict | None = None monthlyPurchaseVolume: str | None = None # Content breadCrumbs: str | None = None description: str | None = None features: list[str] = Field(default_factory=list) sustainabilityFeatures: list[dict] = Field(default_factory=list) videosCount: int | None = None visitStoreLink: VisitStoreLink | None = None thumbnailImage: str | None = None galleryThumbnails: list[str] = Field(default_factory=list) highResolutionImages: list[str] = Field(default_factory=list) importantInformation: dict | None = None bookDescription: str | None = None aPlusContent: dict | None = None brandStory: dict | None = None productComparison: dict | None = None # Commerce returnPolicy: str | None = None support: str | None = None priceRange: dict | None = None variantAsins: list[str] = Field(default_factory=list) variantDetails: list[dict] = Field(default_factory=list) variantAttributes: list[dict] = Field(default_factory=list) attributes: list[dict] = Field(default_factory=list) attributesMapped: dict | None = None productOverview: list[dict] = Field(default_factory=list) manufacturerAttributes: list[dict] = Field(default_factory=list) seller: Seller | None = None bestsellerRanks: list[dict] = Field(default_factory=list) isAmazonChoice: bool | None = None amazonChoiceText: str | None = None offers: list[dict] = Field(default_factory=list) # Reviews (public, on-page only) reviewsLink: str | None = None hasReviews: bool | None = None productPageReviews: list[dict] = Field(default_factory=list) productPageReviewsFromOtherCountries: list[dict] = Field(default_factory=list) # Provenance / meta locationText: str | None = None unNormalizedProductUrl: str | None = None loadedCountryCode: str | None = None categoryPageData: dict | None = None bestsellerPageData: dict | None = None input: str | None = None def to_output(self) -> dict[str, Any]: """Serialize to the flat dict output shape (keeps extras, aliases). ``by_alias`` restores digit-prefixed keys like ``starsBreakdown.5star``; ``exclude_none=False`` keeps unsourced keys present so consumers never break on a missing field. """ return self.model_dump(by_alias=True, exclude_none=False) # --------------------------------------------------------------------------- # # Error item (the failure model — pushed into the stream, not raised). # # --------------------------------------------------------------------------- # ErrorCode = Literal[ "invalid_url", "invalid_input", "product_not_found", "shortened_url_invalid", "bestsellers_category_not_found", "no_results_found", ] class ErrorItem(BaseModel): """A per-input failure emitted into the dataset instead of a normal item. Consumers tell error items apart from products by the presence of ``error``. ``invalid_input`` additionally terminates the run (enforced in later milestones); all other codes are per-input and non-fatal. """ model_config = ConfigDict(extra="allow") error: ErrorCode errorDescription: str | None = None input: str | None = None url: str | None = None