mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-22 23:31:12 +02:00
Merge pull request #1604 from AnishSarkar22/feat/amazon-scraper
feat(amazon): add public multi-marketplace product scraping capability
This commit is contained in:
Rohan Verma
GitHub
commit
d1d839a491
103 changed files with 4130 additions and 121 deletions
|
|
@ -27,6 +27,7 @@ pytestmark = pytest.mark.unit
|
|||
# specialist is a deliberate product change and must be reflected here.
|
||||
_EXPECTED_SUBAGENTS = frozenset(
|
||||
{
|
||||
"amazon",
|
||||
"deliverables",
|
||||
"dropbox",
|
||||
"google_drive",
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
|
||||
|
|
@ -0,0 +1 @@
|
|||
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from app.capabilities.amazon.scrape.executor import build_scrape_executor
|
||||
from app.capabilities.amazon.scrape.schemas import MAX_AMAZON_RESULTS, ScrapeInput
|
||||
from app.proprietary.platforms.amazon import AmazonScrapeInput
|
||||
|
||||
|
||||
class _FakeScraper:
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[tuple[AmazonScrapeInput, int | None]] = []
|
||||
|
||||
async def __call__(
|
||||
self, input_model: AmazonScrapeInput, *, limit: int | None = None
|
||||
) -> list[dict]:
|
||||
self.calls.append((input_model, limit))
|
||||
return [{"asin": "B09V3KXJPB", "title": "Product"}]
|
||||
|
||||
|
||||
async def test_executor_maps_agent_input_to_scraper_input():
|
||||
scraper = _FakeScraper()
|
||||
execute = build_scrape_executor(scraper)
|
||||
|
||||
output = await execute(
|
||||
ScrapeInput(
|
||||
search_terms=["wireless mouse"],
|
||||
urls=["https://www.amazon.com/dp/B09V3KXJPB"],
|
||||
max_items=5,
|
||||
max_offers=2,
|
||||
include_sellers=True,
|
||||
zip_code="10001",
|
||||
country_code="US",
|
||||
)
|
||||
)
|
||||
|
||||
assert output.items[0].asin == "B09V3KXJPB"
|
||||
input_model, limit = scraper.calls[0]
|
||||
assert input_model.categoryOrProductUrls == [
|
||||
{"url": "https://www.amazon.com/dp/B09V3KXJPB"},
|
||||
{"url": "https://www.amazon.com/s?k=wireless+mouse"},
|
||||
]
|
||||
assert input_model.maxItemsPerStartUrl == 5
|
||||
assert input_model.maxOffers == 2
|
||||
assert input_model.scrapeSellers is True
|
||||
assert input_model.zipCode == "10001"
|
||||
assert limit == MAX_AMAZON_RESULTS
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app.capabilities.amazon.scrape.schemas import (
|
||||
MAX_AMAZON_RESULTS,
|
||||
ScrapeInput,
|
||||
ScrapeOutput,
|
||||
)
|
||||
|
||||
|
||||
def test_estimated_units_cover_search_and_direct_variant_fanout():
|
||||
payload = ScrapeInput(
|
||||
search_terms=["mouse", "keyboard"],
|
||||
urls=["https://www.amazon.com/dp/B09V3KXJPB"],
|
||||
max_items=20,
|
||||
max_variants=3,
|
||||
)
|
||||
|
||||
assert payload.estimated_units == 44
|
||||
|
||||
|
||||
def test_estimated_units_respect_hard_run_ceiling():
|
||||
payload = ScrapeInput(search_terms=["x"] * 20, max_items=100)
|
||||
assert payload.estimated_units == MAX_AMAZON_RESULTS
|
||||
|
||||
|
||||
def test_at_least_one_source_is_required():
|
||||
with pytest.raises(ValidationError):
|
||||
ScrapeInput()
|
||||
|
||||
|
||||
def test_error_items_are_not_billable():
|
||||
output = ScrapeOutput(
|
||||
items=[
|
||||
{"asin": "B09V3KXJPB", "title": "Product"},
|
||||
{"error": "product_not_found", "errorDescription": "Missing"},
|
||||
]
|
||||
)
|
||||
|
||||
assert output.billable_units == 1
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from app.capabilities.amazon.scrape.schemas import ScrapeInput, ScrapeOutput
|
||||
from app.capabilities.core import BillingUnit
|
||||
from app.capabilities.core.store import get_capability
|
||||
|
||||
|
||||
def test_amazon_scrape_is_registered_and_metered():
|
||||
capability = get_capability("amazon.scrape")
|
||||
|
||||
assert capability.input_schema is ScrapeInput
|
||||
assert capability.output_schema is ScrapeOutput
|
||||
assert capability.billing_unit is BillingUnit.AMAZON_PRODUCT
|
||||
|
|
@ -7,6 +7,7 @@ import pytest
|
|||
from app.capabilities import (
|
||||
google_maps, # noqa: F401 — importing the namespace registers its verbs
|
||||
)
|
||||
from app.capabilities.core import BillingUnit
|
||||
from app.capabilities.core.store import get_capability
|
||||
from app.capabilities.google_maps.reviews.schemas import ReviewsInput, ReviewsOutput
|
||||
from app.capabilities.google_maps.scrape.schemas import ScrapeInput, ScrapeOutput
|
||||
|
|
@ -14,19 +15,19 @@ from app.capabilities.google_maps.scrape.schemas import ScrapeInput, ScrapeOutpu
|
|||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def test_google_maps_scrape_is_registered_and_free():
|
||||
def test_google_maps_scrape_is_registered_and_billable():
|
||||
cap = get_capability("google_maps.scrape")
|
||||
|
||||
assert cap.name == "google_maps.scrape"
|
||||
assert cap.input_schema is ScrapeInput
|
||||
assert cap.output_schema is ScrapeOutput
|
||||
assert cap.billing_unit is None
|
||||
assert cap.billing_unit is BillingUnit.GOOGLE_MAPS_PLACE
|
||||
|
||||
|
||||
def test_google_maps_reviews_is_registered_and_free():
|
||||
def test_google_maps_reviews_is_registered_and_billable():
|
||||
cap = get_capability("google_maps.reviews")
|
||||
|
||||
assert cap.name == "google_maps.reviews"
|
||||
assert cap.input_schema is ReviewsInput
|
||||
assert cap.output_schema is ReviewsOutput
|
||||
assert cap.billing_unit is None
|
||||
assert cap.billing_unit is BillingUnit.GOOGLE_MAPS_REVIEW
|
||||
|
|
|
|||
|
|
@ -7,16 +7,17 @@ import pytest
|
|||
from app.capabilities import (
|
||||
reddit, # noqa: F401 — importing the namespace registers its verbs
|
||||
)
|
||||
from app.capabilities.core import BillingUnit
|
||||
from app.capabilities.core.store import get_capability
|
||||
from app.capabilities.reddit.scrape.schemas import ScrapeInput, ScrapeOutput
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def test_reddit_scrape_is_registered_and_free():
|
||||
def test_reddit_scrape_is_registered_and_billable():
|
||||
cap = get_capability("reddit.scrape")
|
||||
|
||||
assert cap.name == "reddit.scrape"
|
||||
assert cap.input_schema is ScrapeInput
|
||||
assert cap.output_schema is ScrapeOutput
|
||||
assert cap.billing_unit is None
|
||||
assert cap.billing_unit is BillingUnit.REDDIT_ITEM
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import pytest
|
|||
from app.capabilities import (
|
||||
youtube, # noqa: F401 — importing the namespace registers its verbs
|
||||
)
|
||||
from app.capabilities.core import BillingUnit
|
||||
from app.capabilities.core.store import get_capability
|
||||
from app.capabilities.youtube.comments.schemas import CommentsInput, CommentsOutput
|
||||
from app.capabilities.youtube.scrape.schemas import ScrapeInput, ScrapeOutput
|
||||
|
|
@ -14,19 +15,19 @@ from app.capabilities.youtube.scrape.schemas import ScrapeInput, ScrapeOutput
|
|||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def test_youtube_scrape_is_registered_and_free():
|
||||
def test_youtube_scrape_is_registered_and_billable():
|
||||
cap = get_capability("youtube.scrape")
|
||||
|
||||
assert cap.name == "youtube.scrape"
|
||||
assert cap.input_schema is ScrapeInput
|
||||
assert cap.output_schema is ScrapeOutput
|
||||
assert cap.billing_unit is None
|
||||
assert cap.billing_unit is BillingUnit.YOUTUBE_VIDEO
|
||||
|
||||
|
||||
def test_youtube_comments_is_registered_and_free():
|
||||
def test_youtube_comments_is_registered_and_billable():
|
||||
cap = get_capability("youtube.comments")
|
||||
|
||||
assert cap.name == "youtube.comments"
|
||||
assert cap.input_schema is CommentsInput
|
||||
assert cap.output_schema is CommentsOutput
|
||||
assert cap.billing_unit is None
|
||||
assert cap.billing_unit is BillingUnit.YOUTUBE_COMMENT
|
||||
|
|
|
|||
|
|
@ -0,0 +1,17 @@
|
|||
<!doctype html>
|
||||
<html lang="en">
|
||||
<body>
|
||||
<div id="gridItemRoot">
|
||||
<span class="zg-bdg-text">#1</span>
|
||||
<a href="/dp/B09V3KXJPB"><span class="_cDEzb_p13n-sc-css-line-clamp-3_g3dy1">Example Wireless Headphones</span></a>
|
||||
<span class="a-price"><span class="a-offscreen">$49.99</span></span>
|
||||
<span class="a-icon-alt">4.6 out of 5 stars</span>
|
||||
<img src="https://images.example/headphones.jpg" alt="Example Wireless Headphones">
|
||||
</div>
|
||||
<div id="gridItemRoot">
|
||||
<span class="zg-bdg-text">#2</span>
|
||||
<a href="/dp/B09V3KXJP1"><span class="_cDEzb_p13n-sc-css-line-clamp-3_g3dy1">Example Earbuds</span></a>
|
||||
<span class="a-price"><span class="a-offscreen">$29.99</span></span>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head><title>Robot Check</title></head>
|
||||
<body>
|
||||
<form action="/errors/validateCaptcha">
|
||||
<p>Enter the characters you see below</p>
|
||||
</form>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
<!doctype html>
|
||||
<html lang="en">
|
||||
<body>
|
||||
<div id="aod-offer">
|
||||
<div id="aod-pinned-offer"></div>
|
||||
<div id="aod-offer-heading">New</div>
|
||||
<span class="a-price"><span class="a-offscreen">$47.50</span></span>
|
||||
<div id="aod-offer-soldBy"><a href="/sp?seller=SELLER123">Example Retailer</a></div>
|
||||
<div id="mir-layout-DELIVERY_BLOCK">FREE delivery tomorrow</div>
|
||||
</div>
|
||||
<div id="aod-offer">
|
||||
<div id="aod-offer-heading">Used - Very Good</div>
|
||||
<span class="a-price"><span class="a-offscreen">$39.00</span></span>
|
||||
<div id="aod-offer-soldBy"><a href="/sp?seller=SELLER456">Second Seller</a></div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
<!doctype html>
|
||||
<html lang="en">
|
||||
<body>
|
||||
<div id="wayfinding-breadcrumbs_feature_div"><ul><li><a href="/b?node=172282">Electronics</a></li><li><a href="/b?node=667846011">Audio</a></li></ul></div>
|
||||
<span id="productTitle">Example Wireless Headphones</span>
|
||||
<a id="bylineInfo" href="/stores/Example">Visit the Example Store</a>
|
||||
<a id="sellerProfileTriggerId" href="/sp?seller=SELLER123">Example Retailer</a>
|
||||
<div id="corePrice_feature_div">
|
||||
<span class="priceToPay"><span class="a-offscreen">$49.99</span></span>
|
||||
<span class="basisPrice"><span class="a-offscreen">$79.99</span></span>
|
||||
</div>
|
||||
<div id="averageCustomerReviews">
|
||||
<span id="acrPopover">4.6 out of 5 stars</span>
|
||||
<span id="acrCustomerReviewText">1,234 ratings</span>
|
||||
<a id="acrCustomerReviewLink" href="/product-reviews/B09V3KXJPB">Reviews</a>
|
||||
</div>
|
||||
<table id="histogramTable">
|
||||
<tr><td class="a-text-left">5 star</td><td class="a-text-right">80%</td></tr>
|
||||
<tr><td class="a-text-left">4 star</td><td class="a-text-right">12%</td></tr>
|
||||
</table>
|
||||
<div id="availability">In Stock</div>
|
||||
<div id="social-proofing-faceout-title-tk_bought">2K+ bought in past month</div>
|
||||
<div id="feature-bullets"><ul><li><span class="a-list-item">Long battery life</span></li><li><span class="a-list-item">Noise cancelling</span></li></ul></div>
|
||||
<div id="productDescription">Detailed product description.</div>
|
||||
<img id="landingImage" src="https://images.example/thumb.jpg" data-old-hires="https://images.example/full.jpg" alt="Example Wireless Headphones">
|
||||
<div id="aplus"><p>Premium materials and comfort.</p><img src="https://images.example/aplus.jpg" alt="Premium materials and comfort"></div>
|
||||
<div id="brandStory_feature_div">Designed for everyday listening.</div>
|
||||
<div id="detailBulletsWrapper_feature_div"><ul><li>Best Sellers Rank: #12 in Electronics #3 in Headphones</li></ul></div>
|
||||
<table id="productDetails_techSpec_section_1"><tr><th>Model</th><td>WH-100</td></tr></table>
|
||||
<div id="productOverview_feature_div"><table><tr><td>Color</td><td>Black</td></tr></table></div>
|
||||
<div id="acBadge_feature_div">Amazon's Choice</div>
|
||||
<script>
|
||||
window.twister = {
|
||||
"dimensionValuesDisplayData": {
|
||||
"B09V3KXJPB": ["Black"],
|
||||
"B09V3KXJP1": ["Blue"]
|
||||
},
|
||||
"variationValues": {"color_name": ["Black", "Blue"]}
|
||||
};
|
||||
</script>
|
||||
<div data-hook="review" id="R1">
|
||||
<span class="a-profile-name">Alex</span>
|
||||
<a data-hook="review-title" href="/gp/customer-reviews/R1">Excellent sound</a>
|
||||
<i data-hook="review-star-rating">5.0 out of 5 stars</i>
|
||||
<span data-hook="review-date">Reviewed on July 1, 2026</span>
|
||||
<span data-hook="review-body">Clear and comfortable.</span>
|
||||
<span data-hook="avp-badge">Verified Purchase</span>
|
||||
<span data-hook="helpful-vote-statement">12 people found this helpful</span>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
<!doctype html>
|
||||
<html lang="en">
|
||||
<body>
|
||||
<div data-component-type="s-search-result" data-asin="B09V3KXJPB" class="s-result-item">
|
||||
<h2><a href="/dp/B09V3KXJPB"><span>Example Wireless Headphones</span></a></h2>
|
||||
<span class="a-price"><span class="a-offscreen">$49.99</span></span>
|
||||
<span class="a-icon-alt">4.6 out of 5 stars</span>
|
||||
<span class="s-underline-text">1,234</span>
|
||||
<img class="s-image" src="https://images.example/headphones.jpg" alt="Example Wireless Headphones">
|
||||
<span class="a-badge-text">Best Seller</span>
|
||||
</div>
|
||||
<div data-component-type="s-search-result" data-asin="B09V3KXJP1" class="s-result-item">
|
||||
<span>Sponsored</span>
|
||||
<h2><a href="/dp/B09V3KXJP1"><span>Example Earbuds</span></a></h2>
|
||||
<span class="a-price"><span class="a-offscreen">$29.99</span></span>
|
||||
<span class="a-icon-alt">4.2 out of 5 stars</span>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
<!doctype html>
|
||||
<html lang="en">
|
||||
<body>
|
||||
<h1 id="sellerName">Example Retailer</h1>
|
||||
<div id="seller-feedback-summary">1,875 ratings</div>
|
||||
<table id="feedback-summary-table">
|
||||
<tr><td><span class="a-icon-alt">4.8 out of 5 stars</span></td></tr>
|
||||
<tr><td>1,875 ratings</td></tr>
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
||||
220
surfsense_backend/tests/unit/platforms/amazon/test_flows.py
Normal file
220
surfsense_backend/tests/unit/platforms/amazon/test_flows.py
Normal file
|
|
@ -0,0 +1,220 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from app.proprietary.platforms.amazon import AmazonScrapeInput, scrape_products, scraper
|
||||
from app.proprietary.platforms.amazon.fetch import FetchResult
|
||||
from app.proprietary.platforms.amazon.url_resolver import resolve_url
|
||||
|
||||
_FIXTURES = Path(__file__).parent / "fixtures"
|
||||
|
||||
|
||||
def _fixture(name: str) -> str:
|
||||
return (_FIXTURES / name).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def _response(url: str, html: str, status: int = 200) -> FetchResult:
|
||||
return FetchResult(status=status, html=html, url=url, cookies={})
|
||||
|
||||
|
||||
async def test_product_flow_emits_parsed_item(monkeypatch):
|
||||
async def fetch_page(url: str, **_kwargs):
|
||||
return _response(url, _fixture("product.html"))
|
||||
|
||||
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
|
||||
items = await scrape_products(
|
||||
AmazonScrapeInput(
|
||||
categoryOrProductUrls=[{"url": "https://www.amazon.com/dp/B09V3KXJPB"}]
|
||||
)
|
||||
)
|
||||
|
||||
assert len(items) == 1
|
||||
assert items[0]["asin"] == "B09V3KXJPB"
|
||||
assert items[0]["title"] == "Example Wireless Headphones"
|
||||
|
||||
|
||||
async def test_product_flow_maps_not_found_to_error_item(monkeypatch):
|
||||
async def fetch_page(url: str, **_kwargs):
|
||||
return _response(url, "", status=404)
|
||||
|
||||
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
|
||||
items = await scrape_products(
|
||||
AmazonScrapeInput(
|
||||
categoryOrProductUrls=[{"url": "https://www.amazon.com/dp/B09V3KXJPB"}]
|
||||
)
|
||||
)
|
||||
|
||||
assert items[0]["error"] == "product_not_found"
|
||||
|
||||
|
||||
async def test_search_flow_honors_cap_and_stops_on_empty_page(monkeypatch):
|
||||
calls: list[str] = []
|
||||
|
||||
async def fetch_page(url: str, **_kwargs):
|
||||
calls.append(url)
|
||||
html = _fixture("search.html") if "page=1" in url else "<html></html>"
|
||||
return _response(url, html)
|
||||
|
||||
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
|
||||
items = await scrape_products(
|
||||
AmazonScrapeInput(
|
||||
categoryOrProductUrls=[{"url": "https://www.amazon.com/s?k=headphones"}],
|
||||
maxItemsPerStartUrl=1,
|
||||
maxSearchPagesPerStartUrl=5,
|
||||
scrapeProductDetails=False,
|
||||
)
|
||||
)
|
||||
|
||||
assert len(items) == 1
|
||||
assert items[0]["categoryPageData"]["position"] == 1
|
||||
assert len(calls) == 1
|
||||
|
||||
|
||||
async def test_search_flow_threads_marketplace_locale_to_fetch(monkeypatch):
|
||||
calls: list[dict[str, object]] = []
|
||||
|
||||
async def fetch_page(url: str, **kwargs):
|
||||
calls.append({"url": url, **kwargs})
|
||||
return _response(url, _fixture("search.html"))
|
||||
|
||||
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
|
||||
items = await scrape_products(
|
||||
AmazonScrapeInput(
|
||||
categoryOrProductUrls=[{"url": "https://www.amazon.co.uk/s?k=headphones"}],
|
||||
maxItemsPerStartUrl=1,
|
||||
scrapeProductDetails=False,
|
||||
)
|
||||
)
|
||||
|
||||
assert len(items) == 1
|
||||
assert calls[0]["country"] == "gb"
|
||||
assert calls[0]["accept_language"] == "en-GB"
|
||||
|
||||
|
||||
async def test_search_flow_returns_no_results_error(monkeypatch):
|
||||
async def fetch_page(url: str, **_kwargs):
|
||||
return _response(url, "<html></html>")
|
||||
|
||||
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
|
||||
items = await scrape_products(
|
||||
AmazonScrapeInput(
|
||||
categoryOrProductUrls=[{"url": "https://www.amazon.com/s?k=missing"}],
|
||||
scrapeProductDetails=False,
|
||||
)
|
||||
)
|
||||
|
||||
assert items[0]["error"] == "no_results_found"
|
||||
|
||||
|
||||
async def test_product_flow_enriches_offers_and_seller(monkeypatch):
|
||||
async def fetch_page(url: str, **_kwargs):
|
||||
return _response(url, _fixture("product.html"))
|
||||
|
||||
async def fetch_aod_html(*_args, **_kwargs):
|
||||
return _fixture("offers.html")
|
||||
|
||||
async def fetch_seller_html(*_args, **_kwargs):
|
||||
return _fixture("seller.html")
|
||||
|
||||
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
|
||||
monkeypatch.setattr(scraper, "fetch_aod_html", fetch_aod_html)
|
||||
monkeypatch.setattr(scraper, "fetch_seller_html", fetch_seller_html)
|
||||
items = await scrape_products(
|
||||
AmazonScrapeInput(
|
||||
categoryOrProductUrls=[{"url": "https://www.amazon.com/dp/B09V3KXJPB"}],
|
||||
maxOffers=1,
|
||||
scrapeSellers=True,
|
||||
)
|
||||
)
|
||||
|
||||
assert len(items[0]["offers"]) == 1
|
||||
assert items[0]["offers"][0]["seller"]["averageRating"] == 4.8
|
||||
assert items[0]["seller"]["reviewsCount"] == 1875
|
||||
|
||||
|
||||
async def test_bestsellers_card_only_flow(monkeypatch):
|
||||
async def fetch_page(url: str, **_kwargs):
|
||||
return _response(url, _fixture("bestsellers.html"))
|
||||
|
||||
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
|
||||
items = await scrape_products(
|
||||
AmazonScrapeInput(
|
||||
categoryOrProductUrls=[
|
||||
{
|
||||
"url": (
|
||||
"https://www.amazon.com/Best-Sellers-Electronics/"
|
||||
"zgbs/electronics"
|
||||
)
|
||||
}
|
||||
],
|
||||
maxItemsPerStartUrl=2,
|
||||
scrapeProductDetails=False,
|
||||
)
|
||||
)
|
||||
|
||||
assert [item["bestsellerPageData"]["rank"] for item in items] == [1, 2]
|
||||
|
||||
|
||||
async def test_shortlink_resolves_and_dispatches(monkeypatch):
|
||||
async def resolve(_url: str):
|
||||
return "https://www.amazon.com/dp/B09V3KXJPB"
|
||||
|
||||
async def fetch_page(url: str, **_kwargs):
|
||||
return _response(url, _fixture("product.html"))
|
||||
|
||||
monkeypatch.setattr(scraper, "resolve_shortlink", resolve)
|
||||
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
|
||||
items = await scrape_products(
|
||||
AmazonScrapeInput(categoryOrProductUrls=[{"url": "https://a.co/d/example"}])
|
||||
)
|
||||
|
||||
assert items[0]["asin"] == "B09V3KXJPB"
|
||||
assert items[0]["input"] == "https://a.co/d/example"
|
||||
|
||||
|
||||
async def test_product_flow_expands_variants_and_attaches_prices(monkeypatch):
|
||||
async def fetch_page(url: str, **_kwargs):
|
||||
return _response(url, _fixture("product.html"))
|
||||
|
||||
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
|
||||
items = await scrape_products(
|
||||
AmazonScrapeInput(
|
||||
categoryOrProductUrls=[{"url": "https://www.amazon.com/dp/B09V3KXJPB"}],
|
||||
maxProductVariantsAsSeparateResults=1,
|
||||
scrapeProductVariantPrices=True,
|
||||
)
|
||||
)
|
||||
|
||||
assert [item["asin"] for item in items] == ["B09V3KXJPB", "B09V3KXJP1"]
|
||||
assert items[1]["originalAsin"] == "B09V3KXJPB"
|
||||
variant = next(
|
||||
detail
|
||||
for detail in items[0]["variantDetails"]
|
||||
if detail["asin"] == "B09V3KXJP1"
|
||||
)
|
||||
assert variant["price"]["value"] == 49.99
|
||||
|
||||
|
||||
async def test_route_outside_location_allowlist_skips_session_mint(monkeypatch):
|
||||
calls = 0
|
||||
|
||||
async def get_location_session(*_args, **_kwargs):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(scraper, "get_location_session", get_location_session)
|
||||
resolved = resolve_url("https://www.amazon.com/dp/B09V3KXJPB")
|
||||
assert resolved is not None
|
||||
context = await scraper._location_context(
|
||||
resolved,
|
||||
AmazonScrapeInput(
|
||||
categoryOrProductUrls=[{"url": resolved.url}],
|
||||
zipCode="10001",
|
||||
locationDeliverableRoutes=["OFFERS"],
|
||||
),
|
||||
"PRODUCT",
|
||||
)
|
||||
|
||||
assert context == (None, None, None, None)
|
||||
assert calls == 0
|
||||
33
surfsense_backend/tests/unit/platforms/amazon/test_locale.py
Normal file
33
surfsense_backend/tests/unit/platforms/amazon/test_locale.py
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from app.proprietary.platforms.amazon.locale import (
|
||||
accept_language_for,
|
||||
proxy_country_for,
|
||||
)
|
||||
|
||||
|
||||
def test_proxy_country_for_confirmed_marketplaces():
|
||||
assert proxy_country_for("com") == "us"
|
||||
assert proxy_country_for("co.uk") == "gb"
|
||||
assert proxy_country_for("de") == "de"
|
||||
assert proxy_country_for("fr") == "fr"
|
||||
assert proxy_country_for("it") == "it"
|
||||
assert proxy_country_for("es") == "es"
|
||||
|
||||
|
||||
def test_proxy_country_for_unknown_marketplace_falls_back():
|
||||
assert proxy_country_for("co.jp") is None
|
||||
assert proxy_country_for(None) is None
|
||||
|
||||
|
||||
def test_accept_language_for_marketplaces():
|
||||
assert accept_language_for("co.uk") == "en-GB"
|
||||
assert accept_language_for("de") == "de-DE"
|
||||
assert accept_language_for("fr") == "fr-FR"
|
||||
assert accept_language_for("it") == "it-IT"
|
||||
assert accept_language_for("es") == "es-ES"
|
||||
|
||||
|
||||
def test_accept_language_for_unknown_marketplace_defaults_to_us_english():
|
||||
assert accept_language_for("co.jp") == "en-US"
|
||||
assert accept_language_for(None) == "en-US"
|
||||
176
surfsense_backend/tests/unit/platforms/amazon/test_parsers.py
Normal file
176
surfsense_backend/tests/unit/platforms/amazon/test_parsers.py
Normal file
|
|
@ -0,0 +1,176 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from app.proprietary.platforms.amazon import fetch
|
||||
from app.proprietary.platforms.amazon.fetch import (
|
||||
FetchResult,
|
||||
get_location_session,
|
||||
is_blocked,
|
||||
should_localize,
|
||||
)
|
||||
from app.proprietary.platforms.amazon.parsers import (
|
||||
_float,
|
||||
parse_aod_offers,
|
||||
parse_bestsellers_page,
|
||||
parse_product,
|
||||
parse_search_page,
|
||||
parse_seller,
|
||||
)
|
||||
|
||||
_FIXTURES = Path(__file__).parent / "fixtures"
|
||||
|
||||
|
||||
def _fixture(name: str) -> str:
|
||||
return (_FIXTURES / name).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_product_parser_extracts_core_public_fields():
|
||||
item = parse_product(
|
||||
_fixture("product.html"),
|
||||
asin="B09V3KXJPB",
|
||||
url="https://www.amazon.com/dp/B09V3KXJPB",
|
||||
)
|
||||
|
||||
assert item["title"] == "Example Wireless Headphones"
|
||||
assert item["price"] == {"value": 49.99, "currency": "USD"}
|
||||
assert item["listPrice"] == {"value": 79.99, "currency": "USD"}
|
||||
assert item["stars"] == 4.6
|
||||
assert item["reviewsCount"] == 1234
|
||||
assert item["inStock"] is True
|
||||
assert item["features"] == ["Long battery life", "Noise cancelling"]
|
||||
assert item["starsBreakdown"]["5star"] == 0.8
|
||||
assert "B09V3KXJP1" in item["variantAsins"]
|
||||
assert item["variantAttributes"][0]["name"] == "color_name"
|
||||
assert item["productPageReviews"][0]["id"] == "R1"
|
||||
|
||||
|
||||
def test_block_detection_handles_status_and_markup():
|
||||
blocked = _fixture("blocked.html")
|
||||
assert is_blocked(blocked, 200)
|
||||
assert is_blocked("", 503)
|
||||
assert not is_blocked(_fixture("product.html"), 200)
|
||||
|
||||
|
||||
def test_block_detection_handles_waf_header_and_body_markers():
|
||||
assert is_blocked(
|
||||
"<html>ordinary status</html>",
|
||||
200,
|
||||
{"X-Amzn-Waf-Action": "challenge"},
|
||||
)
|
||||
assert is_blocked(
|
||||
"<script src='https://token.awswaf.com/challenge.js'></script>", 200
|
||||
)
|
||||
assert is_blocked(
|
||||
'<meta http-equiv="refresh" content="5; URL=/s?bm-verify=x">', 200
|
||||
)
|
||||
|
||||
|
||||
def test_float_handles_us_and_eu_price_formats():
|
||||
assert _float("1.234,56 €") == 1234.56
|
||||
assert _float("12,99 €") == 12.99
|
||||
assert _float("$1,234.56") == 1234.56
|
||||
assert _float("1234") == 1234.0
|
||||
|
||||
|
||||
def test_block_retry_proxy_uses_fresh_country_session(monkeypatch):
|
||||
geo_calls: list[str | None] = []
|
||||
sticky_calls: list[tuple[str, str | None]] = []
|
||||
|
||||
def get_geo_proxy_url(country: str | None = None):
|
||||
geo_calls.append(country)
|
||||
return f"http://geo-{country}"
|
||||
|
||||
def get_sticky_proxy_url(session_id: str, country: str | None = None):
|
||||
sticky_calls.append((session_id, country))
|
||||
return f"http://sticky-{country}-{session_id}"
|
||||
|
||||
monkeypatch.setattr(fetch, "get_geo_proxy_url", get_geo_proxy_url)
|
||||
monkeypatch.setattr(fetch, "get_sticky_proxy_url", get_sticky_proxy_url)
|
||||
|
||||
first = fetch._selected_proxy(None, "fr", 1, "https://www.amazon.fr/s?k=x")
|
||||
second = fetch._selected_proxy(None, "fr", 2, "https://www.amazon.fr/s?k=x")
|
||||
|
||||
assert first == "http://geo-fr"
|
||||
assert second.startswith("http://sticky-fr-amazon-fr-2-")
|
||||
assert geo_calls == ["fr"]
|
||||
assert sticky_calls[0][1] == "fr"
|
||||
|
||||
|
||||
def test_search_parser_extracts_cards_and_provenance():
|
||||
cards = parse_search_page(_fixture("search.html"), page=2)
|
||||
|
||||
assert [card["asin"] for card in cards] == ["B09V3KXJPB", "B09V3KXJP1"]
|
||||
assert cards[0]["categoryPageData"] == {
|
||||
"position": 1,
|
||||
"page": 2,
|
||||
"isSponsored": False,
|
||||
"isBestSeller": True,
|
||||
}
|
||||
assert cards[1]["categoryPageData"]["isSponsored"] is True
|
||||
|
||||
|
||||
def test_offer_and_seller_parsers():
|
||||
offers = parse_aod_offers(_fixture("offers.html"))
|
||||
seller = parse_seller(_fixture("seller.html"), seller_id="SELLER123")
|
||||
|
||||
assert len(offers) == 2
|
||||
assert offers[0]["price"]["value"] == 47.5
|
||||
assert offers[0]["seller"]["id"] == "SELLER123"
|
||||
assert offers[0]["isPinnedOffer"] is True
|
||||
assert seller["name"] == "Example Retailer"
|
||||
assert seller["averageRating"] == 4.8
|
||||
assert seller["reviewsCount"] == 1875
|
||||
|
||||
|
||||
def test_bestsellers_parser_extracts_ranked_products():
|
||||
items = parse_bestsellers_page(_fixture("bestsellers.html"))
|
||||
|
||||
assert [item["asin"] for item in items] == ["B09V3KXJPB", "B09V3KXJP1"]
|
||||
assert [item["bestsellerPageData"]["rank"] for item in items] == [1, 2]
|
||||
|
||||
|
||||
def test_location_route_filter_is_explicit():
|
||||
routes = ["PRODUCT", "OFFERS"]
|
||||
assert should_localize("product", routes)
|
||||
assert not should_localize("search", routes)
|
||||
|
||||
|
||||
async def test_location_session_mints_once_and_reuses_cache(monkeypatch):
|
||||
fetch._location_sessions.clear()
|
||||
calls: list[str] = []
|
||||
|
||||
async def sticky_proxy(_session_id: str, _country: str | None = None):
|
||||
return "http://sticky-proxy"
|
||||
|
||||
async def fetch_page(url: str, **_kwargs):
|
||||
calls.append(url)
|
||||
if len(calls) == 1:
|
||||
return FetchResult(
|
||||
status=200,
|
||||
html='<input name="anti-csrftoken-a2z" value="token-123">',
|
||||
url=url,
|
||||
cookies={"session-id": "session", "ubid-main": "ubid"},
|
||||
)
|
||||
return FetchResult(
|
||||
status=200,
|
||||
html='{"isValidAddress":1}',
|
||||
url=url,
|
||||
cookies={"session-id": "session"},
|
||||
)
|
||||
|
||||
monkeypatch.setattr(fetch, "_sticky_proxy", sticky_proxy)
|
||||
monkeypatch.setattr(fetch, "fetch_page", fetch_page)
|
||||
|
||||
first = await get_location_session(
|
||||
"www.amazon.com", zip_code="10001", country_code="US"
|
||||
)
|
||||
second = await get_location_session(
|
||||
"www.amazon.com", zip_code="10001", country_code="US"
|
||||
)
|
||||
|
||||
assert first is second
|
||||
assert first is not None
|
||||
assert first.proxy == "http://sticky-proxy"
|
||||
assert first.country_code == "US"
|
||||
assert len(calls) == 2
|
||||
59
surfsense_backend/tests/unit/platforms/amazon/test_proxy.py
Normal file
59
surfsense_backend/tests/unit/platforms/amazon/test_proxy.py
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.config import Config
|
||||
from app.utils.proxy.providers.dataimpulse import DataImpulseProvider
|
||||
|
||||
|
||||
def test_dataimpulse_sticky_url_is_deterministic(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
Config,
|
||||
"PROXY_URL",
|
||||
"http://token__cr.us:secret@gw.dataimpulse.com:823",
|
||||
)
|
||||
provider = DataImpulseProvider()
|
||||
|
||||
first = provider.get_sticky_proxy_url("location-123")
|
||||
second = provider.get_sticky_proxy_url("location-123")
|
||||
|
||||
assert first == second
|
||||
assert "token__cr.us__sid.location-123" in first
|
||||
assert first.endswith("@gw.dataimpulse.com:823")
|
||||
|
||||
|
||||
def test_dataimpulse_sticky_url_replaces_existing_session(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
Config,
|
||||
"PROXY_URL",
|
||||
"http://token__cr.us__sid.old:secret@gw.dataimpulse.com:823",
|
||||
)
|
||||
|
||||
result = DataImpulseProvider().get_sticky_proxy_url("new")
|
||||
|
||||
assert "__sid.old" not in result
|
||||
assert result.count("__sid.new") == 1
|
||||
|
||||
|
||||
def test_dataimpulse_sticky_url_rewrites_country(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
Config,
|
||||
"PROXY_URL",
|
||||
"http://token__cr.us:secret@gw.dataimpulse.com:823",
|
||||
)
|
||||
|
||||
result = DataImpulseProvider().get_sticky_proxy_url("new", "gb")
|
||||
|
||||
assert "token__cr.gb__sid.new" in result
|
||||
assert "__cr.us" not in result
|
||||
|
||||
|
||||
def test_dataimpulse_sticky_url_rejects_empty_session(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
Config,
|
||||
"PROXY_URL",
|
||||
"http://token:secret@gw.dataimpulse.com:823",
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
DataImpulseProvider().get_sticky_proxy_url("...")
|
||||
177
surfsense_backend/tests/unit/platforms/amazon/test_skeleton.py
Normal file
177
surfsense_backend/tests/unit/platforms/amazon/test_skeleton.py
Normal file
|
|
@ -0,0 +1,177 @@
|
|||
"""Offline contract tests for the Amazon Product Scraper.
|
||||
|
||||
Deterministic (no network, no live Amazon HTML): asserts the input defaults, the
|
||||
full output-item serialization contract, the error-item shape, and URL
|
||||
classification. The live fetch / parse flows are exercised by the e2e script in
|
||||
later milestones, not here.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.proprietary.platforms.amazon import (
|
||||
AmazonScrapeInput,
|
||||
ErrorItem,
|
||||
ProductItem,
|
||||
scrape_products,
|
||||
)
|
||||
from app.proprietary.platforms.amazon.url_resolver import extract_asin, resolve_url
|
||||
|
||||
# A complete input payload should validate without removing optional fields.
|
||||
_SPEC_INPUT = {
|
||||
"categoryOrProductUrls": [{"url": "https://www.amazon.com/s?k=keyboard"}],
|
||||
"maxItemsPerStartUrl": 100,
|
||||
"language": "en",
|
||||
"proxyCountry": "AUTO_SELECT_PROXY_COUNTRY",
|
||||
"maxSearchPagesPerStartUrl": 9999,
|
||||
"maxOffers": 0,
|
||||
"scrapeSellers": False,
|
||||
"useCaptchaSolver": False,
|
||||
"scrapeProductVariantPrices": False,
|
||||
"scrapeProductDetails": True,
|
||||
"countryCode": "US",
|
||||
"zipCode": "10001",
|
||||
"locationDeliverableRoutes": ["PRODUCT", "SEARCH", "OFFERS"],
|
||||
}
|
||||
|
||||
|
||||
def test_input_has_no_auth_fields():
|
||||
# Public, anonymous only: no auth-shaped field may exist on the input surface.
|
||||
forbidden = {"username", "password", "token", "login", "auth", "credentials"}
|
||||
assert forbidden.isdisjoint(AmazonScrapeInput.model_fields)
|
||||
|
||||
|
||||
def test_scrape_input_defaults_match_contract():
|
||||
inp = AmazonScrapeInput(
|
||||
categoryOrProductUrls=[{"url": "https://www.amazon.com/dp/B0"}]
|
||||
)
|
||||
assert inp.maxItemsPerStartUrl is None
|
||||
assert inp.maxSearchPagesPerStartUrl == 9999
|
||||
assert inp.maxProductVariantsAsSeparateResults == 0
|
||||
assert inp.maxOffers == 0
|
||||
assert inp.proxyCountry == "AUTO_SELECT_PROXY_COUNTRY"
|
||||
assert inp.language is None
|
||||
assert inp.countryCode is None
|
||||
assert inp.zipCode is None
|
||||
assert inp.locationDeliverableRoutes == ["PRODUCT", "SEARCH", "OFFERS"]
|
||||
assert inp.scrapeSellers is False
|
||||
assert inp.useCaptchaSolver is False
|
||||
assert inp.scrapeProductVariantPrices is False
|
||||
assert inp.scrapeProductDetails is True
|
||||
|
||||
|
||||
def test_complete_payload_validates():
|
||||
inp = AmazonScrapeInput(**_SPEC_INPUT)
|
||||
assert inp.maxItemsPerStartUrl == 100
|
||||
assert inp.countryCode == "US"
|
||||
|
||||
|
||||
def test_input_allows_extra_inert_fields():
|
||||
# extra="allow": unknown add-ons (e.g. upstream proxy config) are accepted.
|
||||
inp = AmazonScrapeInput(
|
||||
categoryOrProductUrls=[{"url": "https://www.amazon.com/dp/B0"}],
|
||||
proxyConfiguration={"useResidentialProxy": True},
|
||||
someFutureAddon=123,
|
||||
)
|
||||
assert inp.model_dump().get("someFutureAddon") == 123
|
||||
|
||||
|
||||
def test_output_item_serializes_full_shape():
|
||||
item = ProductItem(asin="B08EXAMPLE01").to_output()
|
||||
assert item["asin"] == "B08EXAMPLE01"
|
||||
# Unsourced scalars are still present as None (consumers never KeyError).
|
||||
assert item["title"] is None
|
||||
assert item["monthlyPurchaseVolume"] is None
|
||||
# List fields default to [].
|
||||
assert item["features"] == []
|
||||
assert item["offers"] == []
|
||||
assert item["productPageReviews"] == []
|
||||
# Typed-but-unset nested objects are None.
|
||||
assert item["price"] is None
|
||||
assert item["seller"] is None
|
||||
|
||||
|
||||
def test_stars_breakdown_round_trips_digit_keys():
|
||||
item = ProductItem(
|
||||
stars=4.8, starsBreakdown={"5star": 0.86, "1star": 0.01}
|
||||
).to_output()
|
||||
# by_alias serialization restores Amazon's digit-prefixed keys.
|
||||
assert item["starsBreakdown"]["5star"] == 0.86
|
||||
assert item["starsBreakdown"]["1star"] == 0.01
|
||||
assert item["starsBreakdown"]["4star"] is None
|
||||
|
||||
|
||||
def test_error_item_shape():
|
||||
err = ErrorItem(
|
||||
error="product_not_found",
|
||||
errorDescription="Loaded a 404 page.",
|
||||
input="https://www.amazon.com/dp/B0XXXXXXXX",
|
||||
url="https://www.amazon.com/dp/B0XXXXXXXX",
|
||||
).model_dump()
|
||||
assert set(err) >= {"error", "errorDescription", "input", "url"}
|
||||
assert err["error"] == "product_not_found"
|
||||
|
||||
|
||||
def test_extract_asin():
|
||||
assert extract_asin("https://www.amazon.com/dp/B09X7MPX8L") == "B09X7MPX8L"
|
||||
assert extract_asin("https://www.amazon.com/gp/product/B09X7MPX8L/ref=x") == (
|
||||
"B09X7MPX8L"
|
||||
)
|
||||
assert extract_asin("https://www.amazon.com/s?k=keyboard") is None
|
||||
|
||||
|
||||
def test_resolve_url_classifies_all_kinds():
|
||||
product = resolve_url("https://www.amazon.com/dp/B0EXAMPLE1")
|
||||
assert product is not None
|
||||
assert product.kind == "product"
|
||||
assert product.asin == "B0EXAMPLE1"
|
||||
assert product.marketplace == "com"
|
||||
|
||||
search = resolve_url("https://www.amazon.com/s?k=keyboard")
|
||||
assert search is not None and search.kind == "search"
|
||||
|
||||
# A category URL with no /s path but a bbn/rh query still classifies as search.
|
||||
category = resolve_url(
|
||||
"https://www.amazon.de/s?i=specialty-aps&bbn=16225007011&rh=n%3A16225007011"
|
||||
)
|
||||
assert category is not None
|
||||
assert category.kind == "search"
|
||||
assert category.marketplace == "de"
|
||||
|
||||
bestsellers = resolve_url(
|
||||
"https://www.amazon.com/Best-Sellers-Electronics/zgbs/electronics"
|
||||
)
|
||||
assert bestsellers is not None and bestsellers.kind == "bestsellers"
|
||||
|
||||
shortened = resolve_url("https://a.co/d/abcd123")
|
||||
assert shortened is not None and shortened.kind == "shortened"
|
||||
|
||||
|
||||
def test_resolve_url_rejects_non_amazon_and_unrecognized():
|
||||
assert resolve_url("https://example.com/dp/B08EXAMPLE1") is None
|
||||
# An Amazon host but an unrecognized path (e.g. the homepage) is unrecognized.
|
||||
assert resolve_url("https://www.amazon.com/") is None
|
||||
|
||||
|
||||
async def test_iter_products_unrecognized_yields_error():
|
||||
# A junk / non-Amazon URL yields exactly one invalid_url error item.
|
||||
items = await scrape_products(
|
||||
AmazonScrapeInput(categoryOrProductUrls=[{"url": "https://example.com/foo"}])
|
||||
)
|
||||
assert len(items) == 1
|
||||
assert items[0]["error"] == "invalid_url"
|
||||
assert items[0]["input"] == "https://example.com/foo"
|
||||
|
||||
|
||||
def test_valid_product_url_is_ready_for_dispatch():
|
||||
resolved = resolve_url("https://www.amazon.com/dp/B0EXAMPLE1")
|
||||
assert resolved is not None
|
||||
assert resolved.kind == "product"
|
||||
|
||||
|
||||
async def test_iter_products_missing_url_key_yields_error():
|
||||
# An entry without a "url" key is treated as an invalid start URL, not a crash.
|
||||
items = await scrape_products(
|
||||
AmazonScrapeInput(categoryOrProductUrls=[{"noturl": "x"}])
|
||||
)
|
||||
assert len(items) == 1
|
||||
assert items[0]["error"] == "invalid_url"
|
||||
|
|
@ -59,7 +59,7 @@ def _due_connector(connector_type: SearchSourceConnectorType) -> SimpleNamespace
|
|||
return SimpleNamespace(
|
||||
id=42,
|
||||
connector_type=connector_type,
|
||||
search_space_id=7,
|
||||
workspace_id=7,
|
||||
user_id="00000000-0000-0000-0000-000000000001",
|
||||
config={},
|
||||
periodic_indexing_enabled=True,
|
||||
|
|
@ -86,13 +86,8 @@ async def _run_checker(monkeypatch: pytest.MonkeyPatch, connector: SimpleNamespa
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_due_bookstack_connector_dispatches_indexing_task(monkeypatch):
|
||||
"""A due BookStack connector must dispatch index_bookstack_pages_task.
|
||||
|
||||
Regression test for the connector type missing from the scheduler's
|
||||
task_map, which made periodic BookStack syncs silently no-op with only a
|
||||
"No task found" warning.
|
||||
"""
|
||||
async def test_due_bookstack_connector_disables_deprecated_indexing(monkeypatch):
|
||||
"""A due BookStack connector is retired from periodic indexing, not dispatched."""
|
||||
from app.tasks.celery_tasks import connector_tasks
|
||||
|
||||
task_mock = MagicMock()
|
||||
|
|
@ -101,16 +96,9 @@ async def test_due_bookstack_connector_dispatches_indexing_task(monkeypatch):
|
|||
connector = _due_connector(SearchSourceConnectorType.BOOKSTACK_CONNECTOR)
|
||||
session = await _run_checker(monkeypatch, connector)
|
||||
|
||||
task_mock.delay.assert_called_once_with(
|
||||
connector.id,
|
||||
connector.search_space_id,
|
||||
str(connector.user_id),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
# The next run must be rescheduled, otherwise the connector stays "due"
|
||||
# and is re-examined every minute.
|
||||
assert connector.next_scheduled_at > datetime.now(UTC)
|
||||
task_mock.delay.assert_not_called()
|
||||
assert connector.periodic_indexing_enabled is False
|
||||
assert connector.next_scheduled_at is None
|
||||
assert session.commits == 1
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -44,6 +44,51 @@ def test_location_stops_at_next_param(monkeypatch: pytest.MonkeyPatch) -> None:
|
|||
assert DataImpulseProvider().get_location() == "de"
|
||||
|
||||
|
||||
def test_geo_proxy_url_rewrites_country_suffix(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(Config, "PROXY_URL", _URL)
|
||||
|
||||
result = DataImpulseProvider().get_geo_proxy_url("gb")
|
||||
|
||||
assert result == "http://tok123__cr.gb:secret@gw.dataimpulse.com:823"
|
||||
|
||||
|
||||
def test_geo_proxy_url_replaces_existing_country_once(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
Config,
|
||||
"PROXY_URL",
|
||||
"http://tok123__cr.us__sid.old:secret@gw.dataimpulse.com:823",
|
||||
)
|
||||
|
||||
result = DataImpulseProvider().get_geo_proxy_url("de")
|
||||
|
||||
assert "__cr.us" not in result
|
||||
assert result.count("__cr.de") == 1
|
||||
assert "__sid.old" not in result
|
||||
|
||||
|
||||
def test_geo_proxy_url_without_country_keeps_base_url(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(Config, "PROXY_URL", _URL)
|
||||
|
||||
assert DataImpulseProvider().get_geo_proxy_url(None) == _URL
|
||||
assert DataImpulseProvider().get_geo_proxy_url("") == _URL
|
||||
|
||||
|
||||
def test_sticky_proxy_url_composes_country_and_session(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(Config, "PROXY_URL", _URL)
|
||||
|
||||
result = DataImpulseProvider().get_sticky_proxy_url("location-123", "gb")
|
||||
|
||||
assert result == (
|
||||
"http://tok123__cr.gb__sid.location-123:secret@gw.dataimpulse.com:823"
|
||||
)
|
||||
|
||||
|
||||
def test_no_country_suffix_yields_empty_location(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue