mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-26 23:51:14 +02:00
Merge remote-tracking branch 'upstream/dev' into feature-indeed-jobs-scraper
# Conflicts: # README.es.md # README.hi.md # README.md # README.pt-BR.md # README.zh-CN.md # surfsense_backend/tests/unit/capabilities/google_maps/test_registry.py # surfsense_backend/tests/unit/capabilities/reddit/test_registry.py # surfsense_backend/tests/unit/capabilities/youtube/test_registry.py # surfsense_mcp/mcp_server/features/scrapers/__init__.py # surfsense_web/content/docs/connectors/index.mdx # surfsense_web/content/docs/connectors/native/index.mdx # surfsense_web/content/docs/connectors/native/meta.json # surfsense_web/content/docs/how-to/mcp-server.mdx # surfsense_web/lib/connectors-marketing/index.ts # surfsense_web/lib/playground/catalog.ts
This commit is contained in:
CREDO23
commit
91aa265afb
259 changed files with 13867 additions and 1883 deletions
|
|
@ -27,6 +27,7 @@ pytestmark = pytest.mark.unit
|
|||
# specialist is a deliberate product change and must be reflected here.
|
||||
_EXPECTED_SUBAGENTS = frozenset(
|
||||
{
|
||||
"amazon",
|
||||
"deliverables",
|
||||
"dropbox",
|
||||
"google_drive",
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
|
||||
|
|
@ -0,0 +1 @@
|
|||
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from app.capabilities.amazon.scrape.executor import build_scrape_executor
|
||||
from app.capabilities.amazon.scrape.schemas import MAX_AMAZON_RESULTS, ScrapeInput
|
||||
from app.proprietary.platforms.amazon import AmazonScrapeInput
|
||||
|
||||
|
||||
class _FakeScraper:
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[tuple[AmazonScrapeInput, int | None]] = []
|
||||
|
||||
async def __call__(
|
||||
self, input_model: AmazonScrapeInput, *, limit: int | None = None
|
||||
) -> list[dict]:
|
||||
self.calls.append((input_model, limit))
|
||||
return [{"asin": "B09V3KXJPB", "title": "Product"}]
|
||||
|
||||
|
||||
async def test_executor_maps_agent_input_to_scraper_input():
|
||||
scraper = _FakeScraper()
|
||||
execute = build_scrape_executor(scraper)
|
||||
|
||||
output = await execute(
|
||||
ScrapeInput(
|
||||
search_terms=["wireless mouse"],
|
||||
urls=["https://www.amazon.com/dp/B09V3KXJPB"],
|
||||
max_items=5,
|
||||
max_offers=2,
|
||||
include_sellers=True,
|
||||
zip_code="10001",
|
||||
country_code="US",
|
||||
)
|
||||
)
|
||||
|
||||
assert output.items[0].asin == "B09V3KXJPB"
|
||||
input_model, limit = scraper.calls[0]
|
||||
assert input_model.categoryOrProductUrls == [
|
||||
{"url": "https://www.amazon.com/dp/B09V3KXJPB"},
|
||||
{"url": "https://www.amazon.com/s?k=wireless+mouse"},
|
||||
]
|
||||
assert input_model.maxItemsPerStartUrl == 5
|
||||
assert input_model.maxOffers == 2
|
||||
assert input_model.scrapeSellers is True
|
||||
assert input_model.zipCode == "10001"
|
||||
assert limit == MAX_AMAZON_RESULTS
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app.capabilities.amazon.scrape.schemas import (
|
||||
MAX_AMAZON_RESULTS,
|
||||
ScrapeInput,
|
||||
ScrapeOutput,
|
||||
)
|
||||
|
||||
|
||||
def test_estimated_units_cover_search_and_direct_variant_fanout():
|
||||
payload = ScrapeInput(
|
||||
search_terms=["mouse", "keyboard"],
|
||||
urls=["https://www.amazon.com/dp/B09V3KXJPB"],
|
||||
max_items=20,
|
||||
max_variants=3,
|
||||
)
|
||||
|
||||
assert payload.estimated_units == 44
|
||||
|
||||
|
||||
def test_estimated_units_respect_hard_run_ceiling():
|
||||
payload = ScrapeInput(search_terms=["x"] * 20, max_items=100)
|
||||
assert payload.estimated_units == MAX_AMAZON_RESULTS
|
||||
|
||||
|
||||
def test_at_least_one_source_is_required():
|
||||
with pytest.raises(ValidationError):
|
||||
ScrapeInput()
|
||||
|
||||
|
||||
def test_error_items_are_not_billable():
|
||||
output = ScrapeOutput(
|
||||
items=[
|
||||
{"asin": "B09V3KXJPB", "title": "Product"},
|
||||
{"error": "product_not_found", "errorDescription": "Missing"},
|
||||
]
|
||||
)
|
||||
|
||||
assert output.billable_units == 1
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from app.capabilities.amazon.scrape.schemas import ScrapeInput, ScrapeOutput
|
||||
from app.capabilities.core import BillingUnit
|
||||
from app.capabilities.core.store import get_capability
|
||||
|
||||
|
||||
def test_amazon_scrape_is_registered_and_metered():
|
||||
capability = get_capability("amazon.scrape")
|
||||
|
||||
assert capability.input_schema is ScrapeInput
|
||||
assert capability.output_schema is ScrapeOutput
|
||||
assert capability.billing_unit is BillingUnit.AMAZON_PRODUCT
|
||||
|
|
@ -7,15 +7,15 @@ import pytest
|
|||
from app.capabilities import (
|
||||
google_maps, # noqa: F401 — importing the namespace registers its verbs
|
||||
)
|
||||
from app.capabilities.core import BillingUnit
|
||||
from app.capabilities.core.store import get_capability
|
||||
from app.capabilities.core.types import BillingUnit
|
||||
from app.capabilities.google_maps.reviews.schemas import ReviewsInput, ReviewsOutput
|
||||
from app.capabilities.google_maps.scrape.schemas import ScrapeInput, ScrapeOutput
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def test_google_maps_scrape_is_registered_and_billed_per_place():
|
||||
def test_google_maps_scrape_is_registered_and_billable():
|
||||
cap = get_capability("google_maps.scrape")
|
||||
|
||||
assert cap.name == "google_maps.scrape"
|
||||
|
|
@ -24,7 +24,7 @@ def test_google_maps_scrape_is_registered_and_billed_per_place():
|
|||
assert cap.billing_unit is BillingUnit.GOOGLE_MAPS_PLACE
|
||||
|
||||
|
||||
def test_google_maps_reviews_is_registered_and_billed_per_review():
|
||||
def test_google_maps_reviews_is_registered_and_billable():
|
||||
cap = get_capability("google_maps.reviews")
|
||||
|
||||
assert cap.name == "google_maps.reviews"
|
||||
|
|
|
|||
|
|
@ -7,14 +7,14 @@ import pytest
|
|||
from app.capabilities import (
|
||||
reddit, # noqa: F401 — importing the namespace registers its verbs
|
||||
)
|
||||
from app.capabilities.core import BillingUnit
|
||||
from app.capabilities.core.store import get_capability
|
||||
from app.capabilities.core.types import BillingUnit
|
||||
from app.capabilities.reddit.scrape.schemas import ScrapeInput, ScrapeOutput
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def test_reddit_scrape_is_registered_and_billed_per_item():
|
||||
def test_reddit_scrape_is_registered_and_billable():
|
||||
cap = get_capability("reddit.scrape")
|
||||
|
||||
assert cap.name == "reddit.scrape"
|
||||
|
|
|
|||
|
|
@ -7,15 +7,15 @@ import pytest
|
|||
from app.capabilities import (
|
||||
youtube, # noqa: F401 — importing the namespace registers its verbs
|
||||
)
|
||||
from app.capabilities.core import BillingUnit
|
||||
from app.capabilities.core.store import get_capability
|
||||
from app.capabilities.core.types import BillingUnit
|
||||
from app.capabilities.youtube.comments.schemas import CommentsInput, CommentsOutput
|
||||
from app.capabilities.youtube.scrape.schemas import ScrapeInput, ScrapeOutput
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def test_youtube_scrape_is_registered_and_billed_per_video():
|
||||
def test_youtube_scrape_is_registered_and_billable():
|
||||
cap = get_capability("youtube.scrape")
|
||||
|
||||
assert cap.name == "youtube.scrape"
|
||||
|
|
@ -24,7 +24,7 @@ def test_youtube_scrape_is_registered_and_billed_per_video():
|
|||
assert cap.billing_unit is BillingUnit.YOUTUBE_VIDEO
|
||||
|
||||
|
||||
def test_youtube_comments_is_registered_and_billed_per_comment():
|
||||
def test_youtube_comments_is_registered_and_billable():
|
||||
cap = get_capability("youtube.comments")
|
||||
|
||||
assert cap.name == "youtube.comments"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,17 @@
|
|||
<!doctype html>
|
||||
<html lang="en">
|
||||
<body>
|
||||
<div id="gridItemRoot">
|
||||
<span class="zg-bdg-text">#1</span>
|
||||
<a href="/dp/B09V3KXJPB"><span class="_cDEzb_p13n-sc-css-line-clamp-3_g3dy1">Example Wireless Headphones</span></a>
|
||||
<span class="a-price"><span class="a-offscreen">$49.99</span></span>
|
||||
<span class="a-icon-alt">4.6 out of 5 stars</span>
|
||||
<img src="https://images.example/headphones.jpg" alt="Example Wireless Headphones">
|
||||
</div>
|
||||
<div id="gridItemRoot">
|
||||
<span class="zg-bdg-text">#2</span>
|
||||
<a href="/dp/B09V3KXJP1"><span class="_cDEzb_p13n-sc-css-line-clamp-3_g3dy1">Example Earbuds</span></a>
|
||||
<span class="a-price"><span class="a-offscreen">$29.99</span></span>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head><title>Robot Check</title></head>
|
||||
<body>
|
||||
<form action="/errors/validateCaptcha">
|
||||
<p>Enter the characters you see below</p>
|
||||
</form>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
<!doctype html>
|
||||
<html lang="en">
|
||||
<body>
|
||||
<div id="aod-offer">
|
||||
<div id="aod-pinned-offer"></div>
|
||||
<div id="aod-offer-heading">New</div>
|
||||
<span class="a-price"><span class="a-offscreen">$47.50</span></span>
|
||||
<div id="aod-offer-soldBy"><a href="/sp?seller=SELLER123">Example Retailer</a></div>
|
||||
<div id="mir-layout-DELIVERY_BLOCK">FREE delivery tomorrow</div>
|
||||
</div>
|
||||
<div id="aod-offer">
|
||||
<div id="aod-offer-heading">Used - Very Good</div>
|
||||
<span class="a-price"><span class="a-offscreen">$39.00</span></span>
|
||||
<div id="aod-offer-soldBy"><a href="/sp?seller=SELLER456">Second Seller</a></div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
<!doctype html>
|
||||
<html lang="en">
|
||||
<body>
|
||||
<div id="wayfinding-breadcrumbs_feature_div"><ul><li><a href="/b?node=172282">Electronics</a></li><li><a href="/b?node=667846011">Audio</a></li></ul></div>
|
||||
<span id="productTitle">Example Wireless Headphones</span>
|
||||
<a id="bylineInfo" href="/stores/Example">Visit the Example Store</a>
|
||||
<a id="sellerProfileTriggerId" href="/sp?seller=SELLER123">Example Retailer</a>
|
||||
<div id="corePrice_feature_div">
|
||||
<span class="priceToPay"><span class="a-offscreen">$49.99</span></span>
|
||||
<span class="basisPrice"><span class="a-offscreen">$79.99</span></span>
|
||||
</div>
|
||||
<div id="averageCustomerReviews">
|
||||
<span id="acrPopover">4.6 out of 5 stars</span>
|
||||
<span id="acrCustomerReviewText">1,234 ratings</span>
|
||||
<a id="acrCustomerReviewLink" href="/product-reviews/B09V3KXJPB">Reviews</a>
|
||||
</div>
|
||||
<table id="histogramTable">
|
||||
<tr><td class="a-text-left">5 star</td><td class="a-text-right">80%</td></tr>
|
||||
<tr><td class="a-text-left">4 star</td><td class="a-text-right">12%</td></tr>
|
||||
</table>
|
||||
<div id="availability">In Stock</div>
|
||||
<div id="social-proofing-faceout-title-tk_bought">2K+ bought in past month</div>
|
||||
<div id="feature-bullets"><ul><li><span class="a-list-item">Long battery life</span></li><li><span class="a-list-item">Noise cancelling</span></li></ul></div>
|
||||
<div id="productDescription">Detailed product description.</div>
|
||||
<img id="landingImage" src="https://images.example/thumb.jpg" data-old-hires="https://images.example/full.jpg" alt="Example Wireless Headphones">
|
||||
<div id="aplus"><p>Premium materials and comfort.</p><img src="https://images.example/aplus.jpg" alt="Premium materials and comfort"></div>
|
||||
<div id="brandStory_feature_div">Designed for everyday listening.</div>
|
||||
<div id="detailBulletsWrapper_feature_div"><ul><li>Best Sellers Rank: #12 in Electronics #3 in Headphones</li></ul></div>
|
||||
<table id="productDetails_techSpec_section_1"><tr><th>Model</th><td>WH-100</td></tr></table>
|
||||
<div id="productOverview_feature_div"><table><tr><td>Color</td><td>Black</td></tr></table></div>
|
||||
<div id="acBadge_feature_div">Amazon's Choice</div>
|
||||
<script>
|
||||
window.twister = {
|
||||
"dimensionValuesDisplayData": {
|
||||
"B09V3KXJPB": ["Black"],
|
||||
"B09V3KXJP1": ["Blue"]
|
||||
},
|
||||
"variationValues": {"color_name": ["Black", "Blue"]}
|
||||
};
|
||||
</script>
|
||||
<div data-hook="review" id="R1">
|
||||
<span class="a-profile-name">Alex</span>
|
||||
<a data-hook="review-title" href="/gp/customer-reviews/R1">Excellent sound</a>
|
||||
<i data-hook="review-star-rating">5.0 out of 5 stars</i>
|
||||
<span data-hook="review-date">Reviewed on July 1, 2026</span>
|
||||
<span data-hook="review-body">Clear and comfortable.</span>
|
||||
<span data-hook="avp-badge">Verified Purchase</span>
|
||||
<span data-hook="helpful-vote-statement">12 people found this helpful</span>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
<!doctype html>
|
||||
<html lang="en">
|
||||
<body>
|
||||
<div data-component-type="s-search-result" data-asin="B09V3KXJPB" class="s-result-item">
|
||||
<h2><a href="/dp/B09V3KXJPB"><span>Example Wireless Headphones</span></a></h2>
|
||||
<span class="a-price"><span class="a-offscreen">$49.99</span></span>
|
||||
<span class="a-icon-alt">4.6 out of 5 stars</span>
|
||||
<span class="s-underline-text">1,234</span>
|
||||
<img class="s-image" src="https://images.example/headphones.jpg" alt="Example Wireless Headphones">
|
||||
<span class="a-badge-text">Best Seller</span>
|
||||
</div>
|
||||
<div data-component-type="s-search-result" data-asin="B09V3KXJP1" class="s-result-item">
|
||||
<span>Sponsored</span>
|
||||
<h2><a href="/dp/B09V3KXJP1"><span>Example Earbuds</span></a></h2>
|
||||
<span class="a-price"><span class="a-offscreen">$29.99</span></span>
|
||||
<span class="a-icon-alt">4.2 out of 5 stars</span>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
<!doctype html>
|
||||
<html lang="en">
|
||||
<body>
|
||||
<h1 id="sellerName">Example Retailer</h1>
|
||||
<div id="seller-feedback-summary">1,875 ratings</div>
|
||||
<table id="feedback-summary-table">
|
||||
<tr><td><span class="a-icon-alt">4.8 out of 5 stars</span></td></tr>
|
||||
<tr><td>1,875 ratings</td></tr>
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
||||
220
surfsense_backend/tests/unit/platforms/amazon/test_flows.py
Normal file
220
surfsense_backend/tests/unit/platforms/amazon/test_flows.py
Normal file
|
|
@ -0,0 +1,220 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from app.proprietary.platforms.amazon import AmazonScrapeInput, scrape_products, scraper
|
||||
from app.proprietary.platforms.amazon.fetch import FetchResult
|
||||
from app.proprietary.platforms.amazon.url_resolver import resolve_url
|
||||
|
||||
_FIXTURES = Path(__file__).parent / "fixtures"
|
||||
|
||||
|
||||
def _fixture(name: str) -> str:
|
||||
return (_FIXTURES / name).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def _response(url: str, html: str, status: int = 200) -> FetchResult:
|
||||
return FetchResult(status=status, html=html, url=url, cookies={})
|
||||
|
||||
|
||||
async def test_product_flow_emits_parsed_item(monkeypatch):
|
||||
async def fetch_page(url: str, **_kwargs):
|
||||
return _response(url, _fixture("product.html"))
|
||||
|
||||
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
|
||||
items = await scrape_products(
|
||||
AmazonScrapeInput(
|
||||
categoryOrProductUrls=[{"url": "https://www.amazon.com/dp/B09V3KXJPB"}]
|
||||
)
|
||||
)
|
||||
|
||||
assert len(items) == 1
|
||||
assert items[0]["asin"] == "B09V3KXJPB"
|
||||
assert items[0]["title"] == "Example Wireless Headphones"
|
||||
|
||||
|
||||
async def test_product_flow_maps_not_found_to_error_item(monkeypatch):
|
||||
async def fetch_page(url: str, **_kwargs):
|
||||
return _response(url, "", status=404)
|
||||
|
||||
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
|
||||
items = await scrape_products(
|
||||
AmazonScrapeInput(
|
||||
categoryOrProductUrls=[{"url": "https://www.amazon.com/dp/B09V3KXJPB"}]
|
||||
)
|
||||
)
|
||||
|
||||
assert items[0]["error"] == "product_not_found"
|
||||
|
||||
|
||||
async def test_search_flow_honors_cap_and_stops_on_empty_page(monkeypatch):
|
||||
calls: list[str] = []
|
||||
|
||||
async def fetch_page(url: str, **_kwargs):
|
||||
calls.append(url)
|
||||
html = _fixture("search.html") if "page=1" in url else "<html></html>"
|
||||
return _response(url, html)
|
||||
|
||||
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
|
||||
items = await scrape_products(
|
||||
AmazonScrapeInput(
|
||||
categoryOrProductUrls=[{"url": "https://www.amazon.com/s?k=headphones"}],
|
||||
maxItemsPerStartUrl=1,
|
||||
maxSearchPagesPerStartUrl=5,
|
||||
scrapeProductDetails=False,
|
||||
)
|
||||
)
|
||||
|
||||
assert len(items) == 1
|
||||
assert items[0]["categoryPageData"]["position"] == 1
|
||||
assert len(calls) == 1
|
||||
|
||||
|
||||
async def test_search_flow_threads_marketplace_locale_to_fetch(monkeypatch):
|
||||
calls: list[dict[str, object]] = []
|
||||
|
||||
async def fetch_page(url: str, **kwargs):
|
||||
calls.append({"url": url, **kwargs})
|
||||
return _response(url, _fixture("search.html"))
|
||||
|
||||
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
|
||||
items = await scrape_products(
|
||||
AmazonScrapeInput(
|
||||
categoryOrProductUrls=[{"url": "https://www.amazon.co.uk/s?k=headphones"}],
|
||||
maxItemsPerStartUrl=1,
|
||||
scrapeProductDetails=False,
|
||||
)
|
||||
)
|
||||
|
||||
assert len(items) == 1
|
||||
assert calls[0]["country"] == "gb"
|
||||
assert calls[0]["accept_language"] == "en-GB"
|
||||
|
||||
|
||||
async def test_search_flow_returns_no_results_error(monkeypatch):
|
||||
async def fetch_page(url: str, **_kwargs):
|
||||
return _response(url, "<html></html>")
|
||||
|
||||
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
|
||||
items = await scrape_products(
|
||||
AmazonScrapeInput(
|
||||
categoryOrProductUrls=[{"url": "https://www.amazon.com/s?k=missing"}],
|
||||
scrapeProductDetails=False,
|
||||
)
|
||||
)
|
||||
|
||||
assert items[0]["error"] == "no_results_found"
|
||||
|
||||
|
||||
async def test_product_flow_enriches_offers_and_seller(monkeypatch):
|
||||
async def fetch_page(url: str, **_kwargs):
|
||||
return _response(url, _fixture("product.html"))
|
||||
|
||||
async def fetch_aod_html(*_args, **_kwargs):
|
||||
return _fixture("offers.html")
|
||||
|
||||
async def fetch_seller_html(*_args, **_kwargs):
|
||||
return _fixture("seller.html")
|
||||
|
||||
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
|
||||
monkeypatch.setattr(scraper, "fetch_aod_html", fetch_aod_html)
|
||||
monkeypatch.setattr(scraper, "fetch_seller_html", fetch_seller_html)
|
||||
items = await scrape_products(
|
||||
AmazonScrapeInput(
|
||||
categoryOrProductUrls=[{"url": "https://www.amazon.com/dp/B09V3KXJPB"}],
|
||||
maxOffers=1,
|
||||
scrapeSellers=True,
|
||||
)
|
||||
)
|
||||
|
||||
assert len(items[0]["offers"]) == 1
|
||||
assert items[0]["offers"][0]["seller"]["averageRating"] == 4.8
|
||||
assert items[0]["seller"]["reviewsCount"] == 1875
|
||||
|
||||
|
||||
async def test_bestsellers_card_only_flow(monkeypatch):
|
||||
async def fetch_page(url: str, **_kwargs):
|
||||
return _response(url, _fixture("bestsellers.html"))
|
||||
|
||||
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
|
||||
items = await scrape_products(
|
||||
AmazonScrapeInput(
|
||||
categoryOrProductUrls=[
|
||||
{
|
||||
"url": (
|
||||
"https://www.amazon.com/Best-Sellers-Electronics/"
|
||||
"zgbs/electronics"
|
||||
)
|
||||
}
|
||||
],
|
||||
maxItemsPerStartUrl=2,
|
||||
scrapeProductDetails=False,
|
||||
)
|
||||
)
|
||||
|
||||
assert [item["bestsellerPageData"]["rank"] for item in items] == [1, 2]
|
||||
|
||||
|
||||
async def test_shortlink_resolves_and_dispatches(monkeypatch):
|
||||
async def resolve(_url: str):
|
||||
return "https://www.amazon.com/dp/B09V3KXJPB"
|
||||
|
||||
async def fetch_page(url: str, **_kwargs):
|
||||
return _response(url, _fixture("product.html"))
|
||||
|
||||
monkeypatch.setattr(scraper, "resolve_shortlink", resolve)
|
||||
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
|
||||
items = await scrape_products(
|
||||
AmazonScrapeInput(categoryOrProductUrls=[{"url": "https://a.co/d/example"}])
|
||||
)
|
||||
|
||||
assert items[0]["asin"] == "B09V3KXJPB"
|
||||
assert items[0]["input"] == "https://a.co/d/example"
|
||||
|
||||
|
||||
async def test_product_flow_expands_variants_and_attaches_prices(monkeypatch):
|
||||
async def fetch_page(url: str, **_kwargs):
|
||||
return _response(url, _fixture("product.html"))
|
||||
|
||||
monkeypatch.setattr(scraper, "fetch_page", fetch_page)
|
||||
items = await scrape_products(
|
||||
AmazonScrapeInput(
|
||||
categoryOrProductUrls=[{"url": "https://www.amazon.com/dp/B09V3KXJPB"}],
|
||||
maxProductVariantsAsSeparateResults=1,
|
||||
scrapeProductVariantPrices=True,
|
||||
)
|
||||
)
|
||||
|
||||
assert [item["asin"] for item in items] == ["B09V3KXJPB", "B09V3KXJP1"]
|
||||
assert items[1]["originalAsin"] == "B09V3KXJPB"
|
||||
variant = next(
|
||||
detail
|
||||
for detail in items[0]["variantDetails"]
|
||||
if detail["asin"] == "B09V3KXJP1"
|
||||
)
|
||||
assert variant["price"]["value"] == 49.99
|
||||
|
||||
|
||||
async def test_route_outside_location_allowlist_skips_session_mint(monkeypatch):
|
||||
calls = 0
|
||||
|
||||
async def get_location_session(*_args, **_kwargs):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(scraper, "get_location_session", get_location_session)
|
||||
resolved = resolve_url("https://www.amazon.com/dp/B09V3KXJPB")
|
||||
assert resolved is not None
|
||||
context = await scraper._location_context(
|
||||
resolved,
|
||||
AmazonScrapeInput(
|
||||
categoryOrProductUrls=[{"url": resolved.url}],
|
||||
zipCode="10001",
|
||||
locationDeliverableRoutes=["OFFERS"],
|
||||
),
|
||||
"PRODUCT",
|
||||
)
|
||||
|
||||
assert context == (None, None, None, None)
|
||||
assert calls == 0
|
||||
33
surfsense_backend/tests/unit/platforms/amazon/test_locale.py
Normal file
33
surfsense_backend/tests/unit/platforms/amazon/test_locale.py
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from app.proprietary.platforms.amazon.locale import (
|
||||
accept_language_for,
|
||||
proxy_country_for,
|
||||
)
|
||||
|
||||
|
||||
def test_proxy_country_for_confirmed_marketplaces():
|
||||
assert proxy_country_for("com") == "us"
|
||||
assert proxy_country_for("co.uk") == "gb"
|
||||
assert proxy_country_for("de") == "de"
|
||||
assert proxy_country_for("fr") == "fr"
|
||||
assert proxy_country_for("it") == "it"
|
||||
assert proxy_country_for("es") == "es"
|
||||
|
||||
|
||||
def test_proxy_country_for_unknown_marketplace_falls_back():
|
||||
assert proxy_country_for("co.jp") is None
|
||||
assert proxy_country_for(None) is None
|
||||
|
||||
|
||||
def test_accept_language_for_marketplaces():
|
||||
assert accept_language_for("co.uk") == "en-GB"
|
||||
assert accept_language_for("de") == "de-DE"
|
||||
assert accept_language_for("fr") == "fr-FR"
|
||||
assert accept_language_for("it") == "it-IT"
|
||||
assert accept_language_for("es") == "es-ES"
|
||||
|
||||
|
||||
def test_accept_language_for_unknown_marketplace_defaults_to_us_english():
|
||||
assert accept_language_for("co.jp") == "en-US"
|
||||
assert accept_language_for(None) == "en-US"
|
||||
176
surfsense_backend/tests/unit/platforms/amazon/test_parsers.py
Normal file
176
surfsense_backend/tests/unit/platforms/amazon/test_parsers.py
Normal file
|
|
@ -0,0 +1,176 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from app.proprietary.platforms.amazon import fetch
|
||||
from app.proprietary.platforms.amazon.fetch import (
|
||||
FetchResult,
|
||||
get_location_session,
|
||||
is_blocked,
|
||||
should_localize,
|
||||
)
|
||||
from app.proprietary.platforms.amazon.parsers import (
|
||||
_float,
|
||||
parse_aod_offers,
|
||||
parse_bestsellers_page,
|
||||
parse_product,
|
||||
parse_search_page,
|
||||
parse_seller,
|
||||
)
|
||||
|
||||
_FIXTURES = Path(__file__).parent / "fixtures"
|
||||
|
||||
|
||||
def _fixture(name: str) -> str:
|
||||
return (_FIXTURES / name).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_product_parser_extracts_core_public_fields():
|
||||
item = parse_product(
|
||||
_fixture("product.html"),
|
||||
asin="B09V3KXJPB",
|
||||
url="https://www.amazon.com/dp/B09V3KXJPB",
|
||||
)
|
||||
|
||||
assert item["title"] == "Example Wireless Headphones"
|
||||
assert item["price"] == {"value": 49.99, "currency": "USD"}
|
||||
assert item["listPrice"] == {"value": 79.99, "currency": "USD"}
|
||||
assert item["stars"] == 4.6
|
||||
assert item["reviewsCount"] == 1234
|
||||
assert item["inStock"] is True
|
||||
assert item["features"] == ["Long battery life", "Noise cancelling"]
|
||||
assert item["starsBreakdown"]["5star"] == 0.8
|
||||
assert "B09V3KXJP1" in item["variantAsins"]
|
||||
assert item["variantAttributes"][0]["name"] == "color_name"
|
||||
assert item["productPageReviews"][0]["id"] == "R1"
|
||||
|
||||
|
||||
def test_block_detection_handles_status_and_markup():
|
||||
blocked = _fixture("blocked.html")
|
||||
assert is_blocked(blocked, 200)
|
||||
assert is_blocked("", 503)
|
||||
assert not is_blocked(_fixture("product.html"), 200)
|
||||
|
||||
|
||||
def test_block_detection_handles_waf_header_and_body_markers():
|
||||
assert is_blocked(
|
||||
"<html>ordinary status</html>",
|
||||
200,
|
||||
{"X-Amzn-Waf-Action": "challenge"},
|
||||
)
|
||||
assert is_blocked(
|
||||
"<script src='https://token.awswaf.com/challenge.js'></script>", 200
|
||||
)
|
||||
assert is_blocked(
|
||||
'<meta http-equiv="refresh" content="5; URL=/s?bm-verify=x">', 200
|
||||
)
|
||||
|
||||
|
||||
def test_float_handles_us_and_eu_price_formats():
|
||||
assert _float("1.234,56 €") == 1234.56
|
||||
assert _float("12,99 €") == 12.99
|
||||
assert _float("$1,234.56") == 1234.56
|
||||
assert _float("1234") == 1234.0
|
||||
|
||||
|
||||
def test_block_retry_proxy_uses_fresh_country_session(monkeypatch):
|
||||
geo_calls: list[str | None] = []
|
||||
sticky_calls: list[tuple[str, str | None]] = []
|
||||
|
||||
def get_geo_proxy_url(country: str | None = None):
|
||||
geo_calls.append(country)
|
||||
return f"http://geo-{country}"
|
||||
|
||||
def get_sticky_proxy_url(session_id: str, country: str | None = None):
|
||||
sticky_calls.append((session_id, country))
|
||||
return f"http://sticky-{country}-{session_id}"
|
||||
|
||||
monkeypatch.setattr(fetch, "get_geo_proxy_url", get_geo_proxy_url)
|
||||
monkeypatch.setattr(fetch, "get_sticky_proxy_url", get_sticky_proxy_url)
|
||||
|
||||
first = fetch._selected_proxy(None, "fr", 1, "https://www.amazon.fr/s?k=x")
|
||||
second = fetch._selected_proxy(None, "fr", 2, "https://www.amazon.fr/s?k=x")
|
||||
|
||||
assert first == "http://geo-fr"
|
||||
assert second.startswith("http://sticky-fr-amazon-fr-2-")
|
||||
assert geo_calls == ["fr"]
|
||||
assert sticky_calls[0][1] == "fr"
|
||||
|
||||
|
||||
def test_search_parser_extracts_cards_and_provenance():
|
||||
cards = parse_search_page(_fixture("search.html"), page=2)
|
||||
|
||||
assert [card["asin"] for card in cards] == ["B09V3KXJPB", "B09V3KXJP1"]
|
||||
assert cards[0]["categoryPageData"] == {
|
||||
"position": 1,
|
||||
"page": 2,
|
||||
"isSponsored": False,
|
||||
"isBestSeller": True,
|
||||
}
|
||||
assert cards[1]["categoryPageData"]["isSponsored"] is True
|
||||
|
||||
|
||||
def test_offer_and_seller_parsers():
|
||||
offers = parse_aod_offers(_fixture("offers.html"))
|
||||
seller = parse_seller(_fixture("seller.html"), seller_id="SELLER123")
|
||||
|
||||
assert len(offers) == 2
|
||||
assert offers[0]["price"]["value"] == 47.5
|
||||
assert offers[0]["seller"]["id"] == "SELLER123"
|
||||
assert offers[0]["isPinnedOffer"] is True
|
||||
assert seller["name"] == "Example Retailer"
|
||||
assert seller["averageRating"] == 4.8
|
||||
assert seller["reviewsCount"] == 1875
|
||||
|
||||
|
||||
def test_bestsellers_parser_extracts_ranked_products():
|
||||
items = parse_bestsellers_page(_fixture("bestsellers.html"))
|
||||
|
||||
assert [item["asin"] for item in items] == ["B09V3KXJPB", "B09V3KXJP1"]
|
||||
assert [item["bestsellerPageData"]["rank"] for item in items] == [1, 2]
|
||||
|
||||
|
||||
def test_location_route_filter_is_explicit():
|
||||
routes = ["PRODUCT", "OFFERS"]
|
||||
assert should_localize("product", routes)
|
||||
assert not should_localize("search", routes)
|
||||
|
||||
|
||||
async def test_location_session_mints_once_and_reuses_cache(monkeypatch):
|
||||
fetch._location_sessions.clear()
|
||||
calls: list[str] = []
|
||||
|
||||
async def sticky_proxy(_session_id: str, _country: str | None = None):
|
||||
return "http://sticky-proxy"
|
||||
|
||||
async def fetch_page(url: str, **_kwargs):
|
||||
calls.append(url)
|
||||
if len(calls) == 1:
|
||||
return FetchResult(
|
||||
status=200,
|
||||
html='<input name="anti-csrftoken-a2z" value="token-123">',
|
||||
url=url,
|
||||
cookies={"session-id": "session", "ubid-main": "ubid"},
|
||||
)
|
||||
return FetchResult(
|
||||
status=200,
|
||||
html='{"isValidAddress":1}',
|
||||
url=url,
|
||||
cookies={"session-id": "session"},
|
||||
)
|
||||
|
||||
monkeypatch.setattr(fetch, "_sticky_proxy", sticky_proxy)
|
||||
monkeypatch.setattr(fetch, "fetch_page", fetch_page)
|
||||
|
||||
first = await get_location_session(
|
||||
"www.amazon.com", zip_code="10001", country_code="US"
|
||||
)
|
||||
second = await get_location_session(
|
||||
"www.amazon.com", zip_code="10001", country_code="US"
|
||||
)
|
||||
|
||||
assert first is second
|
||||
assert first is not None
|
||||
assert first.proxy == "http://sticky-proxy"
|
||||
assert first.country_code == "US"
|
||||
assert len(calls) == 2
|
||||
59
surfsense_backend/tests/unit/platforms/amazon/test_proxy.py
Normal file
59
surfsense_backend/tests/unit/platforms/amazon/test_proxy.py
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.config import Config
|
||||
from app.utils.proxy.providers.dataimpulse import DataImpulseProvider
|
||||
|
||||
|
||||
def test_dataimpulse_sticky_url_is_deterministic(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
Config,
|
||||
"PROXY_URL",
|
||||
"http://token__cr.us:secret@gw.dataimpulse.com:823",
|
||||
)
|
||||
provider = DataImpulseProvider()
|
||||
|
||||
first = provider.get_sticky_proxy_url("location-123")
|
||||
second = provider.get_sticky_proxy_url("location-123")
|
||||
|
||||
assert first == second
|
||||
assert "token__cr.us__sid.location-123" in first
|
||||
assert first.endswith("@gw.dataimpulse.com:823")
|
||||
|
||||
|
||||
def test_dataimpulse_sticky_url_replaces_existing_session(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
Config,
|
||||
"PROXY_URL",
|
||||
"http://token__cr.us__sid.old:secret@gw.dataimpulse.com:823",
|
||||
)
|
||||
|
||||
result = DataImpulseProvider().get_sticky_proxy_url("new")
|
||||
|
||||
assert "__sid.old" not in result
|
||||
assert result.count("__sid.new") == 1
|
||||
|
||||
|
||||
def test_dataimpulse_sticky_url_rewrites_country(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
Config,
|
||||
"PROXY_URL",
|
||||
"http://token__cr.us:secret@gw.dataimpulse.com:823",
|
||||
)
|
||||
|
||||
result = DataImpulseProvider().get_sticky_proxy_url("new", "gb")
|
||||
|
||||
assert "token__cr.gb__sid.new" in result
|
||||
assert "__cr.us" not in result
|
||||
|
||||
|
||||
def test_dataimpulse_sticky_url_rejects_empty_session(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
Config,
|
||||
"PROXY_URL",
|
||||
"http://token:secret@gw.dataimpulse.com:823",
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
DataImpulseProvider().get_sticky_proxy_url("...")
|
||||
177
surfsense_backend/tests/unit/platforms/amazon/test_skeleton.py
Normal file
177
surfsense_backend/tests/unit/platforms/amazon/test_skeleton.py
Normal file
|
|
@ -0,0 +1,177 @@
|
|||
"""Offline contract tests for the Amazon Product Scraper.
|
||||
|
||||
Deterministic (no network, no live Amazon HTML): asserts the input defaults, the
|
||||
full output-item serialization contract, the error-item shape, and URL
|
||||
classification. The live fetch / parse flows are exercised by the e2e script in
|
||||
later milestones, not here.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.proprietary.platforms.amazon import (
|
||||
AmazonScrapeInput,
|
||||
ErrorItem,
|
||||
ProductItem,
|
||||
scrape_products,
|
||||
)
|
||||
from app.proprietary.platforms.amazon.url_resolver import extract_asin, resolve_url
|
||||
|
||||
# A complete input payload should validate without removing optional fields.
|
||||
_SPEC_INPUT = {
|
||||
"categoryOrProductUrls": [{"url": "https://www.amazon.com/s?k=keyboard"}],
|
||||
"maxItemsPerStartUrl": 100,
|
||||
"language": "en",
|
||||
"proxyCountry": "AUTO_SELECT_PROXY_COUNTRY",
|
||||
"maxSearchPagesPerStartUrl": 9999,
|
||||
"maxOffers": 0,
|
||||
"scrapeSellers": False,
|
||||
"useCaptchaSolver": False,
|
||||
"scrapeProductVariantPrices": False,
|
||||
"scrapeProductDetails": True,
|
||||
"countryCode": "US",
|
||||
"zipCode": "10001",
|
||||
"locationDeliverableRoutes": ["PRODUCT", "SEARCH", "OFFERS"],
|
||||
}
|
||||
|
||||
|
||||
def test_input_has_no_auth_fields():
|
||||
# Public, anonymous only: no auth-shaped field may exist on the input surface.
|
||||
forbidden = {"username", "password", "token", "login", "auth", "credentials"}
|
||||
assert forbidden.isdisjoint(AmazonScrapeInput.model_fields)
|
||||
|
||||
|
||||
def test_scrape_input_defaults_match_contract():
|
||||
inp = AmazonScrapeInput(
|
||||
categoryOrProductUrls=[{"url": "https://www.amazon.com/dp/B0"}]
|
||||
)
|
||||
assert inp.maxItemsPerStartUrl is None
|
||||
assert inp.maxSearchPagesPerStartUrl == 9999
|
||||
assert inp.maxProductVariantsAsSeparateResults == 0
|
||||
assert inp.maxOffers == 0
|
||||
assert inp.proxyCountry == "AUTO_SELECT_PROXY_COUNTRY"
|
||||
assert inp.language is None
|
||||
assert inp.countryCode is None
|
||||
assert inp.zipCode is None
|
||||
assert inp.locationDeliverableRoutes == ["PRODUCT", "SEARCH", "OFFERS"]
|
||||
assert inp.scrapeSellers is False
|
||||
assert inp.useCaptchaSolver is False
|
||||
assert inp.scrapeProductVariantPrices is False
|
||||
assert inp.scrapeProductDetails is True
|
||||
|
||||
|
||||
def test_complete_payload_validates():
|
||||
inp = AmazonScrapeInput(**_SPEC_INPUT)
|
||||
assert inp.maxItemsPerStartUrl == 100
|
||||
assert inp.countryCode == "US"
|
||||
|
||||
|
||||
def test_input_allows_extra_inert_fields():
|
||||
# extra="allow": unknown add-ons (e.g. upstream proxy config) are accepted.
|
||||
inp = AmazonScrapeInput(
|
||||
categoryOrProductUrls=[{"url": "https://www.amazon.com/dp/B0"}],
|
||||
proxyConfiguration={"useResidentialProxy": True},
|
||||
someFutureAddon=123,
|
||||
)
|
||||
assert inp.model_dump().get("someFutureAddon") == 123
|
||||
|
||||
|
||||
def test_output_item_serializes_full_shape():
|
||||
item = ProductItem(asin="B08EXAMPLE01").to_output()
|
||||
assert item["asin"] == "B08EXAMPLE01"
|
||||
# Unsourced scalars are still present as None (consumers never KeyError).
|
||||
assert item["title"] is None
|
||||
assert item["monthlyPurchaseVolume"] is None
|
||||
# List fields default to [].
|
||||
assert item["features"] == []
|
||||
assert item["offers"] == []
|
||||
assert item["productPageReviews"] == []
|
||||
# Typed-but-unset nested objects are None.
|
||||
assert item["price"] is None
|
||||
assert item["seller"] is None
|
||||
|
||||
|
||||
def test_stars_breakdown_round_trips_digit_keys():
|
||||
item = ProductItem(
|
||||
stars=4.8, starsBreakdown={"5star": 0.86, "1star": 0.01}
|
||||
).to_output()
|
||||
# by_alias serialization restores Amazon's digit-prefixed keys.
|
||||
assert item["starsBreakdown"]["5star"] == 0.86
|
||||
assert item["starsBreakdown"]["1star"] == 0.01
|
||||
assert item["starsBreakdown"]["4star"] is None
|
||||
|
||||
|
||||
def test_error_item_shape():
|
||||
err = ErrorItem(
|
||||
error="product_not_found",
|
||||
errorDescription="Loaded a 404 page.",
|
||||
input="https://www.amazon.com/dp/B0XXXXXXXX",
|
||||
url="https://www.amazon.com/dp/B0XXXXXXXX",
|
||||
).model_dump()
|
||||
assert set(err) >= {"error", "errorDescription", "input", "url"}
|
||||
assert err["error"] == "product_not_found"
|
||||
|
||||
|
||||
def test_extract_asin():
|
||||
assert extract_asin("https://www.amazon.com/dp/B09X7MPX8L") == "B09X7MPX8L"
|
||||
assert extract_asin("https://www.amazon.com/gp/product/B09X7MPX8L/ref=x") == (
|
||||
"B09X7MPX8L"
|
||||
)
|
||||
assert extract_asin("https://www.amazon.com/s?k=keyboard") is None
|
||||
|
||||
|
||||
def test_resolve_url_classifies_all_kinds():
|
||||
product = resolve_url("https://www.amazon.com/dp/B0EXAMPLE1")
|
||||
assert product is not None
|
||||
assert product.kind == "product"
|
||||
assert product.asin == "B0EXAMPLE1"
|
||||
assert product.marketplace == "com"
|
||||
|
||||
search = resolve_url("https://www.amazon.com/s?k=keyboard")
|
||||
assert search is not None and search.kind == "search"
|
||||
|
||||
# A category URL with no /s path but a bbn/rh query still classifies as search.
|
||||
category = resolve_url(
|
||||
"https://www.amazon.de/s?i=specialty-aps&bbn=16225007011&rh=n%3A16225007011"
|
||||
)
|
||||
assert category is not None
|
||||
assert category.kind == "search"
|
||||
assert category.marketplace == "de"
|
||||
|
||||
bestsellers = resolve_url(
|
||||
"https://www.amazon.com/Best-Sellers-Electronics/zgbs/electronics"
|
||||
)
|
||||
assert bestsellers is not None and bestsellers.kind == "bestsellers"
|
||||
|
||||
shortened = resolve_url("https://a.co/d/abcd123")
|
||||
assert shortened is not None and shortened.kind == "shortened"
|
||||
|
||||
|
||||
def test_resolve_url_rejects_non_amazon_and_unrecognized():
|
||||
assert resolve_url("https://example.com/dp/B08EXAMPLE1") is None
|
||||
# An Amazon host but an unrecognized path (e.g. the homepage) is unrecognized.
|
||||
assert resolve_url("https://www.amazon.com/") is None
|
||||
|
||||
|
||||
async def test_iter_products_unrecognized_yields_error():
|
||||
# A junk / non-Amazon URL yields exactly one invalid_url error item.
|
||||
items = await scrape_products(
|
||||
AmazonScrapeInput(categoryOrProductUrls=[{"url": "https://example.com/foo"}])
|
||||
)
|
||||
assert len(items) == 1
|
||||
assert items[0]["error"] == "invalid_url"
|
||||
assert items[0]["input"] == "https://example.com/foo"
|
||||
|
||||
|
||||
def test_valid_product_url_is_ready_for_dispatch():
|
||||
resolved = resolve_url("https://www.amazon.com/dp/B0EXAMPLE1")
|
||||
assert resolved is not None
|
||||
assert resolved.kind == "product"
|
||||
|
||||
|
||||
async def test_iter_products_missing_url_key_yields_error():
|
||||
# An entry without a "url" key is treated as an invalid start URL, not a crash.
|
||||
items = await scrape_products(
|
||||
AmazonScrapeInput(categoryOrProductUrls=[{"noturl": "x"}])
|
||||
)
|
||||
assert len(items) == 1
|
||||
assert items[0]["error"] == "invalid_url"
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
"""Offline checks for the /sorry reCAPTCHA solver's pure helpers.
|
||||
|
||||
The network solve can't be unit-tested, but the parsing around it can: a missed
|
||||
sitekey/data-s silently burns paid solves, so these guard the boundary logic.
|
||||
(Proxy reformatting now lives in ``app.utils.captcha.solvers``.)
|
||||
"""
|
||||
|
||||
from app.proprietary.platforms.google_search import captcha
|
||||
|
||||
|
||||
class _Page:
|
||||
def __init__(self, url: str) -> None:
|
||||
self.url = url
|
||||
|
||||
|
||||
def test_on_sorry_detects_wall_only():
|
||||
assert captcha.on_sorry(_Page("https://www.google.com/sorry/index?continue=x"))
|
||||
assert not captcha.on_sorry(_Page("https://www.google.com/search?q=notebooklm"))
|
||||
assert not captcha.on_sorry(_Page(""))
|
||||
|
||||
|
||||
def test_sitekey_and_data_s_extraction():
|
||||
html = (
|
||||
'<div class="g-recaptcha" data-sitekey="6LdLLIMbAAAAAIl-KLj9p1ePhM"'
|
||||
' data-s="zBB1ixry9YzY_tok-en"></div>'
|
||||
)
|
||||
assert captcha._SITEKEY_RE.search(html).group(1) == "6LdLLIMbAAAAAIl-KLj9p1ePhM"
|
||||
assert captcha._DATA_S_RE.search(html).group(1) == "zBB1ixry9YzY_tok-en"
|
||||
|
||||
|
||||
def test_latch_roundtrip():
|
||||
captcha.reset_solver_latch()
|
||||
assert not captcha.solver_latched()
|
||||
captcha._latch("no balance")
|
||||
assert captcha.solver_latched()
|
||||
captcha.reset_solver_latch()
|
||||
assert not captcha.solver_latched()
|
||||
|
|
@ -16,7 +16,7 @@ class _FakeSession:
|
|||
self.release = asyncio.Event()
|
||||
self.closed = 0
|
||||
|
||||
async def fetch(self, url, proxy=None):
|
||||
async def fetch(self, url, proxy=None, **kwargs):
|
||||
await self.release.wait()
|
||||
return "page"
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,99 @@
|
|||
"""Warm sticky-IP pool bookkeeping (the scale-critical logic in fetch.py).
|
||||
|
||||
These pure, lock-guarded helpers decide whether a render reuses a warm IP,
|
||||
grows the pool (a paid solve), or waits — and how a finished render admits or
|
||||
evicts its IP. Get the pending/inflight accounting wrong and the pool either
|
||||
over-solves (cost) or funnels every render onto one IP (re-wall), so the
|
||||
transitions are worth pinning down offline.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from app.proprietary.platforms.google_search import fetch
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clean_pool():
|
||||
def reset():
|
||||
fetch._pool.clear()
|
||||
fetch._pool_inflight.clear()
|
||||
fetch._pool_pending = 0
|
||||
fetch._exemption_jar.clear()
|
||||
|
||||
reset()
|
||||
yield
|
||||
reset()
|
||||
|
||||
|
||||
def test_take_grows_when_pool_empty():
|
||||
action, proxy = fetch._pool_take()
|
||||
assert action == "grow" and proxy is None
|
||||
assert fetch._pool_pending == 1
|
||||
|
||||
|
||||
def test_take_reuses_warm_under_cap():
|
||||
fetch._pool["p1"] = 1.0
|
||||
action, proxy = fetch._pool_take()
|
||||
assert action == "reuse" and proxy == "p1"
|
||||
assert fetch._pool_inflight["p1"] == 1
|
||||
|
||||
|
||||
def test_take_spreads_across_least_loaded_ip():
|
||||
fetch._pool.update({"p1": 1.0, "p2": 1.0})
|
||||
fetch._pool_inflight["p1"] = 1 # p2 is idle
|
||||
action, proxy = fetch._pool_take()
|
||||
assert action == "reuse" and proxy == "p2"
|
||||
|
||||
|
||||
def test_take_waits_when_full_and_every_ip_capped(monkeypatch):
|
||||
monkeypatch.setattr(fetch, "_WARM_POOL_TARGET", 2)
|
||||
monkeypatch.setattr(fetch, "_WARM_IP_MAX_CONCURRENCY", 1)
|
||||
fetch._pool.update({"p1": 1.0, "p2": 1.0})
|
||||
fetch._pool_inflight.update({"p1": 1, "p2": 1})
|
||||
action, proxy = fetch._pool_take()
|
||||
assert action == "wait" and proxy is None
|
||||
assert fetch._pool_pending == 0 # a wait must NOT reserve a solve
|
||||
|
||||
|
||||
def test_take_grows_only_up_to_target_counting_pending(monkeypatch):
|
||||
monkeypatch.setattr(fetch, "_WARM_POOL_TARGET", 2)
|
||||
monkeypatch.setattr(fetch, "_WARM_IP_MAX_CONCURRENCY", 1)
|
||||
fetch._pool["p1"] = 1.0
|
||||
fetch._pool_inflight["p1"] = 1 # warm but capped
|
||||
assert fetch._pool_take()[0] == "grow" # pool(1)+pending(0) < 2
|
||||
assert fetch._pool_pending == 1
|
||||
assert fetch._pool_take()[0] == "wait" # pool(1)+pending(1) == 2 → no more solves
|
||||
|
||||
|
||||
def test_settle_good_admits_and_releases():
|
||||
fetch._pool_pending = 1
|
||||
fetch._pool_inflight["p1"] = 1
|
||||
fetch._pool_settle("p1", good=True, grew=True)
|
||||
assert "p1" in fetch._pool
|
||||
assert "p1" not in fetch._pool_inflight
|
||||
assert fetch._pool_pending == 0
|
||||
|
||||
|
||||
def test_settle_walled_evicts_ip_and_drops_exemption():
|
||||
fetch._pool["p1"] = 1.0
|
||||
fetch._pool_inflight["p1"] = 1
|
||||
fetch._exemption_jar["p1"] = [{"name": "GOOGLE_ABUSE_EXEMPTION"}]
|
||||
fetch._pool_settle("p1", good=False, grew=False)
|
||||
assert "p1" not in fetch._pool
|
||||
assert "p1" not in fetch._exemption_jar
|
||||
|
||||
|
||||
def test_adopt_releases_pending_and_pins_ip():
|
||||
fetch._pool_pending = 1 # reserved by a grow that we satisfy via the store
|
||||
fetch._pool_adopt("shared")
|
||||
assert fetch._pool_pending == 0
|
||||
assert "shared" in fetch._pool
|
||||
assert fetch._pool_inflight["shared"] == 1
|
||||
|
||||
|
||||
def test_abort_grow_releases_the_reserved_slot():
|
||||
fetch._pool_pending = 2
|
||||
fetch._pool_abort_grow()
|
||||
assert fetch._pool_pending == 1
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
"""Cross-process warm-IP exemption store (best-effort Redis cache).
|
||||
|
||||
Redis itself isn't unit-testable here, but the boundary is: adopt must skip IPs
|
||||
this process already holds (or the fleet re-uses nothing new), and every path
|
||||
must degrade to a silent no-op when Redis is down (or a hiccup would break the
|
||||
fetch instead of just costing a solve).
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from app.proprietary.platforms.google_search import pool_store as ps
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
class _FakeRedis:
|
||||
def __init__(self):
|
||||
self.store: dict[str, str] = {}
|
||||
|
||||
def ping(self):
|
||||
return True
|
||||
|
||||
def set(self, k, v, ex=None):
|
||||
self.store[k] = v
|
||||
|
||||
def get(self, k):
|
||||
return self.store.get(k)
|
||||
|
||||
def delete(self, k):
|
||||
self.store.pop(k, None)
|
||||
|
||||
def scan_iter(self, match=None, count=None):
|
||||
return list(self.store.keys())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake(monkeypatch):
|
||||
r = _FakeRedis()
|
||||
monkeypatch.setattr(ps, "_client", r)
|
||||
monkeypatch.setattr(ps, "_disabled", False)
|
||||
return r
|
||||
|
||||
|
||||
def test_key_is_stable_and_prefixed():
|
||||
assert ps._key("http://a").startswith(ps._PREFIX)
|
||||
assert ps._key("http://a") == ps._key("http://a")
|
||||
assert ps._key("http://a") != ps._key("http://b")
|
||||
|
||||
|
||||
def test_publish_then_adopt_excludes_held_ips(fake):
|
||||
ps._publish_sync("http://a", [{"n": 1}])
|
||||
ps._publish_sync("http://b", [{"n": 2}])
|
||||
proxy, cookies = ps._adopt_sync(exclude={"http://a"})
|
||||
assert proxy == "http://b" and cookies == [{"n": 2}]
|
||||
|
||||
|
||||
def test_adopt_none_when_all_held(fake):
|
||||
ps._publish_sync("http://a", [])
|
||||
assert ps._adopt_sync(exclude={"http://a"}) is None
|
||||
|
||||
|
||||
def test_evict_removes_entry(fake):
|
||||
ps._publish_sync("http://a", [{"n": 1}])
|
||||
ps._evict_sync("http://a")
|
||||
assert ps._adopt_sync(exclude=set()) is None
|
||||
|
||||
|
||||
def test_disabled_is_silent_noop(monkeypatch):
|
||||
monkeypatch.setattr(ps, "_disabled", True)
|
||||
monkeypatch.setattr(ps, "_client", None)
|
||||
assert ps._adopt_sync(set()) is None
|
||||
ps._publish_sync("x", [{"n": 1}]) # must not raise
|
||||
ps._evict_sync("x") # must not raise
|
||||
|
|
@ -1,9 +1,9 @@
|
|||
"""Offline resilience tests for the Reddit fetch seam and fan-out worker pool.
|
||||
|
||||
No network. Fake sessions drive the ``loid`` warm-up + rotate-on-block + backoff
|
||||
No network. Fake sessions drive the browser-warm + rotate-on-block + backoff
|
||||
paths deterministically (in live runs the first IP warms and returns 200s, so
|
||||
these branches rarely fire). Mirrors the youtube sibling's
|
||||
``test_fetch_resilience.py`` shape, extended with a fake warm-up.
|
||||
these branches rarely fire). The warm is now a browser mint (``holder.warm()``),
|
||||
so the fake holder fakes minting a cookie jar instead of HTTP GETs to a warm URL.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -22,9 +22,8 @@ _LISTING = {"kind": "Listing", "data": {"children": [], "after": None}}
|
|||
|
||||
|
||||
class _FakePage:
|
||||
def __init__(self, status: int, *, cookies: dict | None = None, payload=None):
|
||||
def __init__(self, status: int, *, payload=None):
|
||||
self.status = status
|
||||
self.cookies = cookies or {}
|
||||
self._payload = payload if payload is not None else _LISTING
|
||||
|
||||
def json(self):
|
||||
|
|
@ -36,48 +35,53 @@ class _FakePage:
|
|||
|
||||
|
||||
class _FakeSession:
|
||||
"""One 'IP': warm-up mints loid per flags; ``.json`` GETs return ``status``."""
|
||||
"""One 'IP': ``can_warm`` gates the browser mint; ``.json`` GETs return ``status``."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
status: int = 200,
|
||||
*,
|
||||
shreddit_loid: bool = True,
|
||||
old_loid: bool = False,
|
||||
can_warm: bool = True,
|
||||
payload=None,
|
||||
) -> None:
|
||||
self.status = status
|
||||
self.shreddit_loid = shreddit_loid
|
||||
self.old_loid = old_loid
|
||||
self.can_warm = can_warm
|
||||
self.payload = payload
|
||||
self.json_calls = 0
|
||||
self.warm_calls = 0
|
||||
|
||||
async def get(self, url, headers=None, cookies=None):
|
||||
if "svc/shreddit" in url:
|
||||
self.warm_calls += 1
|
||||
ck = {"loid": "x", "session_tracker": "y"} if self.shreddit_loid else {}
|
||||
return _FakePage(200, cookies=ck)
|
||||
if "old.reddit.com" in url:
|
||||
self.warm_calls += 1
|
||||
return _FakePage(200, cookies={"loid": "x"} if self.old_loid else {})
|
||||
self.json_calls += 1
|
||||
return _FakePage(self.status, payload=self.payload)
|
||||
|
||||
|
||||
class _FakeHolder:
|
||||
"""Holder whose ``rotate()`` advances to the next fake session (a new IP)."""
|
||||
"""Holder whose ``rotate()`` advances to the next fake session (a new IP).
|
||||
|
||||
``warm()`` fakes the browser mint: it succeeds (stashing a jar) only when the
|
||||
current fake session's ``can_warm`` is set, mirroring a clean vs dirty exit.
|
||||
"""
|
||||
|
||||
def __init__(self, sessions: list[_FakeSession]) -> None:
|
||||
self._sessions = sessions
|
||||
self.session = sessions[0]
|
||||
self.rotations = 0
|
||||
self.warmed = False
|
||||
self.proxy = "http://sticky.example:823"
|
||||
self.cookies: dict[str, str] = {}
|
||||
self.warm_calls = 0
|
||||
|
||||
async def warm(self) -> bool:
|
||||
self.warm_calls += 1
|
||||
if self.session.can_warm:
|
||||
self.cookies = {"loid": "x"}
|
||||
return True
|
||||
return False
|
||||
|
||||
async def rotate(self):
|
||||
self.rotations += 1
|
||||
self.session = self._sessions[min(self.rotations, len(self._sessions) - 1)]
|
||||
self.warmed = False # loid binds to the IP: re-warm on the fresh one
|
||||
self.cookies = {}
|
||||
return self.session
|
||||
|
||||
async def pace(self) -> None:
|
||||
|
|
@ -95,21 +99,8 @@ def _no_sleep(monkeypatch) -> None:
|
|||
|
||||
|
||||
async def test_warms_then_returns_json():
|
||||
# shreddit is tried first and mints loid -> a single warm call.
|
||||
holder = _FakeHolder([_FakeSession(200, shreddit_loid=True)])
|
||||
token = _current_session.set(holder)
|
||||
try:
|
||||
result = await fetch_json("r/python/hot")
|
||||
finally:
|
||||
_current_session.reset(token)
|
||||
assert result == _LISTING
|
||||
assert holder.rotations == 0
|
||||
assert holder.session.warm_calls == 1 # warmed exactly once
|
||||
|
||||
|
||||
async def test_warm_falls_back_to_old_reddit():
|
||||
# shreddit doesn't mint loid, old.reddit does -> still warms on the same IP.
|
||||
holder = _FakeHolder([_FakeSession(200, shreddit_loid=False, old_loid=True)])
|
||||
# The first IP warms (mints a jar) -> a single warm call, no rotation.
|
||||
holder = _FakeHolder([_FakeSession(200, can_warm=True)])
|
||||
token = _current_session.set(holder)
|
||||
try:
|
||||
result = await fetch_json("r/python/hot")
|
||||
|
|
@ -117,14 +108,16 @@ async def test_warm_falls_back_to_old_reddit():
|
|||
_current_session.reset(token)
|
||||
assert result == _LISTING
|
||||
assert holder.rotations == 0
|
||||
assert holder.warm_calls == 1 # warmed exactly once
|
||||
assert holder.cookies == {"loid": "x"} # jar stashed for the .json fetch
|
||||
|
||||
|
||||
async def test_rotates_when_warm_fails_then_succeeds():
|
||||
# IP0 can't mint loid at all -> rotate; IP1 warms fine.
|
||||
holder = _FakeHolder(
|
||||
[
|
||||
_FakeSession(200, shreddit_loid=False, old_loid=False),
|
||||
_FakeSession(200, shreddit_loid=True),
|
||||
_FakeSession(200, can_warm=False),
|
||||
_FakeSession(200, can_warm=True),
|
||||
]
|
||||
)
|
||||
token = _current_session.set(holder)
|
||||
|
|
@ -138,10 +131,7 @@ async def test_rotates_when_warm_fails_then_succeeds():
|
|||
|
||||
async def test_raises_when_no_ip_can_warm():
|
||||
holder = _FakeHolder(
|
||||
[
|
||||
_FakeSession(200, shreddit_loid=False, old_loid=False)
|
||||
for _ in range(fetch._MAX_ROTATIONS + 1)
|
||||
]
|
||||
[_FakeSession(200, can_warm=False) for _ in range(fetch._MAX_ROTATIONS + 1)]
|
||||
)
|
||||
token = _current_session.set(holder)
|
||||
try:
|
||||
|
|
@ -157,7 +147,7 @@ async def test_raises_when_no_ip_can_warm():
|
|||
|
||||
|
||||
async def test_rotates_and_rewarms_on_403():
|
||||
holder = _FakeHolder([_FakeSession(403), _FakeSession(200, old_loid=True)])
|
||||
holder = _FakeHolder([_FakeSession(403), _FakeSession(200, can_warm=True)])
|
||||
token = _current_session.set(holder)
|
||||
try:
|
||||
result = await fetch_json("r/python/hot")
|
||||
|
|
@ -165,7 +155,7 @@ async def test_rotates_and_rewarms_on_403():
|
|||
_current_session.reset(token)
|
||||
assert result == _LISTING
|
||||
assert holder.rotations == 1
|
||||
assert holder.session.warm_calls == 1 # re-warmed on the fresh IP
|
||||
assert holder.warm_calls == 2 # re-warmed on the fresh IP after the 403
|
||||
|
||||
|
||||
async def test_404_returns_none_without_rotating():
|
||||
|
|
@ -185,9 +175,6 @@ async def test_429_backs_off_without_rotating(monkeypatch):
|
|||
session = _FakeSession(429)
|
||||
|
||||
async def _get(url, headers=None, cookies=None):
|
||||
if "svc/shreddit" in url or "old.reddit.com" in url:
|
||||
session.warm_calls += 1
|
||||
return _FakePage(200, cookies={"loid": "x"})
|
||||
session.json_calls += 1
|
||||
return _FakePage(429 if session.json_calls == 1 else 200)
|
||||
|
||||
|
|
@ -266,3 +253,7 @@ async def test_fan_out_closes_all_sessions_on_early_stop(monkeypatch):
|
|||
async def test_fan_out_empty_jobs_is_noop():
|
||||
out = [x async for x in scraper.fan_out([])]
|
||||
assert out == []
|
||||
|
||||
|
||||
# Cross-country rotation lives in app.utils.proxy.rotation and is shared with the
|
||||
# TikTok sibling; its unit tests live in tests/unit/utils/proxy/test_rotation.py.
|
||||
|
|
|
|||
|
|
@ -169,6 +169,37 @@ async def test_listing_dedupes_then_caps_per_target():
|
|||
assert [i["id"] for i in items] == ["1", "2"]
|
||||
|
||||
|
||||
async def test_search_query_resolves_to_listing_target():
|
||||
# searchQueries must produce a search target (not be silently dropped): a
|
||||
# query with results flows through the listing parse/dedupe/cap path.
|
||||
async def fake_listing(url: str, _count: int) -> list[dict]:
|
||||
assert "search?q=meal%20prep" in url # query wired into the search URL
|
||||
return [{"id": "1", "author": {"uniqueId": "a"}}]
|
||||
|
||||
items = await scrape_tiktok(
|
||||
TikTokScrapeInput(searchQueries=["meal prep"], resultsPerPage=5),
|
||||
fetch=_no_html,
|
||||
fetch_listing=fake_listing,
|
||||
)
|
||||
assert [i["id"] for i in items] == ["1"]
|
||||
|
||||
|
||||
async def test_empty_search_query_degrades_to_error_item():
|
||||
# A withheld anonymous search feed must surface one honest ErrorItem tagged
|
||||
# with the query, never a silent empty.
|
||||
async def fake_listing(_url: str, _count: int) -> list[dict]:
|
||||
return []
|
||||
|
||||
items = await scrape_tiktok(
|
||||
TikTokScrapeInput(searchQueries=["meal prep"], resultsPerPage=5),
|
||||
fetch=_no_html,
|
||||
fetch_listing=fake_listing,
|
||||
)
|
||||
assert len(items) == 1
|
||||
assert items[0]["errorCode"] == "no_items"
|
||||
assert items[0]["input"] == "meal prep"
|
||||
|
||||
|
||||
async def test_empty_listing_emits_error_item():
|
||||
# A trust-gated/empty feed (0 videos) must surface one honest ErrorItem,
|
||||
# tagged with errorCode, rather than vanishing silently.
|
||||
|
|
|
|||
|
|
@ -2,8 +2,9 @@
|
|||
|
||||
The browser/solver boundary is mocked: a fake Playwright ``page`` and a
|
||||
monkeypatched ``_harvest_token`` / injection. We assert the glue logic —
|
||||
detection, proxy reformatting, attempt counting + state surfacing, the per-URL
|
||||
cap, and the no-balance process latch.
|
||||
detection, proxy pass-through, attempt counting + state surfacing, the per-URL
|
||||
cap, and the no-balance process latch. (Vendor proxy reformatting now lives in
|
||||
``app.utils.captcha.solvers`` and is tested there.)
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
|
@ -76,24 +77,6 @@ def _clear_latch():
|
|||
cap.reset_solver_latch()
|
||||
|
||||
|
||||
# --- proxy reformat --------------------------------------------------------
|
||||
|
||||
|
||||
class TestProxyReformat:
|
||||
def test_with_auth(self):
|
||||
assert (
|
||||
cap.proxy_url_to_captchatools("http://user:pass@1.2.3.4:8080")
|
||||
== "1.2.3.4:8080:user:pass"
|
||||
)
|
||||
|
||||
def test_without_auth(self):
|
||||
assert cap.proxy_url_to_captchatools("http://1.2.3.4:8080") == "1.2.3.4:8080"
|
||||
|
||||
def test_none_and_garbage(self):
|
||||
assert cap.proxy_url_to_captchatools(None) is None
|
||||
assert cap.proxy_url_to_captchatools("not-a-url") is None
|
||||
|
||||
|
||||
# --- detection -------------------------------------------------------------
|
||||
|
||||
|
||||
|
|
@ -175,8 +158,9 @@ class TestPageAction:
|
|||
action(page)
|
||||
|
||||
assert state == {"attempts": 1, "solved": True}
|
||||
# Solver egressed from the crawl's proxy, reformatted, with the page UA.
|
||||
assert captured["proxy"] == "1.2.3.4:9000:u:p"
|
||||
# Solver egressed from the crawl's proxy (raw URL; the seam reformats
|
||||
# per vendor), with the page UA.
|
||||
assert captured["proxy"] == "http://u:p@1.2.3.4:9000"
|
||||
assert captured["ua"] == "UA/1.0"
|
||||
assert captured["ctype"] == "v2"
|
||||
assert injected == {"ctype": "v2", "token": "TOKEN"}
|
||||
|
|
|
|||
326
surfsense_backend/tests/unit/routes/test_llm_setup_status.py
Normal file
326
surfsense_backend/tests/unit/routes/test_llm_setup_status.py
Normal file
|
|
@ -0,0 +1,326 @@
|
|||
"""Unit tests for the server-authoritative LLM onboarding verdict.
|
||||
|
||||
``compute_llm_setup_status`` is the single source of truth for whether a
|
||||
workspace can chat. These tests cover the two pieces of genuinely new logic:
|
||||
|
||||
1. ``_global_catalog_has_usable_chat`` — a pure check over the operator
|
||||
global catalog (usable model, not mere file presence).
|
||||
2. The decision tree in ``compute_llm_setup_status`` — exercised by faking
|
||||
the DB-touching seams (``_clear_invalid_roles`` heals dangling pins,
|
||||
``_workspace_has_enabled_chat_model`` reports BYOK models) so the routing
|
||||
between ready / needs_setup / global_config / models is asserted directly.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import ExitStack
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from app.auth.context import AuthContext
|
||||
from app.db import Permission
|
||||
from app.routes import model_connections_routes as mc
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeUser:
|
||||
id: str = "u1"
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeWorkspace:
|
||||
chat_model_id: int | None = 0
|
||||
vision_model_id: int | None = 0
|
||||
image_gen_model_id: int | None = 0
|
||||
llm_setup_completed_at: datetime | None = None
|
||||
|
||||
|
||||
def _global_model(
|
||||
*,
|
||||
model_id: int = -1,
|
||||
connection_id: int = -1,
|
||||
enabled: bool = True,
|
||||
supports_chat: bool = True,
|
||||
) -> dict:
|
||||
return {
|
||||
"id": model_id,
|
||||
"connection_id": connection_id,
|
||||
"enabled": enabled,
|
||||
"supports_chat": supports_chat,
|
||||
"capabilities_override": {},
|
||||
}
|
||||
|
||||
|
||||
class TestGlobalCatalogHasUsableChat:
|
||||
"""Usability, not file existence, is what counts."""
|
||||
|
||||
def test_usable_when_enabled_connection_and_chat_model(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": True}]
|
||||
)
|
||||
monkeypatch.setattr(mc.config, "GLOBAL_MODELS", [_global_model()])
|
||||
assert mc._global_catalog_has_usable_chat() is True
|
||||
|
||||
def test_empty_catalog_is_not_usable(self, monkeypatch):
|
||||
monkeypatch.setattr(mc.config, "GLOBAL_CONNECTIONS", [])
|
||||
monkeypatch.setattr(mc.config, "GLOBAL_MODELS", [])
|
||||
assert mc._global_catalog_has_usable_chat() is False
|
||||
|
||||
def test_disabled_connection_is_not_usable(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": False}]
|
||||
)
|
||||
monkeypatch.setattr(mc.config, "GLOBAL_MODELS", [_global_model()])
|
||||
assert mc._global_catalog_has_usable_chat() is False
|
||||
|
||||
def test_disabled_model_is_not_usable(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": True}]
|
||||
)
|
||||
monkeypatch.setattr(mc.config, "GLOBAL_MODELS", [_global_model(enabled=False)])
|
||||
assert mc._global_catalog_has_usable_chat() is False
|
||||
|
||||
def test_non_chat_model_is_not_usable(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": True}]
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
mc.config, "GLOBAL_MODELS", [_global_model(supports_chat=False)]
|
||||
)
|
||||
assert mc._global_catalog_has_usable_chat() is False
|
||||
|
||||
|
||||
async def _run_status(
|
||||
*,
|
||||
file_exists: bool,
|
||||
global_usable: bool,
|
||||
chat_model_id: int,
|
||||
ws_has_chat: bool = False,
|
||||
permissions: list[str] | None = None,
|
||||
workspace: _FakeWorkspace | None = None,
|
||||
):
|
||||
"""Drive the decision tree with DB-touching seams stubbed out.
|
||||
|
||||
Pass ``workspace`` to preset/inspect ``llm_setup_completed_at`` — it is the
|
||||
object ``_clear_invalid_roles`` returns, and the lazy stamp mutates it in
|
||||
place (``session.commit`` is a no-op AsyncMock).
|
||||
"""
|
||||
if permissions is None:
|
||||
permissions = [Permission.FULL_ACCESS.value]
|
||||
if workspace is None:
|
||||
workspace = _FakeWorkspace(chat_model_id=chat_model_id)
|
||||
else:
|
||||
workspace.chat_model_id = chat_model_id
|
||||
with ExitStack() as stack:
|
||||
stack.enter_context(
|
||||
patch.object(mc.config, "GLOBAL_LLM_CONFIG_FILE_EXISTS", file_exists)
|
||||
)
|
||||
stack.enter_context(
|
||||
patch.object(
|
||||
mc, "_global_catalog_has_usable_chat", return_value=global_usable
|
||||
)
|
||||
)
|
||||
stack.enter_context(patch.object(mc, "check_permission", AsyncMock()))
|
||||
stack.enter_context(
|
||||
patch.object(
|
||||
mc, "get_user_permissions", AsyncMock(return_value=permissions)
|
||||
)
|
||||
)
|
||||
stack.enter_context(
|
||||
patch.object(
|
||||
mc,
|
||||
"_clear_invalid_roles",
|
||||
AsyncMock(return_value=workspace),
|
||||
)
|
||||
)
|
||||
stack.enter_context(
|
||||
patch.object(
|
||||
mc,
|
||||
"_workspace_has_enabled_chat_model",
|
||||
AsyncMock(return_value=ws_has_chat),
|
||||
)
|
||||
)
|
||||
return await mc.compute_llm_setup_status(
|
||||
AsyncMock(), AuthContext.session(_FakeUser()), 1
|
||||
)
|
||||
|
||||
|
||||
class TestComputeLlmSetupStatus:
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_yaml_no_models_needs_setup(self):
|
||||
result = await _run_status(
|
||||
file_exists=False, global_usable=False, chat_model_id=0, ws_has_chat=False
|
||||
)
|
||||
assert result.status == "needs_setup"
|
||||
assert result.source == "none"
|
||||
assert result.stage == "initial_setup"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_usable_global_catalog_is_ready(self):
|
||||
ws = _FakeWorkspace()
|
||||
result = await _run_status(
|
||||
file_exists=True, global_usable=True, chat_model_id=0, workspace=ws
|
||||
)
|
||||
assert result.status == "ready"
|
||||
assert result.source == "global_config"
|
||||
assert result.stage == "ready"
|
||||
# Global readiness is never stamped as this workspace's own setup.
|
||||
assert ws.llm_setup_completed_at is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_yaml_present_but_empty_catalog_falls_through(self):
|
||||
# File exists but no usable model AND no BYOK => onboarding, not a
|
||||
# dead composer. This is the empty/broken-YAML regression.
|
||||
result = await _run_status(
|
||||
file_exists=True, global_usable=False, chat_model_id=0, ws_has_chat=False
|
||||
)
|
||||
assert result.status == "needs_setup"
|
||||
assert result.source == "none"
|
||||
assert result.stage == "initial_setup"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_auto_mode_with_workspace_model_is_ready(self):
|
||||
ws = _FakeWorkspace()
|
||||
result = await _run_status(
|
||||
file_exists=False,
|
||||
global_usable=False,
|
||||
chat_model_id=0,
|
||||
ws_has_chat=True,
|
||||
workspace=ws,
|
||||
)
|
||||
assert result.status == "ready"
|
||||
assert result.source == "models"
|
||||
assert result.stage == "ready"
|
||||
assert ws.llm_setup_completed_at is not None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_auto_mode_counts_global_catalog_without_file(self):
|
||||
result = await _run_status(
|
||||
file_exists=False, global_usable=True, chat_model_id=0, ws_has_chat=False
|
||||
)
|
||||
assert result.status == "ready"
|
||||
assert result.source == "global_config"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pinned_workspace_model_is_ready(self):
|
||||
# chat_model_id > 0 survived _clear_invalid_roles => valid + enabled.
|
||||
ws = _FakeWorkspace()
|
||||
result = await _run_status(
|
||||
file_exists=False, global_usable=False, chat_model_id=5, workspace=ws
|
||||
)
|
||||
assert result.status == "ready"
|
||||
assert result.source == "models"
|
||||
assert result.stage == "ready"
|
||||
assert ws.llm_setup_completed_at is not None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pinned_global_model_is_ready(self):
|
||||
ws = _FakeWorkspace()
|
||||
result = await _run_status(
|
||||
file_exists=False, global_usable=False, chat_model_id=-3, workspace=ws
|
||||
)
|
||||
assert result.status == "ready"
|
||||
assert result.source == "global_config"
|
||||
assert result.stage == "ready"
|
||||
# A negative pin is global-config readiness, not own setup.
|
||||
assert ws.llm_setup_completed_at is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pinned_dead_model_healed_to_needs_setup(self):
|
||||
# A pin to a deleted/disabled model collapses to 0 in
|
||||
# _clear_invalid_roles; with no fallback model it is needs_setup.
|
||||
result = await _run_status(
|
||||
file_exists=False, global_usable=False, chat_model_id=0, ws_has_chat=False
|
||||
)
|
||||
assert result.status == "needs_setup"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_can_configure_owner(self):
|
||||
result = await _run_status(
|
||||
file_exists=True,
|
||||
global_usable=True,
|
||||
chat_model_id=0,
|
||||
permissions=[Permission.FULL_ACCESS.value],
|
||||
)
|
||||
assert result.can_configure is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_can_configure_editor(self):
|
||||
result = await _run_status(
|
||||
file_exists=True,
|
||||
global_usable=True,
|
||||
chat_model_id=0,
|
||||
permissions=[
|
||||
Permission.LLM_CONFIGS_CREATE.value,
|
||||
Permission.LLM_CONFIGS_READ.value,
|
||||
],
|
||||
)
|
||||
assert result.can_configure is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_can_configure_viewer_is_false(self):
|
||||
result = await _run_status(
|
||||
file_exists=True,
|
||||
global_usable=True,
|
||||
chat_model_id=0,
|
||||
permissions=[Permission.LLM_CONFIGS_READ.value],
|
||||
)
|
||||
assert result.can_configure is False
|
||||
|
||||
|
||||
class TestOnboardingStage:
|
||||
"""First-run vs. recovery: the durable timestamp splits needs_setup."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fresh_workspace_is_initial_setup(self):
|
||||
result = await _run_status(
|
||||
file_exists=False, global_usable=False, chat_model_id=0, ws_has_chat=False
|
||||
)
|
||||
assert result.stage == "initial_setup"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_previously_configured_then_lost_is_recovery(self):
|
||||
# Configured before (timestamp set), then deleted: needs_setup but recovery.
|
||||
ws = _FakeWorkspace(llm_setup_completed_at=datetime.now(UTC))
|
||||
result = await _run_status(
|
||||
file_exists=False,
|
||||
global_usable=False,
|
||||
chat_model_id=0,
|
||||
ws_has_chat=False,
|
||||
workspace=ws,
|
||||
)
|
||||
assert result.status == "needs_setup"
|
||||
assert result.stage == "recovery"
|
||||
assert ws.llm_setup_completed_at is not None # preserved, never cleared
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stamp_is_not_overwritten_on_subsequent_ready(self):
|
||||
original = datetime(2020, 1, 1, tzinfo=UTC)
|
||||
ws = _FakeWorkspace(llm_setup_completed_at=original)
|
||||
result = await _run_status(
|
||||
file_exists=False,
|
||||
global_usable=False,
|
||||
chat_model_id=0,
|
||||
ws_has_chat=True,
|
||||
workspace=ws,
|
||||
)
|
||||
assert result.stage == "ready"
|
||||
assert ws.llm_setup_completed_at == original
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_global_only_loss_is_initial_setup_not_recovery(self):
|
||||
# Rode global (never stamped), then global lost: a genuine first own-setup.
|
||||
ws = _FakeWorkspace()
|
||||
result = await _run_status(
|
||||
file_exists=False,
|
||||
global_usable=False,
|
||||
chat_model_id=0,
|
||||
ws_has_chat=False,
|
||||
workspace=ws,
|
||||
)
|
||||
assert result.status == "needs_setup"
|
||||
assert result.stage == "initial_setup"
|
||||
assert ws.llm_setup_completed_at is None
|
||||
|
|
@ -59,7 +59,7 @@ def _due_connector(connector_type: SearchSourceConnectorType) -> SimpleNamespace
|
|||
return SimpleNamespace(
|
||||
id=42,
|
||||
connector_type=connector_type,
|
||||
search_space_id=7,
|
||||
workspace_id=7,
|
||||
user_id="00000000-0000-0000-0000-000000000001",
|
||||
config={},
|
||||
periodic_indexing_enabled=True,
|
||||
|
|
@ -86,13 +86,8 @@ async def _run_checker(monkeypatch: pytest.MonkeyPatch, connector: SimpleNamespa
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_due_bookstack_connector_dispatches_indexing_task(monkeypatch):
|
||||
"""A due BookStack connector must dispatch index_bookstack_pages_task.
|
||||
|
||||
Regression test for the connector type missing from the scheduler's
|
||||
task_map, which made periodic BookStack syncs silently no-op with only a
|
||||
"No task found" warning.
|
||||
"""
|
||||
async def test_due_bookstack_connector_disables_deprecated_indexing(monkeypatch):
|
||||
"""A due BookStack connector is retired from periodic indexing, not dispatched."""
|
||||
from app.tasks.celery_tasks import connector_tasks
|
||||
|
||||
task_mock = MagicMock()
|
||||
|
|
@ -101,16 +96,9 @@ async def test_due_bookstack_connector_dispatches_indexing_task(monkeypatch):
|
|||
connector = _due_connector(SearchSourceConnectorType.BOOKSTACK_CONNECTOR)
|
||||
session = await _run_checker(monkeypatch, connector)
|
||||
|
||||
task_mock.delay.assert_called_once_with(
|
||||
connector.id,
|
||||
connector.search_space_id,
|
||||
str(connector.user_id),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
# The next run must be rescheduled, otherwise the connector stays "due"
|
||||
# and is re-examined every minute.
|
||||
assert connector.next_scheduled_at > datetime.now(UTC)
|
||||
task_mock.delay.assert_not_called()
|
||||
assert connector.periodic_indexing_enabled is False
|
||||
assert connector.next_scheduled_at is None
|
||||
assert session.commits == 1
|
||||
|
||||
|
||||
|
|
|
|||
90
surfsense_backend/tests/unit/utils/captcha/test_solvers.py
Normal file
90
surfsense_backend/tests/unit/utils/captcha/test_solvers.py
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
"""Offline checks for the in-house captcha-solver seam.
|
||||
|
||||
The network solve can't be unit-tested, but the boundary logic can: a wrong
|
||||
proxy reformat (2captcha's ``ERROR_PROXY_FORMAT``) or dispatching to a provider
|
||||
we haven't wired silently burns paid solves / leaks the key to the wrong vendor.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from app.utils.captcha import solvers
|
||||
from app.utils.captcha.config import CaptchaConfig
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def _cfg(**overrides) -> CaptchaConfig:
|
||||
base = {
|
||||
"enabled": True,
|
||||
"solving_site": "2captcha",
|
||||
"api_key": "key-123",
|
||||
"max_attempts_per_url": 1,
|
||||
"timeout_s": 30,
|
||||
"captcha_type_default": "v2",
|
||||
"v3_min_score": 0.7,
|
||||
"v3_action": "verify",
|
||||
}
|
||||
base.update(overrides)
|
||||
return CaptchaConfig(**base)
|
||||
|
||||
|
||||
# --- proxy reformat (2captcha wants login:pass@host:port, NO scheme) --------
|
||||
|
||||
|
||||
def test_proxy_login_form_strips_scheme():
|
||||
got = solvers.proxy_login_form("http://user:pass@gw.dataimpulse.com:15673")
|
||||
assert got == "user:pass@gw.dataimpulse.com:15673"
|
||||
|
||||
|
||||
def test_proxy_login_form_without_credentials():
|
||||
assert (
|
||||
solvers.proxy_login_form("http://gw.dataimpulse.com:823")
|
||||
== "gw.dataimpulse.com:823"
|
||||
)
|
||||
|
||||
|
||||
def test_proxy_login_form_none_on_missing_or_bad():
|
||||
assert solvers.proxy_login_form(None) is None
|
||||
assert solvers.proxy_login_form("not a url") is None
|
||||
|
||||
|
||||
# --- capsolver proxy reformat (colon-delimited scheme:host:port:user:pass) --
|
||||
|
||||
|
||||
def test_capsolver_proxy_colon_delimited_with_creds():
|
||||
got = solvers.capsolver_proxy("http://user:pass@gw.dataimpulse.com:823")
|
||||
assert got == "http:gw.dataimpulse.com:823:user:pass"
|
||||
|
||||
|
||||
def test_capsolver_proxy_without_credentials():
|
||||
assert (
|
||||
solvers.capsolver_proxy("http://gw.dataimpulse.com:823")
|
||||
== "http:gw.dataimpulse.com:823"
|
||||
)
|
||||
|
||||
|
||||
def test_capsolver_proxy_none_on_missing_or_bad():
|
||||
assert solvers.capsolver_proxy(None) is None
|
||||
assert solvers.capsolver_proxy("not a url") is None
|
||||
|
||||
|
||||
# --- dispatch --------------------------------------------------------------
|
||||
|
||||
|
||||
def test_unsupported_provider_raises_solvererror():
|
||||
# An unconfigured provider must fail loudly (latch) rather than POST the key
|
||||
# to a wired vendor's endpoint under the wrong account.
|
||||
with pytest.raises(solvers.SolverUnsupportedError):
|
||||
solvers.solve(
|
||||
_cfg(solving_site="anticaptcha"),
|
||||
challenge_type="v2",
|
||||
sitekey="SK",
|
||||
page_url="https://t.test",
|
||||
)
|
||||
assert issubclass(solvers.SolverUnsupportedError, solvers.SolverError)
|
||||
|
||||
|
||||
def test_wired_providers_are_registered():
|
||||
supported = solvers.supported_providers()
|
||||
assert "2captcha" in supported
|
||||
assert "capsolver" in supported
|
||||
|
|
@ -44,6 +44,51 @@ def test_location_stops_at_next_param(monkeypatch: pytest.MonkeyPatch) -> None:
|
|||
assert DataImpulseProvider().get_location() == "de"
|
||||
|
||||
|
||||
def test_geo_proxy_url_rewrites_country_suffix(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(Config, "PROXY_URL", _URL)
|
||||
|
||||
result = DataImpulseProvider().get_geo_proxy_url("gb")
|
||||
|
||||
assert result == "http://tok123__cr.gb:secret@gw.dataimpulse.com:823"
|
||||
|
||||
|
||||
def test_geo_proxy_url_replaces_existing_country_once(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
Config,
|
||||
"PROXY_URL",
|
||||
"http://tok123__cr.us__sid.old:secret@gw.dataimpulse.com:823",
|
||||
)
|
||||
|
||||
result = DataImpulseProvider().get_geo_proxy_url("de")
|
||||
|
||||
assert "__cr.us" not in result
|
||||
assert result.count("__cr.de") == 1
|
||||
assert "__sid.old" not in result
|
||||
|
||||
|
||||
def test_geo_proxy_url_without_country_keeps_base_url(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(Config, "PROXY_URL", _URL)
|
||||
|
||||
assert DataImpulseProvider().get_geo_proxy_url(None) == _URL
|
||||
assert DataImpulseProvider().get_geo_proxy_url("") == _URL
|
||||
|
||||
|
||||
def test_sticky_proxy_url_composes_country_and_session(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(Config, "PROXY_URL", _URL)
|
||||
|
||||
result = DataImpulseProvider().get_sticky_proxy_url("location-123", "gb")
|
||||
|
||||
assert result == (
|
||||
"http://tok123__cr.gb__sid.location-123:secret@gw.dataimpulse.com:823"
|
||||
)
|
||||
|
||||
|
||||
def test_no_country_suffix_yields_empty_location(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
|
|
|
|||
53
surfsense_backend/tests/unit/utils/proxy/test_rotation.py
Normal file
53
surfsense_backend/tests/unit/utils/proxy/test_rotation.py
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
"""Unit tests for the shared cross-country exit rotation helper.
|
||||
|
||||
Offline: the provider is faked via monkeypatch, so no network/proxy is touched.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.utils.proxy import rotation
|
||||
|
||||
|
||||
class _Prov:
|
||||
def __init__(self, location: str) -> None:
|
||||
self._location = location
|
||||
|
||||
def get_location(self) -> str:
|
||||
return self._location
|
||||
|
||||
|
||||
def test_lead_country_leads_and_dedupes(monkeypatch):
|
||||
# The configured/default exit country leads and isn't duplicated in the walk.
|
||||
monkeypatch.setattr(rotation, "get_active_provider", lambda: _Prov("gb"))
|
||||
countries = rotation.rotation_countries()
|
||||
|
||||
assert countries[0] == "gb"
|
||||
assert len(countries) == len(set(countries)) # de-duplicated
|
||||
assert set(rotation.FALLBACK_COUNTRIES) <= set(countries) # fallbacks kept
|
||||
|
||||
|
||||
def test_no_configured_country_uses_fallbacks_only(monkeypatch):
|
||||
# A bare PROXY_URL (no country) leaves just the fallback pools, in order.
|
||||
monkeypatch.setattr(rotation, "get_active_provider", lambda: _Prov(""))
|
||||
assert rotation.rotation_countries() == rotation.FALLBACK_COUNTRIES
|
||||
|
||||
|
||||
def test_walk_covers_every_country_and_wraps(monkeypatch):
|
||||
# A whole-pool block can't stall the walk: every country is reached, and the
|
||||
# index cycles (wraps) rather than running off the end.
|
||||
monkeypatch.setattr(rotation, "get_active_provider", lambda: _Prov("us"))
|
||||
countries = rotation.rotation_countries()
|
||||
tried = {rotation.country_for_rotation(n) for n in range(len(countries))}
|
||||
|
||||
assert tried == set(countries)
|
||||
assert rotation.country_for_rotation(len(countries)) == countries[0] # wraps
|
||||
|
||||
|
||||
def test_caller_budgets_cover_every_country():
|
||||
# Each warm-on-block caller must budget enough rotations to try every pool at
|
||||
# least once, else a wholly-blocked lead pool could fail a job prematurely.
|
||||
from app.proprietary.platforms.reddit import fetch as reddit_fetch
|
||||
from app.proprietary.platforms.tiktok.session import client as tiktok_client
|
||||
|
||||
assert len(rotation.FALLBACK_COUNTRIES) <= reddit_fetch._MAX_ROTATIONS
|
||||
assert len(rotation.FALLBACK_COUNTRIES) <= tiktok_client._MAX_ROTATIONS
|
||||
Loading…
Add table
Add a link
Reference in a new issue