Merge remote-tracking branch 'upstream/dev' into feature-indeed-jobs-scraper

# Conflicts:
#	README.es.md
#	README.hi.md
#	README.md
#	README.pt-BR.md
#	README.zh-CN.md
#	surfsense_backend/tests/unit/capabilities/google_maps/test_registry.py
#	surfsense_backend/tests/unit/capabilities/reddit/test_registry.py
#	surfsense_backend/tests/unit/capabilities/youtube/test_registry.py
#	surfsense_mcp/mcp_server/features/scrapers/__init__.py
#	surfsense_web/content/docs/connectors/index.mdx
#	surfsense_web/content/docs/connectors/native/index.mdx
#	surfsense_web/content/docs/connectors/native/meta.json
#	surfsense_web/content/docs/how-to/mcp-server.mdx
#	surfsense_web/lib/connectors-marketing/index.ts
#	surfsense_web/lib/playground/catalog.ts

surfsense_backend/.env.example

@@ -277,8 +277,8 @@ MICROS_PER_PAGE=1000
 # WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE=3000
 
 # Debit the credit wallet per *item returned* by the platform-native scrapers
-# (Reddit, Google Search, Google Maps, YouTube). Default FALSE keeps scraping
-# effectively free for self-hosted/OSS installs; hosted deployments set TRUE.
+# (Reddit, Google Search, Google Maps, Amazon, YouTube). Default FALSE keeps
+# scraping effectively free for self-hosted/OSS installs; hosted set TRUE.
 # Each rate is micro-USD per item, fully config-driven (no hardcoded rate):
 #   <KEY> = round(USD_per_1000_items * 1_000)
 #   3500 == $3.50/1000 | 5000 == $5/1000 | 2000 == $2/1000
@@ -290,6 +290,7 @@ MICROS_PER_PAGE=1000
 # GOOGLE_SEARCH_MICROS_PER_SERP=5500
 # GOOGLE_MAPS_MICROS_PER_PLACE=3500
 # GOOGLE_MAPS_MICROS_PER_REVIEW=1500
+# AMAZON_MICROS_PER_PRODUCT=3500
 # YOUTUBE_MICROS_PER_VIDEO=2500
 # YOUTUBE_MICROS_PER_COMMENT=1500
 # INSTAGRAM_SCRAPE_MICROS_PER_ITEM=3500
@@ -373,16 +374,47 @@ TURNSTILE_SECRET_KEY=
 #   (cyclic); server-side-rotating gateways ignore it.
 # PROXY_URL=http://user:pass@host:port
 # PROXY_URLS=http://user:pass@host1:port,http://user:pass@host2:port
+#
+# NOTE (dataimpulse): its default *worldwide* pool (a PROXY_URL with no
+# "__cr.<country>" suffix) hands out exit IPs some targets hard-block — Reddit
+# 403s every one, TikTok withholds its ttwid cookie — so those scrapers
+# pin/rotate exit countries on their own (no config needed). Other scrapers
+# (e.g. Google SERP) deliberately prefer the worldwide pool, so leave PROXY_URL
+# unpinned unless you have a reason; pin a country only for your whole install by
+# adding a "__cr.<country>" suffix above.
+
+# --- Google Search scraper: scale / warm sticky-IP pool -----------------------
+# Per-process render concurrency AND the throughput lever: ceiling =
+# MAX_CONCURRENT_PAGES / warm-render-secs (~4/14s ≈ 17 SERP/min). Renders are
+# resource-light (disable_resources), so one Chromium holds many text contexts —
+# raise this to trade RAM/CPU for throughput before adding processes
+# (16 → ~69/min/process ⇒ ~7 processes cover 500/min). Validate with
+# scripts/scale_google_search.py.
+# GOOGLE_SEARCH_MAX_CONCURRENT_PAGES=4
+# Warm sticky-IP pool: concurrent renders spread across this many solved IPs so
+# no single IP is hammered (Google re-walls a hot IP). Steady-state paid solves
+# ≈ pool size, not request count. Size for the fleet: fleet per-IP load =
+# IP_MAX_CONCURRENCY × processes must stay gentle. Solved IPs are shared across
+# workers via Redis (REDIS_APP_URL) so the fleet solves each IP once.
+# GOOGLE_SEARCH_WARM_POOL_TARGET=8
+# GOOGLE_SEARCH_IP_MAX_CONCURRENCY=2
+# How long a shared exemption stays reusable before a re-solve (Google's cookie
+# outlives this; conservative TTL just re-solves idle IPs occasionally).
+# GOOGLE_SEARCH_EXEMPTION_TTL_S=3600
+# Per-fetch budget before giving up under saturation (a cold solve is ~45s).
+# GOOGLE_SEARCH_FETCH_DEADLINE_S=180
 
 # =====================================================================
-# Captcha solving (Phase 3d) — LAST-resort bypass tier via captchatools.
-# Only fires on the stealth browser tier when a sitekey is detected AND
-# CAPTCHA_SOLVING_ENABLED=TRUE. Cloudflare Turnstile is already solved free
-# in-framework (no config needed). Off by default => zero attempts, zero cost.
+# Captcha solving (Phase 3d) — LAST-resort bypass tier (in-house solver seam,
+# app/utils/captcha/solvers.py). Only fires on the stealth browser tier when a
+# sitekey is detected AND CAPTCHA_SOLVING_ENABLED=TRUE. Also unlocks Google's
+# reCAPTCHA-Enterprise /sorry wall for the SERP scraper. Cloudflare Turnstile is
+# already solved free in-framework. Off by default => zero attempts, zero cost.
 # NOTE: automated solving may violate a target site's ToS — opt-in, public
-# data only (no logged-in bypass). captchatools is itself the vendor registry.
+# data only (no logged-in bypass).
 # CAPTCHA_SOLVING_ENABLED=FALSE
-# solving_site: capmonster | 2captcha | anticaptcha | capsolver | captchaai
+# Provider: "capsolver" (AI-native, fastest on reCAPTCHA-Enterprise) or
+# "2captcha" have in-house clients today (more added over time).
 # CAPTCHA_SOLVER_PROVIDER=capsolver
 # CAPTCHA_SOLVER_API_KEY=
 # Per-URL solve cap (bounds solver spend on a hostile page).

surfsense_backend/alembic/versions/174_add_llm_setup_completed_at.py

@@ -0,0 +1,30 @@
+"""Add workspaces.llm_setup_completed_at for first-run vs. recovery onboarding.
+
+No backfill: NULL is correct for every existing row. Configured workspaces
+self-heal via lazy stamping on their next status read (which fires while still
+``ready``, before they could reach ``needs_setup``); a blanket
+``SET ... = created_at`` would misclassify abandoned and global-only workspaces.
+
+Revision ID: 174
+Revises: 173
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "174"
+down_revision: str | None = "173"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.execute(
+        "ALTER TABLE workspaces "
+        "ADD COLUMN IF NOT EXISTS llm_setup_completed_at TIMESTAMPTZ"
+    )
+
+
+def downgrade() -> None:
+    op.execute("ALTER TABLE workspaces DROP COLUMN IF EXISTS llm_setup_completed_at")

surfsense_backend/app/agents/chat/anonymous_chat/agent.py

@@ -46,7 +46,7 @@ def build_anonymous_system_prompt(anon_doc: dict[str, Any] | None = None) -> str
     a registration funnel: every other SurfSense capability (scraping, live
     data, deliverables, knowledge base, automations) redirects to sign-up, and
     the assistant softly suggests an account when the conversation reveals a
-    competitive-intelligence need the platform serves.
+    live-web-research need the platform serves.
     """
     today = datetime.now(UTC).strftime("%A, %B %d, %Y")
 
@@ -73,9 +73,10 @@ def build_anonymous_system_prompt(anon_doc: dict[str, Any] | None = None) -> str
 
     return (
         "You are SurfSense's free AI assistant, available to everyone without "
-        "login. SurfSense is the open-source competitive intelligence platform: "
-        "registered users get specialist agents that pull live market data from "
-        "Reddit, YouTube, Google Maps, Google Search, and the open web, turn it "
+        "login. SurfSense is the open-source open web research platform: "
+        "registered users get specialist agents that pull live web data from "
+        "Reddit, YouTube, Amazon, Google Maps, Google Search, and the open "
+        "web, turn it "
         "into cited briefs, reports, podcasts, and presentations, keep findings "
         "in a searchable knowledge base, and run scheduled monitoring "
         "automations — plus a REST scraping API and MCP server for their own "
@@ -93,7 +94,7 @@ def build_anonymous_system_prompt(anon_doc: dict[str, Any] | None = None) -> str
         f"{doc_section}\n\n"
         "## What is not available here\n"
         "This is the free, no-login experience. You CANNOT search the web or "
-        "scrape any platform (Reddit, YouTube, Google Maps, Google Search, "
+        "scrape any platform (Reddit, YouTube, Amazon, Google Maps, Google Search, "
         "websites), save files or notes, upload additional files, generate "
         "reports, podcasts, resumes, presentations, or images, search or build "
         "a knowledge base, connect to apps (Gmail, Google Drive, Notion, "
@@ -110,10 +111,11 @@ def build_anonymous_system_prompt(anon_doc: dict[str, Any] | None = None) -> str
         "your own knowledge and about any uploaded document).\n\n"
         "## Suggesting SurfSense when it genuinely fits\n"
         "You are also the front door to SurfSense. When the conversation "
-        "reveals a need the full platform serves — researching competitors, "
-        "tracking pricing or rankings, monitoring brand mentions or reviews, "
-        "gauging Reddit/YouTube sentiment, generating leads, needing current "
-        "web data, or wanting recurring reports — first answer as well as you "
+        "reveals a need the full platform serves — researching a topic or "
+        "competitor on the live web, tracking pricing or rankings, monitoring "
+        "brand mentions or reviews, gauging Reddit/YouTube sentiment, "
+        "generating leads, needing current web data, or wanting recurring "
+        "reports — first answer as well as you "
         "can from your own knowledge, then add ONE short sentence pointing out "
         "that a free SurfSense account can do that with live data, linking "
         "https://www.surfsense.com/register.\n"

surfsense_backend/app/agents/chat/multi_agent_chat/constants.py

@@ -29,6 +29,7 @@ CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS: dict[str, str] = {
 # connected app. Tokens are searchable-type strings (Composio Gmail/Calendar
 # map to the GOOGLE_* tokens in connector_searchable_types).
 SUBAGENT_TO_REQUIRED_CONNECTOR_MAP: dict[str, frozenset[str]] = {
+    "amazon": frozenset(),
     "deliverables": frozenset(),
     "knowledge_base": frozenset(),
     "web_crawler": frozenset(),

surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/private.md

@@ -1,14 +1,15 @@
 <agent_identity>
 You are **SurfSense's main agent**, the orchestrator of an open-source
-competitive intelligence platform. Users come to you to understand their
-market: what competitors are doing, how audiences react, where rankings and
-reviews are moving, and what is being said across the open web — and to put
-that intelligence to work alongside their own knowledge base.
+open web research platform. Users come to you to research the live web:
+what communities and audiences are saying, how rankings, reviews, and pages
+are changing, and what is being published across the open web — and to put
+that research to work alongside their own knowledge base.
 
 You do this by dispatching **specialist subagents** via the `task` tool:
-- **Live market data** — Reddit, YouTube, TikTok, Google Maps, Google Search,
-  and the web crawler return structured, current platform data (posts,
-  comments, transcripts, videos, reviews, SERPs, full page content).
+- **Live web data** — Reddit, YouTube, Instagram, TikTok, Amazon, Google
+  Maps, Google Search, and the web crawler return structured, current
+  platform data (posts, comments, transcripts, videos, products, reviews,
+  SERPs, full page content).
 - **The user's own context** — their knowledge base, connected apps, and
   persistent memory.
 - **Deliverables** — reports, podcasts, and presentations built from what the

surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/team.md

@@ -1,14 +1,15 @@
 <agent_identity>
 You are **SurfSense's main agent**, the orchestrator of an open-source
-competitive intelligence platform. This team comes to you to understand its
-market: what competitors are doing, how audiences react, where rankings and
-reviews are moving, and what is being said across the open web — and to put
-that intelligence to work alongside the team's shared knowledge base.
+open web research platform. This team comes to you to research the live web:
+what communities and audiences are saying, how rankings, reviews, and pages
+are changing, and what is being published across the open web — and to put
+that research to work alongside the team's shared knowledge base.
 
 You do this by dispatching **specialist subagents** via the `task` tool:
-- **Live market data** — Reddit, YouTube, TikTok, Google Maps, Google Search,
-  and the web crawler return structured, current platform data (posts,
-  comments, transcripts, videos, reviews, SERPs, full page content).
+- **Live web data** — Reddit, YouTube, Instagram, TikTok, Amazon, Google
+  Maps, Google Search, and the web crawler return structured, current
+  platform data (posts, comments, transcripts, videos, products, reviews,
+  SERPs, full page content).
 - **The team's own context** — its shared knowledge base, connected apps, and
   persistent team memory.
 - **Deliverables** — reports, podcasts, and presentations built from what the

surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md

@@ -1,9 +1,9 @@
 <knowledge_base_first>
 CRITICAL — ground factual answers in what you actually receive this turn:
 - **live platform data** via the market specialists —
-  `task(reddit, ...)`, `task(youtube, ...)`, `task(tiktok, ...)`,
-  `task(google_maps, ...)`, `task(google_search, ...)`,
-  `task(web_crawler, ...)`. Anything about
+  `task(reddit, ...)`, `task(youtube, ...)`, `task(instagram, ...)`,
+  `task(tiktok, ...)`, `task(amazon, ...)`, `task(google_maps, ...)`,
+  `task(google_search, ...)`, `task(web_crawler, ...)`. Anything about
   competitors, markets, rankings, reviews, or audience sentiment is answered
   from what these return **this turn**, never from your training data: your
   general knowledge of companies, prices, and rankings is stale by definition,

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/amazon/__init__.py

@@ -0,0 +1 @@
+"""``amazon`` builtin subagent: structured public Amazon product data."""

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/amazon/agent.py

@@ -0,0 +1,43 @@
+"""``amazon`` route: ``SurfSenseSubagentSpec`` builder for deepagents."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from langchain_core.language_models import BaseChatModel
+from langchain_core.tools import BaseTool
+
+from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import (
+    read_md_file,
+)
+from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec
+from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import (
+    pack_subagent,
+)
+
+from .tools.index import NAME, RULESET, load_tools
+
+
+def build_subagent(
+    *,
+    dependencies: dict[str, Any],
+    model: BaseChatModel | None = None,
+    middleware_stack: dict[str, Any] | None = None,
+    mcp_tools: list[BaseTool] | None = None,
+) -> SurfSenseSubagentSpec:
+    tools = [*load_tools(dependencies=dependencies), *(mcp_tools or [])]
+    description = (
+        read_md_file(__package__, "description").strip()
+        or "Scrapes public Amazon product data for a URL or search term."
+    )
+    system_prompt = read_md_file(__package__, "system_prompt").strip()
+    return pack_subagent(
+        name=NAME,
+        description=description,
+        system_prompt=system_prompt,
+        tools=tools,
+        ruleset=RULESET,
+        dependencies=dependencies,
+        model=model,
+        middleware_stack=middleware_stack,
+    )

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/amazon/description.md

@@ -0,0 +1,2 @@
+Amazon product specialist: scrapes public Amazon listings and returns structured product data — title, ASIN, brand, price and list price, star rating and review breakdown, availability, images, features, best-seller ranks, marketplace offers, third-party sellers, product variants, and on-page customer reviews. Works from a search term (e.g. "mechanical keyboard") or from Amazon product, search, category, best-seller, or short (a.co / amzn.to) URLs, on any Amazon marketplace domain. Only public, anonymous data — no login or seller account. Can enrich results with extra offers and seller profiles, expand product variants, and localize pricing/availability by country and ZIP.
+Use it for product research, price and buy-box tracking, review mining, catalog enrichment by ASIN, and best-seller/category monitoring. Triggers include "find X on Amazon", "Amazon price of X", "reviews for this Amazon product", "compare these Amazon products", "top-selling X on Amazon", and "look up this ASIN/Amazon URL". Not for general web search (use the Google Search specialist), reading an arbitrary non-Amazon page (use the web crawling specialist), or other marketplaces.

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/amazon/system_prompt.md

@@ -0,0 +1,67 @@
+You are the SurfSense Amazon sub-agent.
+You receive delegated instructions from a supervisor agent and return structured results for supervisor synthesis.
+
+<goal>
+Answer the delegated question from public Amazon product data gathered with your verb, comparing against earlier results already in this conversation when the task calls for it.
+</goal>
+
+<available_tools>
+- `amazon_scrape`
+- `read_run` / `search_run` (free readers for stored scrape output)
+</available_tools>
+
+<playbook>
+- Discovering products: call `amazon_scrape` with `search_terms` (e.g. ["mechanical keyboard"]), setting `domain` when a non-US marketplace matters (e.g. "www.amazon.co.uk").
+- Specific products: pass Amazon product URLs (or search / category / best-seller / short a.co URLs) in `urls`.
+- Faster listings: set `include_details=false` to return card-only results without opening each product page.
+- Pricing and buy box: raise `max_offers` to pull additional marketplace offers; set `include_sellers=true` to attach seller profiles.
+- Variants: raise `max_variants` to return variants as separate results; set `include_variant_prices=true` for per-variant prices.
+- Localized pricing/availability: set `country_code` and `zip_code`.
+- Batch multiple URLs or search terms into one call rather than many single-source calls.
+<include snippet="run_reader"/>
+- Comparison requests: pull the current products, compare against prior values already in this conversation's earlier tool results, and report concrete deltas (price up/down, rating change, rank moves, stock changes).
+</playbook>
+
+<tool_policy>
+- Use only tools in `<available_tools>`.
+- Report only results present in the tool output. Never invent titles, ASINs, prices, ratings, or rankings.
+- Provide at least one of `urls` or `search_terms`; they cannot be combined arbitrarily beyond the source cap.
+</tool_policy>
+
+<out_of_scope>
+- Do not perform general web search — that is the Google Search specialist's job.
+- Do not read or extract an arbitrary non-Amazon page — return the URL for the web crawling specialist.
+- Do not generate deliverables or perform connector mutations; return findings for the supervisor to act on.
+- Only public, anonymous Amazon data — never anything behind a login or seller account.
+</out_of_scope>
+
+<safety>
+- Report uncertainty explicitly when evidence is incomplete or conflicting.
+- Never present unverified claims as facts.
+</safety>
+
+<failure_policy>
+- Underspecified request — no usable search term or URL — return `status=blocked` with the missing fields.
+- Tool failure: return `status=error` with a concise recovery `next_step`.
+- No useful evidence: return `status=blocked` with a narrower query or the scope you still need.
+</failure_policy>
+
+<output_contract>
+Return **only** one JSON object (no markdown/prose):
+{
+  "status": "success" | "partial" | "blocked" | "error",
+  "action_summary": string,
+  "evidence": {
+    "findings": string[],
+    "sources": string[],
+    "confidence": "high" | "medium" | "low"
+  },
+  "next_step": string | null,
+  "missing_fields": string[] | null,
+  "assumptions": string[] | null
+}
+<include snippet="output_contract_base"/>
+Route-specific rules:
+- `evidence.findings`: max 10 entries, each a single sentence stating one distinct product or delta. Do not paste raw payloads.
+- `evidence.sources`: max 10 URLs, one per finding when applicable. List each URL once.
+</output_contract>

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/amazon/tools/index.py

@@ -0,0 +1,27 @@
+"""``amazon`` sub-agent tools: the Amazon product scrape capability verb."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from langchain_core.tools import BaseTool
+
+from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset
+from app.capabilities.amazon.scrape.definition import AMAZON_SCRAPE
+from app.capabilities.core.access.agent import build_capability_tools
+
+NAME = "amazon"
+
+RULESET = Ruleset(origin=NAME, rules=[])
+
+_CI_VERBS = [AMAZON_SCRAPE]
+
+
+def load_tools(
+    *, dependencies: dict[str, Any] | None = None, **kwargs: Any
+) -> list[BaseTool]:
+    d = {**(dependencies or {}), **kwargs}
+    return build_capability_tools(
+        workspace_id=d.get("workspace_id"),
+        capabilities=_CI_VERBS,
+    )

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/tools/report.py

@@ -773,7 +773,7 @@ def create_generate_report_tool(
 
             if not llm:
                 error_msg = (
-                    "No LLM configured. Please configure a language model in Settings."
+                    "No LLM configured. Please configure a chat model in Settings."
                 )
                 report_id = await _save_failed_report(error_msg)
                 return _failed(

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/tools/resume.py

@@ -585,7 +585,7 @@ def create_generate_resume_tool(
 
             if not llm:
                 error_msg = (
-                    "No LLM configured. Please configure a language model in Settings."
+                    "No LLM configured. Please configure a chat model in Settings."
                 )
                 report_id = await _save_failed_report(error_msg)
                 return _failed(

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/google_search/system_prompt.md

@@ -11,11 +11,11 @@ Answer the delegated question from live Google Search data gathered with your ve
 </available_tools>
 
 <playbook>
-- Finding pages on a topic: call `google_search_scrape` with `queries`, scoping with `country_code`/`language_code` when locale matters.
+- Google scraping is SLOW (each query is a live browser render on a vetted proxy; many queries in one call can blow the subagent's time budget and return nothing). Spend queries conservatively: start with a SINGLE best query, read the results, and only add further queries one at a time when that query genuinely fell short. Never fan out a batch of speculative query variations up front.
+- Finding pages on a topic: call `google_search_scrape` with a single-entry `queries`, scoping with `country_code`/`language_code` when locale matters. Broaden or reformulate in a follow-up call only if needed.
 - Restricting to one website: set `site` (e.g. "example.com") to only return results from that domain.
 - Scraping a specific results page: pass the full Google Search URL in `queries`.
-- Need more results: raise `max_pages_per_query` to page beyond the first page.
-- Batch multiple search terms into one call rather than many single-term calls.
+- Need more results: raise `max_pages_per_query` to page beyond the first page (cheaper and faster than adding more distinct queries).
 <include snippet="run_reader"/>
 - Handing URLs off for crawling: return the organic result URLs so the supervisor can route them to the web crawling specialist.
 - Comparison requests: pull the current results, compare against prior values already in this conversation's earlier tool results, and report concrete deltas (added, removed, moved up/down).

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py

@@ -12,6 +12,9 @@ from langchain_core.tools import BaseTool
 from app.agents.chat.multi_agent_chat.constants import (
     SUBAGENT_TO_REQUIRED_CONNECTOR_MAP,
 )
+from app.agents.chat.multi_agent_chat.subagents.builtins.amazon.agent import (
+    build_subagent as build_amazon_subagent,
+)
 from app.agents.chat.multi_agent_chat.subagents.builtins.deliverables.agent import (
     build_subagent as build_deliverables_subagent,
 )
@@ -83,6 +86,7 @@ class SubagentBuilder(Protocol):
 
 
 SUBAGENT_BUILDERS_BY_NAME: dict[str, SubagentBuilder] = {
+    "amazon": build_amazon_subagent,
     "deliverables": build_deliverables_subagent,
     "dropbox": build_dropbox_subagent,
     "google_drive": build_google_drive_subagent,

surfsense_backend/app/auth/csrf.py

@@ -32,6 +32,15 @@ def _allowed_origins() -> set[str]:
     return origins
 
 
+# Lets self-hosted deployments work from any address (LAN IP, custom domain)
+# without pre-configuring the static allowlist on .env.
+def _is_same_origin(origin: str | None, host: str | None) -> bool:
+    if not origin or not host:
+        return False
+    parsed_origin = urlparse(origin)
+    return parsed_origin.netloc == host
+
+
 class CsrfOriginMiddleware(BaseHTTPMiddleware):
     async def dispatch(
         self,
@@ -52,6 +61,11 @@ class CsrfOriginMiddleware(BaseHTTPMiddleware):
         origin = request.headers.get("Origin") or _origin_from_url(
             request.headers.get("Referer")
         )
+        host = request.headers.get("Host")
+
+        if _is_same_origin(origin, host):
+            return await call_next(request)
+
         if origin not in _allowed_origins():
             return JSONResponse(
                 {"detail": "CSRF origin check failed"},

surfsense_backend/app/capabilities/__init__.py

@@ -2,4 +2,6 @@
 
 from __future__ import annotations
 
+from app.capabilities import amazon as _amazon  # noqa: F401
+
 __all__: list[str] = []

surfsense_backend/app/capabilities/amazon/__init__.py

@@ -0,0 +1,5 @@
+"""Amazon capability namespace."""
+
+from __future__ import annotations
+
+from app.capabilities.amazon.scrape import definition as _scrape  # noqa: F401

surfsense_backend/app/capabilities/amazon/scrape/__init__.py

@@ -0,0 +1,3 @@
+"""Amazon product scraping capability."""
+
+from __future__ import annotations

surfsense_backend/app/capabilities/amazon/scrape/definition.py

@@ -0,0 +1,22 @@
+"""Registration for the ``amazon.scrape`` capability."""
+
+from __future__ import annotations
+
+from app.capabilities.amazon.scrape.executor import build_scrape_executor
+from app.capabilities.amazon.scrape.schemas import ScrapeInput, ScrapeOutput
+from app.capabilities.core import BillingUnit, Capability, register_capability
+
+AMAZON_SCRAPE = Capability(
+    name="amazon.scrape",
+    description=(
+        "Scrape public Amazon product details, search results, offers, sellers, "
+        "best-seller rankings, and on-page reviews."
+    ),
+    input_schema=ScrapeInput,
+    output_schema=ScrapeOutput,
+    executor=build_scrape_executor(),
+    billing_unit=BillingUnit.AMAZON_PRODUCT,
+    docs_url="/docs/connectors/native/amazon",
+)
+
+register_capability(AMAZON_SCRAPE)

surfsense_backend/app/capabilities/amazon/scrape/executor.py

@@ -0,0 +1,59 @@
+"""Executor for the ``amazon.scrape`` capability."""
+
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable
+from urllib.parse import quote_plus
+
+from app.capabilities.amazon.scrape.schemas import (
+    MAX_AMAZON_RESULTS,
+    ScrapeInput,
+    ScrapeOutput,
+)
+from app.capabilities.core import Executor
+from app.capabilities.core.progress import emit_progress
+from app.proprietary.platforms.amazon import AmazonScrapeInput, scrape_products
+
+ScrapeFn = Callable[..., Awaitable[list[dict]]]
+
+
+def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor:
+    """Bind the capability input mapping to a replaceable scraper function."""
+    scrape_fn = scrape_fn or scrape_products
+
+    async def execute(payload: ScrapeInput) -> ScrapeOutput:
+        search_urls = [
+            f"https://{payload.domain}/s?k={quote_plus(term)}"
+            for term in payload.search_terms
+        ]
+        input_model = AmazonScrapeInput(
+            categoryOrProductUrls=[
+                {"url": url} for url in [*payload.urls, *search_urls]
+            ],
+            maxItemsPerStartUrl=payload.max_items,
+            language=payload.language,
+            countryCode=payload.country_code,
+            zipCode=payload.zip_code,
+            scrapeProductDetails=payload.include_details,
+            maxOffers=payload.max_offers,
+            scrapeSellers=payload.include_sellers,
+            maxProductVariantsAsSeparateResults=payload.max_variants,
+            scrapeProductVariantPrices=payload.include_variant_prices,
+        )
+        emit_progress(
+            "starting",
+            "Scraping Amazon products",
+            total=payload.estimated_units,
+            unit="product",
+        )
+        items = await scrape_fn(input_model, limit=MAX_AMAZON_RESULTS)
+        emit_progress(
+            "done",
+            f"Scraped {sum('error' not in item for item in items)} product(s)",
+            current=len(items),
+            total=payload.estimated_units,
+            unit="product",
+        )
+        return ScrapeOutput(items=items)
+
+    return execute

surfsense_backend/app/capabilities/amazon/scrape/schemas.py

@@ -0,0 +1,61 @@
+"""Input and output contracts for ``amazon.scrape``."""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, Field, model_validator
+
+from app.proprietary.platforms.amazon import ProductItem
+
+MAX_AMAZON_SOURCES = 20
+MAX_AMAZON_RESULTS = 1000
+
+
+class ScrapeInput(BaseModel):
+    """Agent-facing controls for public product discovery and enrichment."""
+
+    urls: list[str] = Field(default_factory=list, max_length=MAX_AMAZON_SOURCES)
+    search_terms: list[str] = Field(default_factory=list, max_length=MAX_AMAZON_SOURCES)
+    max_items: int = Field(default=10, ge=1, le=100)
+    domain: str = Field(
+        default="www.amazon.com", pattern=r"^(?:www\.)?amazon\.[a-z.]+$"
+    )
+    language: str | None = None
+    country_code: str | None = Field(default=None, min_length=2, max_length=2)
+    zip_code: str | None = Field(default=None, min_length=1, max_length=20)
+    include_details: bool = True
+    max_offers: int = Field(default=0, ge=0, le=100)
+    include_sellers: bool = False
+    max_variants: int = Field(default=0, ge=0, le=100)
+    include_variant_prices: bool = False
+
+    @model_validator(mode="after")
+    def _require_source(self) -> ScrapeInput:
+        if not (self.urls or self.search_terms):
+            raise ValueError("Provide at least one URL or search term.")
+        if len(self.urls) + len(self.search_terms) > MAX_AMAZON_SOURCES:
+            raise ValueError(
+                f"Provide no more than {MAX_AMAZON_SOURCES} combined sources."
+            )
+        return self
+
+    @property
+    def estimated_units(self) -> int:
+        """Worst-case returned products within the hard per-run ceiling."""
+        search_products = len(self.search_terms) * self.max_items
+        direct_products = len(self.urls) * (1 + self.max_variants)
+        return min(search_products + direct_products, MAX_AMAZON_RESULTS)
+
+
+class ScrapeOutput(BaseModel):
+    """Products and structured per-input errors in emission order."""
+
+    items: list[ProductItem] = Field(default_factory=list)
+
+    @property
+    def billable_units(self) -> int:
+        """Count successful products; error items are never billed."""
+        return sum(
+            1
+            for item in self.items
+            if not (item.model_extra and item.model_extra.get("error"))
+        )

surfsense_backend/app/capabilities/core/billing.py

@@ -33,6 +33,7 @@ _PLATFORM_RATE_KEYS: dict[BillingUnit, str] = {
     BillingUnit.GOOGLE_SEARCH_SERP: "GOOGLE_SEARCH_MICROS_PER_SERP",
     BillingUnit.GOOGLE_MAPS_PLACE: "GOOGLE_MAPS_MICROS_PER_PLACE",
     BillingUnit.GOOGLE_MAPS_REVIEW: "GOOGLE_MAPS_MICROS_PER_REVIEW",
+    BillingUnit.AMAZON_PRODUCT: "AMAZON_MICROS_PER_PRODUCT",
     BillingUnit.YOUTUBE_VIDEO: "YOUTUBE_MICROS_PER_VIDEO",
     BillingUnit.YOUTUBE_COMMENT: "YOUTUBE_MICROS_PER_COMMENT",
     BillingUnit.INSTAGRAM_ITEM: "INSTAGRAM_SCRAPE_MICROS_PER_ITEM",
@@ -55,6 +56,7 @@ _UNIT_NOUNS: dict[BillingUnit, str] = {
     BillingUnit.GOOGLE_SEARCH_SERP: "SERP",
     BillingUnit.GOOGLE_MAPS_PLACE: "place",
     BillingUnit.GOOGLE_MAPS_REVIEW: "review",
+    BillingUnit.AMAZON_PRODUCT: "product",
     BillingUnit.YOUTUBE_VIDEO: "video",
     BillingUnit.YOUTUBE_COMMENT: "comment",
     BillingUnit.INSTAGRAM_ITEM: "item",

surfsense_backend/app/capabilities/core/types.py

@@ -23,6 +23,7 @@ class BillingUnit(StrEnum):
     GOOGLE_SEARCH_SERP = "google_search_serp"
     GOOGLE_MAPS_PLACE = "google_maps_place"
     GOOGLE_MAPS_REVIEW = "google_maps_review"
+    AMAZON_PRODUCT = "amazon_product"
     YOUTUBE_VIDEO = "youtube_video"
     YOUTUBE_COMMENT = "youtube_comment"
     INSTAGRAM_ITEM = "instagram_item"

surfsense_backend/app/config/__init__.py

@@ -697,9 +697,8 @@ class Config:
     # per item; retune with an env change + restart (no code/migration):
     #   <KEY> = round(USD_per_1000_items * 1_000)
     #   $3.50/1000 -> 3500 | $5.00/1000 -> 5000 | $2.00/1000 -> 2000
-    # Defaults sit at/above Apify's first-party actor rates (Jul 2026), which
-    # is justified because SurfSense charges no subscription tiers, no
-    # per-run actor-start fees, and no separate proxy/compute/storage billing.
+    # Defaults include margin for proxy, compute, and storage costs while
+    # remaining independently adjustable for each platform.
     PLATFORM_SCRAPE_BILLING_ENABLED = (
         os.getenv("PLATFORM_SCRAPE_BILLING_ENABLED", "FALSE").upper() == "TRUE"
     )
@@ -715,6 +714,7 @@ class Config:
     GOOGLE_MAPS_MICROS_PER_REVIEW = int(
         os.getenv("GOOGLE_MAPS_MICROS_PER_REVIEW", "1500")
     )
+    AMAZON_MICROS_PER_PRODUCT = int(os.getenv("AMAZON_MICROS_PER_PRODUCT", "3500"))
     YOUTUBE_MICROS_PER_VIDEO = int(os.getenv("YOUTUBE_MICROS_PER_VIDEO", "2500"))
     # Kept separate from the video rate so comments can be re-tuned toward the
     # cheaper per-comment market ($0.40-2.00/1k) without touching video pricing.
@@ -1107,7 +1107,8 @@ class Config:
     PROXY_URLS = os.getenv("PROXY_URLS")
 
     # =====================================================================
-    # Phase 3d — Captcha solving (reCAPTCHA v2/v3, hCaptcha) via captchatools.
+    # Phase 3d — Captcha solving (reCAPTCHA v2/v3, hCaptcha, v2-Enterprise) via
+    # the in-house solver seam (app/utils/captcha/solvers.py).
     # The LAST-resort bypass tier: only fires on the StealthyFetcher browser
     # tier, only when a sitekey is detected, and only when explicitly enabled.
     # Cloudflare Turnstile is already handled free in-framework (03a), NOT here.
@@ -1119,9 +1120,9 @@ class Config:
     CAPTCHA_SOLVING_ENABLED = (
         os.getenv("CAPTCHA_SOLVING_ENABLED", "FALSE").upper() == "TRUE"
     )
-    # captchatools "solving_site": capmonster | 2captcha | anticaptcha |
-    # capsolver | captchaai. captchatools is itself the provider registry, so we
-    # do not rebuild a vendor hierarchy.
+    # Solver vendor. "capsolver" (AI-native, fastest on reCAPTCHA-Enterprise) and
+    # "2captcha" have in-house clients today; anticaptcha / capmonster are added
+    # progressively in solvers._PROVIDERS.
     CAPTCHA_SOLVER_PROVIDER = os.getenv("CAPTCHA_SOLVER_PROVIDER", "capsolver")
     CAPTCHA_SOLVER_API_KEY = os.getenv("CAPTCHA_SOLVER_API_KEY")
     # Per-URL solve cap so one hostile page can't burn unbounded solver credit.

surfsense_backend/app/db.py

@@ -1732,6 +1732,11 @@ class Workspace(BaseModel, TimestampMixin):
         Integer, nullable=True, default=0, server_default="0"
     )  # For vision/screenshot analysis, defaults to Auto mode
 
+    # First time this workspace went ready via its own model (source=="models").
+    # NULL = never self-configured. Set once, never cleared; splits a needs_setup
+    # verdict into first-run vs. recovery.
+    llm_setup_completed_at = Column(TIMESTAMP(timezone=True), nullable=True)
+
     user_id = Column(
         UUID(as_uuid=True), ForeignKey("user.id", ondelete="CASCADE"), nullable=False
     )

surfsense_backend/app/proprietary/platforms/amazon/README.md

@@ -0,0 +1,50 @@
+# Amazon Product Scraper
+
+Current status: product details, search and category discovery, offers, sellers,
+best-seller rankings, on-page reviews, localized delivery sessions, and product
+variants are implemented. The `amazon.scrape` capability exposes the scraper
+with per-product billing and a hard per-run result ceiling.
+
+## Scope
+
+The scraper reads public pages available to anonymous visitors. It does not log
+in, use account cookies, or retrieve account-gated content. Delivery
+localization uses an anonymous session cookie pinned to the same proxy exit.
+
+Deep review pagination currently requires an account on Amazon and is therefore
+out of scope. Reviews embedded in a public product page are returned in
+`productPageReviews` and `productPageReviewsFromOtherCountries`.
+
+## Architecture
+
+- `schemas.py` defines the stable input, product, and error models.
+- `url_resolver.py` classifies product, search, best-sellers, and shortened URLs.
+- `fetch.py` owns proxy-aware HTTP access, block detection, retries, and
+  anonymous location sessions.
+- `parsers.py` contains pure, defensive HTML parsers.
+- `scraper.py` coordinates discovery, enrichment, concurrency, limits, and
+  in-stream errors.
+
+## Implementation progress
+
+- Done: public product detail parsing and shortened-link dispatch.
+- Done: search/category paging with card-only and detailed modes.
+- Done: public offers and seller enrichment.
+- Done: best-seller rankings and product-page reviews.
+- Done: anonymous localized delivery sessions on sticky proxy exits.
+- Done: variant expansion, variant prices, capability registration, and billing.
+
+## Verification
+
+Offline fixtures cover every parser and flow:
+
+```bash
+cd surfsense_backend
+uv run pytest tests/unit/platforms/amazon tests/unit/capabilities/amazon
+```
+
+The manual live check requires proxy credentials:
+
+```bash
+uv run python scripts/e2e_amazon_scraper.py
+```

surfsense_backend/app/proprietary/platforms/amazon/__init__.py

@@ -0,0 +1,12 @@
+"""Platform-native Amazon Product Scraper."""
+
+from .schemas import AmazonScrapeInput, ErrorItem, ProductItem
+from .scraper import iter_products, scrape_products
+
+__all__ = [
+    "AmazonScrapeInput",
+    "ErrorItem",
+    "ProductItem",
+    "iter_products",
+    "scrape_products",
+]

surfsense_backend/app/proprietary/platforms/amazon/fetch.py

@@ -0,0 +1,388 @@
+"""Network access for public Amazon pages.
+
+All requests use the configured residential proxy. Responses that contain an
+anti-bot interstitial are retried with a fresh proxy exit, while ordinary HTTP
+failures are returned to the caller for domain-specific error handling.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import re
+import time
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass, field
+from typing import Any
+
+from scrapling.fetchers import AsyncFetcher
+
+from app.utils.proxy import get_geo_proxy_url, get_sticky_proxy_url
+
+logger = logging.getLogger(__name__)
+
+_MAX_IP_ATTEMPTS = 8
+_REQUEST_TIMEOUT_S = 30
+_HEADERS = {
+    "Accept-Language": "en-US,en;q=0.9",
+    "Accept": "text/html,application/xhtml+xml,application/json;q=0.9,*/*;q=0.8",
+}
+_BLOCK_MARKERS = (
+    "/errors/validatecaptcha",
+    "api-services-support@amazon.com",
+    "robot check",
+    "enter the characters you see below",
+    "token.awswaf.com",
+    "/challenge.js",
+    "awswafintegration",
+    "bm-verify=",
+)
+_CSRF_PATTERNS = (
+    re.compile(
+        r"""(?:name|id)=["'](?:anti-csrftoken-a2z|csrf-token)["'][^>]*"""
+        r"""(?:value|content)=["']([^"']+)""",
+        re.IGNORECASE,
+    ),
+    re.compile(
+        r"""["'](?:anti-csrftoken-a2z|csrf-token)["']\s*[:=]\s*["']([^"']+)""",
+        re.IGNORECASE,
+    ),
+)
+
+
+@dataclass(frozen=True)
+class FetchResult:
+    """The response details needed by scraper flows."""
+
+    status: int
+    html: str
+    url: str
+    cookies: dict[str, str]
+    headers: dict[str, str] = field(default_factory=dict)
+
+
+@dataclass
+class LocationSession:
+    """Anonymous delivery-location state pinned to one proxy exit."""
+
+    proxy: str | None
+    cookies: dict[str, str]
+    country_code: str | None
+    zip_code: str
+    location_text: str | None
+    created_at: float
+
+
+async def gather_bounded[T](
+    factories: list[Callable[[], Awaitable[T]]], *, concurrency: int
+) -> list[T]:
+    """Run async factories concurrently while preserving input order."""
+    if not factories:
+        return []
+    semaphore = asyncio.Semaphore(max(1, concurrency))
+
+    async def run(factory: Callable[[], Awaitable[T]]) -> T:
+        async with semaphore:
+            return await factory()
+
+    return await asyncio.gather(*(run(factory) for factory in factories))
+
+
+def is_blocked(
+    html: str | None, status: int, headers: dict[str, str] | None = None
+) -> bool:
+    """Return whether a response is an Amazon anti-bot interstitial.
+
+    ``202`` is a soft anti-bot response Amazon serves to some proxy exits (an
+    empty/accepted body rather than the page), so it is treated as blocked and
+    retried on a fresh exit rather than parsed as a real page.
+    """
+    if status in {202, 429, 503}:
+        return True
+    if _header_value(headers, "x-amzn-waf-action") == "challenge":
+        return True
+    text = (html or "")[:200_000].lower()
+    return any(marker in text for marker in _BLOCK_MARKERS)
+
+
+def _header_value(headers: dict[str, str] | None, name: str) -> str | None:
+    if not headers:
+        return None
+    needle = name.lower()
+    for key, value in headers.items():
+        if key.lower() == needle:
+            return str(value).strip().lower()
+    return None
+
+
+def _response_url(page: Any, fallback: str) -> str:
+    value = getattr(page, "url", None)
+    return str(value) if value else fallback
+
+
+def _response_cookies(page: Any) -> dict[str, str]:
+    cookies = getattr(page, "cookies", None)
+    return dict(cookies) if isinstance(cookies, dict) else {}
+
+
+def _response_headers(page: Any) -> dict[str, str]:
+    headers = getattr(page, "headers", None)
+    return dict(headers) if isinstance(headers, dict) else {}
+
+
+def _selected_proxy(
+    proxy: str | None, country: str | None, attempt: int, url: str
+) -> str | None:
+    if proxy is not None:
+        return proxy
+    if country and attempt > 1:
+        session_id = f"amazon-{country}-{attempt}-{abs(hash((url, time.time_ns()))):x}"
+        return get_sticky_proxy_url(session_id, country)
+    return get_geo_proxy_url(country)
+
+
+async def fetch_page(
+    url: str,
+    *,
+    cookies: dict[str, str] | None = None,
+    proxy: str | None = None,
+    country: str | None = None,
+    accept_language: str | None = None,
+    method: str = "GET",
+    data: dict[str, str] | None = None,
+    headers: dict[str, str] | None = None,
+    rotate_on_block: bool = True,
+) -> FetchResult | None:
+    """Fetch a page and retry blocked responses with fresh proxy exits."""
+    attempts = _MAX_IP_ATTEMPTS if rotate_on_block else 1
+    for attempt in range(1, attempts + 1):
+        selected_proxy = _selected_proxy(proxy, country, attempt, url)
+        started = time.perf_counter()
+        try:
+            request = AsyncFetcher.post if method == "POST" else AsyncFetcher.get
+            request_headers = {**_HEADERS}
+            if accept_language:
+                request_headers["Accept-Language"] = (
+                    f"{accept_language},{accept_language.split('-', 1)[0]};q=0.9"
+                )
+            request_headers.update(headers or {})
+            kwargs: dict[str, Any] = {
+                "headers": request_headers,
+                "cookies": cookies or {},
+                "proxy": selected_proxy,
+                "stealthy_headers": True,
+                "timeout": _REQUEST_TIMEOUT_S,
+            }
+            if method == "POST":
+                kwargs["data"] = data or {}
+            page = await request(url, **kwargs)
+        except Exception as exc:
+            logger.warning("Amazon request failed for %s: %s", url, exc)
+            if proxy is not None:
+                return None
+            continue
+
+        status = int(getattr(page, "status", 0) or 0)
+        html = getattr(page, "html_content", None) or ""
+        response_headers = _response_headers(page)
+        logger.info(
+            "[amazon][perf] method=%s status=%s attempt=%s fetch_ms=%.1f url=%s",
+            method,
+            status,
+            attempt,
+            (time.perf_counter() - started) * 1000,
+            url,
+        )
+        if rotate_on_block and is_blocked(html, status, response_headers):
+            logger.info(
+                "Amazon blocked proxy attempt %s/%s for %s", attempt, attempts, url
+            )
+            continue
+        return FetchResult(
+            status=status,
+            html=html,
+            url=_response_url(page, url),
+            cookies=_response_cookies(page),
+            headers=response_headers,
+        )
+    logger.warning("Amazon exhausted %s proxy attempts for %s", attempts, url)
+    return None
+
+
+async def fetch_html(
+    url: str,
+    *,
+    cookies: dict[str, str] | None = None,
+    proxy: str | None = None,
+    country: str | None = None,
+    accept_language: str | None = None,
+) -> str | None:
+    """Return public page HTML, or ``None`` when no usable response is obtained."""
+    result = await fetch_page(
+        url,
+        cookies=cookies,
+        proxy=proxy,
+        country=country,
+        accept_language=accept_language,
+    )
+    return result.html if result is not None and result.status == 200 else None
+
+
+async def resolve_shortlink(
+    url: str, *, country: str | None = None, accept_language: str | None = None
+) -> str | None:
+    """Follow a shortened Amazon URL and return its final destination."""
+    result = await fetch_page(url, country=country, accept_language=accept_language)
+    return result.url if result is not None and result.status == 200 else None
+
+
+def _origin(domain: str) -> str:
+    return f"https://{domain}"
+
+
+def _csrf_token(html: str) -> str | None:
+    for pattern in _CSRF_PATTERNS:
+        match = pattern.search(html)
+        if match:
+            return match.group(1)
+    return None
+
+
+async def fetch_aod_html(
+    asin: str,
+    domain: str,
+    *,
+    cookies: dict[str, str] | None = None,
+    proxy: str | None = None,
+    country: str | None = None,
+    accept_language: str | None = None,
+) -> str | None:
+    """Fetch the public all-offers panel for one product.
+
+    ``aodAjaxMain`` is a path segment, not a query param. Amazon also serves this
+    panel as a JS-only modal for many (especially US) ASINs, so a 404 here is
+    expected and the caller falls back to the PDP buy-box winner.
+    """
+    url = f"{_origin(domain)}/gp/product/ajax/aodAjaxMain/?asin={asin}"
+    return await fetch_html(
+        url,
+        cookies=cookies,
+        proxy=proxy,
+        country=country,
+        accept_language=accept_language,
+    )
+
+
+async def fetch_seller_html(
+    seller_id: str,
+    domain: str,
+    *,
+    cookies: dict[str, str] | None = None,
+    proxy: str | None = None,
+    country: str | None = None,
+    accept_language: str | None = None,
+) -> str | None:
+    """Fetch a public seller profile."""
+    return await fetch_html(
+        f"{_origin(domain)}/sp?seller={seller_id}",
+        cookies=cookies,
+        proxy=proxy,
+        country=country,
+        accept_language=accept_language,
+    )
+
+
+_LOCATION_SESSION_TTL_S = 30 * 60
+_location_sessions: dict[tuple[str, str, str | None], LocationSession] = {}
+_location_locks: dict[tuple[str, str, str | None], asyncio.Lock] = {}
+
+
+def should_localize(route: str, deliverable_routes: list[str] | None) -> bool:
+    """Return whether a request route should use delivery-location state."""
+    return route.upper() in {value.upper() for value in (deliverable_routes or [])}
+
+
+async def _sticky_proxy(session_id: str, country: str | None = None) -> str | None:
+    """Request a stable proxy exit when the active provider supports it."""
+    try:
+        return get_sticky_proxy_url(session_id, country)
+    except (ImportError, NotImplementedError):
+        return get_geo_proxy_url(country)
+
+
+async def get_location_session(
+    domain: str,
+    *,
+    zip_code: str,
+    country_code: str | None,
+    country: str | None = None,
+    accept_language: str | None = None,
+) -> LocationSession | None:
+    """Create or reuse an anonymous delivery-location session."""
+    key = (domain, zip_code, country_code)
+    cached = _location_sessions.get(key)
+    if cached and time.time() - cached.created_at < _LOCATION_SESSION_TTL_S:
+        return cached
+
+    lock = _location_locks.setdefault(key, asyncio.Lock())
+    async with lock:
+        cached = _location_sessions.get(key)
+        if cached and time.time() - cached.created_at < _LOCATION_SESSION_TTL_S:
+            return cached
+
+        session_id = f"amazon-{abs(hash(key)) & 0xFFFFFFFF:x}"
+        proxy = await _sticky_proxy(session_id, country)
+        home = await fetch_page(
+            f"{_origin(domain)}/?ref_=nav_logo",
+            proxy=proxy,
+            country=country,
+            accept_language=accept_language,
+            rotate_on_block=False,
+        )
+        if home is None or home.status != 200:
+            return None
+        csrf = _csrf_token(home.html)
+        if csrf is None:
+            logger.warning("Amazon location session did not expose a CSRF token")
+            return None
+
+        endpoint = f"{_origin(domain)}/gp/delivery/ajax/address-change.html"
+        payload = {
+            "locationType": "LOCATION_INPUT",
+            "zipCode": zip_code,
+            "storeContext": "generic",
+            "deviceType": "web",
+            "pageType": "Gateway",
+            "actionSource": "glow",
+        }
+        cookies = dict(home.cookies)
+        cookies["anti-csrftoken-a2z"] = csrf
+        changed = await fetch_page(
+            endpoint,
+            cookies=cookies,
+            proxy=proxy,
+            method="POST",
+            data=payload,
+            headers={
+                "anti-csrftoken-a2z": csrf,
+                "X-Requested-With": "XMLHttpRequest",
+                "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
+            },
+            country=country,
+            accept_language=accept_language,
+            rotate_on_block=False,
+        )
+        if changed is None or changed.status != 200:
+            return None
+        cookies.update(changed.cookies)
+        location_text = zip_code
+        session = LocationSession(
+            proxy=proxy,
+            cookies=cookies,
+            country_code=country_code.upper() if country_code else None,
+            zip_code=zip_code,
+            location_text=location_text,
+            created_at=time.time(),
+        )
+        _location_sessions[key] = session
+        return session

surfsense_backend/app/proprietary/platforms/amazon/locale.py

@@ -0,0 +1,33 @@
+"""Marketplace locale helpers for Amazon fetch routing.
+
+Only proxy countries verified against the configured DataImpulse plan are mapped
+here. Unmapped marketplaces fall back to the default proxy exit.
+"""
+
+_MARKETPLACE_COUNTRY = {
+    "com": "us",
+    "co.uk": "gb",
+    "de": "de",
+    "fr": "fr",
+    "it": "it",
+    "es": "es",
+}
+
+_MARKETPLACE_LANGUAGE = {
+    "com": "en-US",
+    "co.uk": "en-GB",
+    "de": "de-DE",
+    "fr": "fr-FR",
+    "it": "it-IT",
+    "es": "es-ES",
+}
+
+
+def proxy_country_for(marketplace: str | None) -> str | None:
+    """Return a confirmed proxy exit country for one Amazon marketplace."""
+    return _MARKETPLACE_COUNTRY.get((marketplace or "").lower())
+
+
+def accept_language_for(marketplace: str | None) -> str:
+    """Return the browser language header that matches the marketplace UI."""
+    return _MARKETPLACE_LANGUAGE.get((marketplace or "").lower(), "en-US")

surfsense_backend/app/proprietary/platforms/amazon/parsers.py

@@ -0,0 +1,586 @@
+"""Pure parsers for public Amazon HTML.
+
+Selectors cover stable element IDs first and use small, explicit fallbacks for
+layout variants. Missing sections return empty or nullable values so isolated
+markup changes do not discard an otherwise usable product.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from html import unescape
+from typing import Any
+from urllib.parse import parse_qs, urljoin, urlparse
+
+from scrapling.parser import Adaptor
+
+_NUMBER_RE = re.compile(r"[\d,.]+")
+_ASIN_RE = re.compile(r"^[A-Z0-9]{10}$")
+_ASIN_IN_TEXT_RE = re.compile(r"\b[A-Z0-9]{10}\b")
+_CURRENCY_BY_SYMBOL = {"$": "USD", "€": "EUR", "£": "GBP", "¥": "JPY", "₹": "INR"}
+_JSON_ASSIGNMENT_RE = re.compile(
+    r"(?:dimensionValuesDisplayData|variationValues|dimensionToAsinMap)"
+    r"""\s*["']?\s*:\s*(\{.*?\})(?:,\s*["']|\s*;)""",
+    re.DOTALL,
+)
+
+
+def _one(node: Any, selector: str) -> Any | None:
+    found = node.css(selector)
+    return found[0] if found else None
+
+
+def _text(node: Any | None) -> str | None:
+    if node is None:
+        return None
+    raw = node.get_all_text(strip=True)
+    return re.sub(r"\s+", " ", raw).strip() or None
+
+
+def _texts(node: Any, selector: str) -> list[str]:
+    values: list[str] = []
+    for match in node.css(selector):
+        value = _text(match)
+        if value and value not in values:
+            values.append(value)
+    return values
+
+
+def _first_text(node: Any, *selectors: str) -> str | None:
+    for selector in selectors:
+        value = _text(_one(node, selector))
+        if value:
+            return value
+    return None
+
+
+def _attr(node: Any | None, name: str) -> str | None:
+    if node is None:
+        return None
+    value = node.attrib.get(name)
+    return str(value).strip() if value else None
+
+
+def _first_attr(node: Any, name: str, *selectors: str) -> str | None:
+    for selector in selectors:
+        value = _attr(_one(node, selector), name)
+        if value:
+            return value
+    return None
+
+
+def _integer(value: str | None) -> int | None:
+    if not value:
+        return None
+    match = _NUMBER_RE.search(value)
+    if not match:
+        return None
+    digits = re.sub(r"\D", "", match.group(0))
+    return int(digits) if digits else None
+
+
+def _float(value: str | None) -> float | None:
+    if not value:
+        return None
+    match = _NUMBER_RE.search(value)
+    if not match:
+        return None
+    token = match.group(0)
+    if "," in token and "." in token:
+        decimal = "," if token.rfind(",") > token.rfind(".") else "."
+        grouping = "." if decimal == "," else ","
+        token = token.replace(grouping, "").replace(decimal, ".")
+    elif (
+        token.count(",") == 1 and "." not in token and len(token.rsplit(",", 1)[1]) <= 2
+    ):
+        token = token.replace(",", ".")
+    else:
+        token = token.replace(",", "")
+    try:
+        return float(token)
+    except ValueError:
+        return None
+
+
+def _price(value: str | None) -> dict[str, Any] | None:
+    amount = _float(value)
+    if amount is None:
+        return None
+    currency = next(
+        (
+            code
+            for symbol, code in _CURRENCY_BY_SYMBOL.items()
+            if symbol in (value or "")
+        ),
+        None,
+    )
+    if currency is None and value:
+        match = re.search(r"\b(USD|EUR|GBP|JPY|INR|CAD|AUD)\b", value)
+        currency = match.group(1) if match else None
+    return {"value": amount, "currency": currency}
+
+
+def _absolute(domain: str, href: str | None) -> str | None:
+    return urljoin(f"https://{domain}", href) if href else None
+
+
+def _asin_from_href(href: str | None) -> str | None:
+    if not href:
+        return None
+    match = re.search(r"/(?:dp|gp/product)/([A-Z0-9]{10})(?:[/?]|$)", href)
+    return match.group(1) if match else None
+
+
+def _embedded_json(html: str, key: str) -> Any | None:
+    """Decode a JSON value assigned to ``key`` inside an inline script."""
+    marker = re.search(rf"""["']?{re.escape(key)}["']?\s*:\s*""", html)
+    if marker is None:
+        return None
+    start = marker.end()
+    while start < len(html) and html[start].isspace():
+        start += 1
+    if start >= len(html) or html[start] not in "[{":
+        return None
+    opening = html[start]
+    closing = "}" if opening == "{" else "]"
+    depth = 0
+    quoted = False
+    escaped = False
+    for index in range(start, len(html)):
+        char = html[index]
+        if quoted:
+            if escaped:
+                escaped = False
+            elif char == "\\":
+                escaped = True
+            elif char == '"':
+                quoted = False
+            continue
+        if char == '"':
+            quoted = True
+        elif char == opening:
+            depth += 1
+        elif char == closing:
+            depth -= 1
+            if depth == 0:
+                try:
+                    return json.loads(html[start : index + 1])
+                except json.JSONDecodeError:
+                    return None
+    return None
+
+
+def _stars_breakdown(doc: Adaptor) -> dict[str, float | None] | None:
+    breakdown: dict[str, float | None] = {}
+    for row in doc.css("#histogramTable tr, [data-hook='rating-histogram'] tr"):
+        label = _first_text(row, ".a-text-left", "td:first-child", "span")
+        percent = _first_text(row, ".a-text-right", "td:last-child")
+        star = _integer(label)
+        value = _float(percent)
+        if star and 1 <= star <= 5 and value is not None:
+            breakdown[f"{star}star"] = value / 100 if value > 1 else value
+    return breakdown or None
+
+
+def _variant_data(
+    html: str,
+) -> tuple[list[str], list[dict[str, Any]], list[dict[str, Any]]]:
+    display = _embedded_json(html, "dimensionValuesDisplayData")
+    variants = _embedded_json(html, "variationValues")
+    asins: list[str] = []
+    details: list[dict[str, Any]] = []
+    attributes: list[dict[str, Any]] = []
+
+    if isinstance(display, dict):
+        for asin, values in display.items():
+            if _ASIN_RE.match(str(asin)):
+                asins.append(str(asin))
+                details.append({"asin": str(asin), "values": values})
+    if isinstance(variants, dict):
+        for name, values in variants.items():
+            attributes.append({"name": name, "values": values})
+        for value in variants.values():
+            if isinstance(value, list):
+                for candidate in value:
+                    if isinstance(candidate, str) and _ASIN_RE.match(candidate):
+                        asins.append(candidate)
+    return list(dict.fromkeys(asins)), details, attributes
+
+
+def _attributes(doc: Adaptor) -> tuple[list[dict[str, str]], dict[str, str]]:
+    pairs: list[dict[str, str]] = []
+    mapped: dict[str, str] = {}
+    for row in doc.css(
+        "#productDetails_techSpec_section_1 tr, "
+        "#productDetails_detailBullets_sections1 tr, "
+        "#detailBullets_feature_div li"
+    ):
+        name = _first_text(row, "th", ".a-text-bold")
+        value = _first_text(row, "td", "span:not(.a-text-bold)")
+        if name and value:
+            name = name.rstrip(": \u200e")
+            pairs.append({"name": name, "value": value})
+            mapped[name] = value
+    return pairs, mapped
+
+
+def _reviews(
+    doc: Adaptor, domain: str
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    local: list[dict[str, Any]] = []
+    foreign: list[dict[str, Any]] = []
+    for node in doc.css("[data-hook='review']"):
+        review = {
+            "id": _attr(node, "id"),
+            "title": _first_text(node, "[data-hook='review-title']"),
+            "stars": _float(_first_text(node, "[data-hook='review-star-rating']")),
+            "text": _first_text(node, "[data-hook='review-body']"),
+            "author": _first_text(node, ".a-profile-name"),
+            "date": _first_text(node, "[data-hook='review-date']"),
+            "verified": _one(node, "[data-hook='avp-badge']") is not None,
+            "helpfulVotes": _integer(
+                _first_text(node, "[data-hook='helpful-vote-statement']")
+            ),
+        }
+        if any(value is not None for value in review.values()):
+            target = (
+                foreign
+                if "other countries" in (_text(node) or "").lower()
+                or "cm_cr_arp_d_rvw_rvwer" in (_attr(node, "class") or "")
+                else local
+            )
+            target.append(review)
+    return local, foreign
+
+
+def parse_product(
+    html: str, *, asin: str, url: str, domain: str | None = None
+) -> dict[str, Any]:
+    """Parse one product detail page into output-model fields."""
+    doc = Adaptor(html)
+    domain = domain or (urlparse(url).hostname or "www.amazon.com")
+    title = _first_text(doc, "#productTitle", "#title")
+    price_text = _first_text(
+        doc,
+        "#corePrice_feature_div .a-offscreen",
+        "#corePriceDisplay_desktop_feature_div .a-offscreen",
+        "#apex_desktop .a-price .a-offscreen",
+        "#priceblock_ourprice",
+        "#priceblock_dealprice",
+        ".priceToPay .a-offscreen",
+        "#price_inside_buybox",
+        "#centerCol .a-price .a-offscreen",
+    )
+    list_price_text = _first_text(
+        doc,
+        "#corePrice_feature_div .basisPrice .a-offscreen",
+        "#corePriceDisplay_desktop_feature_div .basisPrice .a-offscreen",
+        ".priceBlockStrikePriceString",
+        "#centerCol .a-price.a-text-price .a-offscreen",
+    )
+    availability = _first_text(doc, "#availability", "#outOfStock")
+    availability_lower = (availability or "").lower()
+    in_stock = (
+        None
+        if availability is None
+        else not any(
+            word in availability_lower for word in ("unavailable", "out of stock")
+        )
+    )
+    brand_link = _one(doc, "#bylineInfo")
+    brand_text = _text(brand_link)
+    brand = (
+        re.sub(r"^(visit the |brand:\s*)| store$", "", brand_text, flags=re.IGNORECASE)
+        if brand_text
+        else None
+    )
+    variant_asins, variant_details, variant_attributes = _variant_data(html)
+    attrs, attrs_mapped = _attributes(doc)
+    local_reviews, foreign_reviews = _reviews(doc, domain)
+
+    high_res: list[str] = []
+    gallery: list[str] = []
+    for image in doc.css("#altImages img, #landingImage, #imgTagWrapperId img"):
+        thumb = _attr(image, "src")
+        if thumb and thumb not in gallery:
+            gallery.append(thumb)
+        dynamic = _attr(image, "data-a-dynamic-image")
+        if dynamic:
+            try:
+                decoded = json.loads(unescape(dynamic))
+                for candidate in decoded:
+                    if candidate not in high_res:
+                        high_res.append(candidate)
+            except (json.JSONDecodeError, TypeError):
+                pass
+        zoom = _attr(image, "data-old-hires")
+        if zoom and zoom not in high_res:
+            high_res.append(zoom)
+
+    ranks: list[dict[str, Any]] = []
+    for node in doc.css(
+        "#detailBulletsWrapper_feature_div li, #productDetails_detailBullets_sections1 tr"
+    ):
+        text = _text(node)
+        if not text or "best sellers rank" not in text.lower():
+            continue
+        for rank, category in re.findall(r"#([\d,]+)\s+in\s+([^#(]+)", text):
+            ranks.append(
+                {"rank": int(rank.replace(",", "")), "category": category.strip()}
+            )
+
+    aplus = _one(doc, "#aplus, #aplus_feature_div")
+    brand_story = _one(doc, "#brandStory_feature_div, .apm-brand-story")
+    store_href = _attr(brand_link, "href")
+    reviews_link = _first_attr(
+        doc, "href", "#acrCustomerReviewLink", "[data-hook='see-all-reviews-link-foot']"
+    )
+    seller_link = _one(
+        doc,
+        "#sellerProfileTriggerId, #merchant-info a[href*='seller='], a[href*='/sp?seller=']",
+    )
+    seller_href = _attr(seller_link, "href")
+    seller_id = (parse_qs(urlparse(seller_href or "").query).get("seller") or [None])[0]
+    # Fall back to the buy-box "Sold by" name when there is no seller profile link
+    # (Amazon-direct items and some link-less third-party merchants).
+    seller_name = _text(seller_link) or _first_text(
+        doc,
+        "#tabular-buybox [tabular-attribute-name='Sold by'] .tabular-buybox-text",
+        "#merchantInfoFeature_feature_div .offer-display-feature-text-message",
+    )
+    return {
+        "title": title,
+        "url": url,
+        "asin": asin,
+        "originalAsin": asin,
+        "brand": brand,
+        "author": _first_text(
+            doc, ".author", "#bylineInfo_feature_div .contributorNameID"
+        ),
+        "price": _price(price_text),
+        "listPrice": _price(list_price_text),
+        "shippingPrice": _price(
+            _first_text(doc, "#deliveryBlockMessage .a-color-secondary")
+        ),
+        "inStock": in_stock,
+        "inStockText": availability,
+        "delivery": _first_text(
+            doc, "#mir-layout-DELIVERY_BLOCK-slot-PRIMARY_DELIVERY_MESSAGE_LARGE"
+        ),
+        "fastestDelivery": _first_text(
+            doc, "#mir-layout-DELIVERY_BLOCK-slot-SECONDARY_DELIVERY_MESSAGE_LARGE"
+        ),
+        "condition": _first_text(
+            doc, "#buyNew_noncbb .a-color-price", "#usedAccordionRow"
+        ),
+        "stars": _float(
+            _first_text(doc, "#acrPopover", "[data-hook='rating-out-of-text']")
+        ),
+        "starsBreakdown": _stars_breakdown(doc),
+        "reviewsCount": _integer(_first_text(doc, "#acrCustomerReviewText")),
+        "answeredQuestions": _integer(_first_text(doc, "#askATFLink")),
+        "aiReviewsSummary": (
+            {
+                "text": _first_text(
+                    doc, "#product-summary, [data-hook='cr-insights-widget']"
+                )
+            }
+            if _first_text(doc, "#product-summary, [data-hook='cr-insights-widget']")
+            else None
+        ),
+        "monthlyPurchaseVolume": _first_text(
+            doc, "#social-proofing-faceout-title-tk_bought"
+        ),
+        "breadCrumbs": " > ".join(
+            _texts(doc, "#wayfinding-breadcrumbs_feature_div li a")
+        )
+        or None,
+        "description": _first_text(
+            doc, "#productDescription", "#bookDescription_feature_div"
+        ),
+        "features": _texts(doc, "#feature-bullets li span.a-list-item"),
+        "sustainabilityFeatures": [
+            {"text": value}
+            for value in _texts(doc, "#sustainability_feature_div .a-list-item")
+        ],
+        "videosCount": _integer(_first_text(doc, "#videoCount")),
+        "visitStoreLink": (
+            {"text": brand_text, "url": _absolute(domain, store_href)}
+            if store_href
+            else None
+        ),
+        "thumbnailImage": _first_attr(doc, "src", "#landingImage", "#imgBlkFront"),
+        "galleryThumbnails": gallery,
+        "highResolutionImages": high_res,
+        "aPlusContent": (
+            {
+                "text": _text(aplus),
+                "images": [
+                    _attr(img, "src") for img in aplus.css("img") if _attr(img, "src")
+                ],
+            }
+            if aplus is not None
+            else None
+        ),
+        "brandStory": {"text": _text(brand_story)} if brand_story is not None else None,
+        "returnPolicy": _first_text(
+            doc, "#returnsInfoFeature_feature_div", "#RETURNS_POLICY"
+        ),
+        "support": _first_text(doc, "#support_feature_div"),
+        "variantAsins": variant_asins,
+        "variantDetails": variant_details,
+        "variantAttributes": variant_attributes,
+        "attributes": attrs,
+        "attributesMapped": attrs_mapped or None,
+        "productOverview": [
+            {
+                "name": _first_text(row, "td:first-child"),
+                "value": _first_text(row, "td:last-child"),
+            }
+            for row in doc.css("#productOverview_feature_div tr")
+            if _first_text(row, "td:first-child")
+        ],
+        "seller": (
+            {
+                "id": seller_id,
+                "name": seller_name,
+                "url": _absolute(domain, seller_href),
+            }
+            if (seller_link is not None or seller_name)
+            else None
+        ),
+        "bestsellerRanks": ranks,
+        "isAmazonChoice": _one(doc, "#acBadge_feature_div, .ac-badge-wrapper")
+        is not None,
+        "amazonChoiceText": _first_text(doc, "#acBadge_feature_div, .ac-badge-wrapper"),
+        "reviewsLink": _absolute(domain, reviews_link),
+        "hasReviews": bool(local_reviews or foreign_reviews),
+        "productPageReviews": local_reviews,
+        "productPageReviewsFromOtherCountries": foreign_reviews,
+    }
+
+
+def parse_search_page(
+    html: str, *, page: int = 1, domain: str = "www.amazon.com"
+) -> list[dict[str, Any]]:
+    """Parse product cards from a search or category page."""
+    doc = Adaptor(html)
+    cards: list[dict[str, Any]] = []
+    for position, node in enumerate(
+        doc.css(
+            "[data-component-type='s-search-result'][data-asin], [data-asin].s-result-item"
+        ),
+        start=1,
+    ):
+        asin = (_attr(node, "data-asin") or "").upper()
+        if not _ASIN_RE.match(asin):
+            continue
+        href = _first_attr(node, "href", "h2 a", "a.a-link-normal")
+        title = _first_text(node, "h2", "h2 span", ".a-size-base-plus")
+        cards.append(
+            {
+                "asin": asin,
+                "originalAsin": asin,
+                "title": title,
+                "url": _absolute(domain, href) or f"https://{domain}/dp/{asin}",
+                "price": _price(_first_text(node, ".a-price .a-offscreen")),
+                "stars": _float(_first_text(node, ".a-icon-alt")),
+                "reviewsCount": _integer(
+                    _first_text(node, "[aria-label$='ratings']", ".s-underline-text")
+                ),
+                "thumbnailImage": _first_attr(node, "src", ".s-image"),
+                "categoryPageData": {
+                    "position": position,
+                    "page": page,
+                    "isSponsored": "sponsored" in (_text(node) or "").lower(),
+                    "isBestSeller": _one(node, ".a-badge-text, .s-badge-text")
+                    is not None,
+                },
+            }
+        )
+    return cards
+
+
+def parse_aod_offers(
+    html: str, *, domain: str = "www.amazon.com"
+) -> list[dict[str, Any]]:
+    """Parse rows from the public all-offers panel."""
+    doc = Adaptor(html)
+    offers: list[dict[str, Any]] = []
+    for position, node in enumerate(
+        doc.css("#aod-offer, .aod-information-block"), start=1
+    ):
+        seller_link = _one(
+            node, "#aod-offer-soldBy a[href*='seller='], a[href*='/sp?']"
+        )
+        href = _attr(seller_link, "href")
+        query = parse_qs(urlparse(href or "").query)
+        seller_id = (query.get("seller") or [None])[0]
+        offers.append(
+            {
+                "position": position,
+                "price": _price(_first_text(node, ".a-price .a-offscreen")),
+                "condition": _first_text(
+                    node, "#aod-offer-heading", ".aod-information-block"
+                ),
+                "delivery": _first_text(node, "#mir-layout-DELIVERY_BLOCK"),
+                "seller": {
+                    "id": seller_id,
+                    "name": _text(seller_link)
+                    or _first_text(node, "#aod-offer-soldBy .a-color-base"),
+                    "url": _absolute(domain, href),
+                },
+                "isPinnedOffer": _one(node, "#aod-pinned-offer, .aod-pinned-offer")
+                is not None,
+            }
+        )
+    return offers
+
+
+def parse_seller(
+    html: str, *, seller_id: str | None = None, domain: str = "www.amazon.com"
+) -> dict[str, Any]:
+    """Parse the public summary of a seller profile."""
+    doc = Adaptor(html)
+    return {
+        "id": seller_id,
+        "name": _first_text(doc, "#sellerName", "#seller-profile-container h1", "h1"),
+        "url": f"https://{domain}/sp?seller={seller_id}" if seller_id else None,
+        "reviewsCount": _integer(
+            _first_text(doc, "#seller-feedback-summary", "#feedback-summary-table")
+        ),
+        "averageRating": _float(
+            _first_text(doc, "#feedback-summary-table .a-icon-alt", ".a-icon-star")
+        ),
+    }
+
+
+def parse_bestsellers_page(
+    html: str, *, page: int = 1, domain: str = "www.amazon.com"
+) -> list[dict[str, Any]]:
+    """Parse ranked products from a best-sellers page."""
+    doc = Adaptor(html)
+    items: list[dict[str, Any]] = []
+    nodes = doc.css("#gridItemRoot, .zg-grid-general-faceout, [id^='p13n-asin-index-']")
+    for fallback_rank, node in enumerate(nodes, start=1):
+        href = _first_attr(node, "href", "a[href*='/dp/']", "a[href*='/gp/product/']")
+        asin = _asin_from_href(href) or (_attr(node, "data-asin") or None)
+        if not asin or not _ASIN_RE.match(asin):
+            continue
+        rank = _integer(_first_text(node, ".zg-bdg-text")) or fallback_rank
+        items.append(
+            {
+                "asin": asin,
+                "originalAsin": asin,
+                "title": _first_text(
+                    node, "._cDEzb_p13n-sc-css-line-clamp-3_g3dy1", "img"
+                ),
+                "url": _absolute(domain, href) or f"https://{domain}/dp/{asin}",
+                "price": _price(_first_text(node, ".a-price .a-offscreen")),
+                "stars": _float(_first_text(node, ".a-icon-alt")),
+                "thumbnailImage": _first_attr(node, "src", "img"),
+                "bestsellerPageData": {"rank": rank, "page": page},
+            }
+        )
+    return items

surfsense_backend/app/proprietary/platforms/amazon/schemas.py

@@ -0,0 +1,245 @@
+# ruff: noqa: N815
+"""Input/output models for the Amazon Product Scraper.
+
+The skeleton accepts the full input surface; output fields the implementation
+does not source yet are emitted as ``None``/``[]`` so parity is additive —
+exactly like the YouTube / Maps / Google Search models.
+
+Outputs use ``extra="allow"`` on purpose: it lets the output shape grow without
+breaking existing consumers. Only a small set of stable, always-populated nested
+objects (``price``, ``starsBreakdown``, ``seller``, ``visitStoreLink``) are
+typed; the sprawling, layout-volatile sections (A+ content, brand story,
+attribute tables, offers, on-page reviews, ...) stay loose ``dict``/``list`` and
+lean on ``extra="allow"`` rather than pinning a shape the parsers haven't
+verified against live HTML yet.
+
+Public, anonymous data only: there is no auth/login field on the input surface.
+Deep review pagination is login-gated on today's Amazon and is out of scope; the
+on-page ``productPageReviews`` are the only reviews modeled here.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+# --------------------------------------------------------------------------- #
+# Typed nested objects (the stable, always-populated ones).                   #
+# --------------------------------------------------------------------------- #
+
+
+class Price(BaseModel):
+    """A monetary amount as Amazon renders it (``price``, ``listPrice``,
+    ``shippingPrice``, and the ``min``/``max`` of ``priceRange``)."""
+
+    model_config = ConfigDict(extra="allow")
+
+    value: float | None = None
+    currency: str | None = None
+
+
+class StarsBreakdown(BaseModel):
+    """Per-star rating distribution (fractions summing to ~1.0).
+
+    Amazon's keys start with a digit (``5star`` ... ``1star``), which is not a
+    valid Python identifier, so each field is aliased. ``populate_by_name`` lets
+    tests construct it either way; ``to_output`` serializes ``by_alias`` so the
+    wire shape retains the digit-prefixed keys.
+    """
+
+    model_config = ConfigDict(extra="allow", populate_by_name=True)
+
+    five_star: float | None = Field(default=None, alias="5star")
+    four_star: float | None = Field(default=None, alias="4star")
+    three_star: float | None = Field(default=None, alias="3star")
+    two_star: float | None = Field(default=None, alias="2star")
+    one_star: float | None = Field(default=None, alias="1star")
+
+
+class Seller(BaseModel):
+    """The featured-offer seller stamped on the product item."""
+
+    model_config = ConfigDict(extra="allow")
+
+    id: str | None = None
+    name: str | None = None
+    url: str | None = None
+    reviewsCount: int | None = None
+    averageRating: float | None = None
+
+
+class VisitStoreLink(BaseModel):
+    """The brand-store link (``visitStoreLink``)."""
+
+    model_config = ConfigDict(extra="allow")
+
+    text: str | None = None
+    url: str | None = None
+
+
+# --------------------------------------------------------------------------- #
+# Input surface.                                                              #
+# --------------------------------------------------------------------------- #
+
+
+class AmazonScrapeInput(BaseModel):
+    """Full input surface for the Amazon Product Scraper.
+
+    ``categoryOrProductUrls`` mixes category/search, product, bestsellers, and
+    shortened URLs; ``proxyCountry`` defaults to ``AUTO_SELECT_PROXY_COUNTRY``
+    (derived from each URL's domain); ``scrapeProductDetails`` defaults on (deep
+    scrape). ``extra="allow"`` keeps a verbatim payload valid even for add-ons
+    this scraper does not model.
+    """
+
+    model_config = ConfigDict(extra="allow")
+
+    # Discovery (required)
+    categoryOrProductUrls: list[dict] = Field(min_length=1)
+
+    # Result caps
+    maxItemsPerStartUrl: int | None = Field(default=None, ge=0)
+    maxSearchPagesPerStartUrl: int = Field(default=9999, ge=1)
+    maxProductVariantsAsSeparateResults: int = Field(default=0, ge=0)
+    maxOffers: int = Field(default=0, ge=0)
+
+    # Localization
+    language: str | None = None
+    proxyCountry: str = "AUTO_SELECT_PROXY_COUNTRY"
+    countryCode: str | None = None
+    zipCode: str | None = None
+    locationDeliverableRoutes: list[str] | None = Field(
+        default_factory=lambda: ["PRODUCT", "SEARCH", "OFFERS"]
+    )
+
+    # Feature toggles
+    scrapeSellers: bool = False
+    useCaptchaSolver: bool = False
+    scrapeProductVariantPrices: bool = False
+    scrapeProductDetails: bool | None = True
+
+
+# --------------------------------------------------------------------------- #
+# Output item.                                                                #
+# --------------------------------------------------------------------------- #
+
+
+class ProductItem(BaseModel):
+    """Amazon Product Scraper output item (one per product).
+
+    Unsourced fields default to ``None``/``[]``; ``extra="allow"`` keeps the
+    contract open so later milestones add fields without breaking consumers.
+    """
+
+    model_config = ConfigDict(extra="allow", populate_by_name=True)
+
+    # Identity / core
+    title: str | None = None
+    url: str | None = None
+    asin: str | None = None
+    originalAsin: str | None = None
+    brand: str | None = None
+    author: str | None = None
+    price: Price | None = None
+    listPrice: Price | None = None
+    shippingPrice: Price | None = None
+    inStock: bool | None = None
+    inStockText: str | None = None
+    delivery: str | None = None
+    fastestDelivery: str | None = None
+    condition: str | None = None
+
+    # Ratings / social
+    stars: float | None = None
+    starsBreakdown: StarsBreakdown | None = None
+    reviewsCount: int | None = None
+    answeredQuestions: int | None = None
+    aiReviewsSummary: dict | None = None
+    monthlyPurchaseVolume: str | None = None
+
+    # Content
+    breadCrumbs: str | None = None
+    description: str | None = None
+    features: list[str] = Field(default_factory=list)
+    sustainabilityFeatures: list[dict] = Field(default_factory=list)
+    videosCount: int | None = None
+    visitStoreLink: VisitStoreLink | None = None
+    thumbnailImage: str | None = None
+    galleryThumbnails: list[str] = Field(default_factory=list)
+    highResolutionImages: list[str] = Field(default_factory=list)
+    importantInformation: dict | None = None
+    bookDescription: str | None = None
+    aPlusContent: dict | None = None
+    brandStory: dict | None = None
+    productComparison: dict | None = None
+
+    # Commerce
+    returnPolicy: str | None = None
+    support: str | None = None
+    priceRange: dict | None = None
+    variantAsins: list[str] = Field(default_factory=list)
+    variantDetails: list[dict] = Field(default_factory=list)
+    variantAttributes: list[dict] = Field(default_factory=list)
+    attributes: list[dict] = Field(default_factory=list)
+    attributesMapped: dict | None = None
+    productOverview: list[dict] = Field(default_factory=list)
+    manufacturerAttributes: list[dict] = Field(default_factory=list)
+    seller: Seller | None = None
+    bestsellerRanks: list[dict] = Field(default_factory=list)
+    isAmazonChoice: bool | None = None
+    amazonChoiceText: str | None = None
+    offers: list[dict] = Field(default_factory=list)
+
+    # Reviews (public, on-page only)
+    reviewsLink: str | None = None
+    hasReviews: bool | None = None
+    productPageReviews: list[dict] = Field(default_factory=list)
+    productPageReviewsFromOtherCountries: list[dict] = Field(default_factory=list)
+
+    # Provenance / meta
+    locationText: str | None = None
+    unNormalizedProductUrl: str | None = None
+    loadedCountryCode: str | None = None
+    categoryPageData: dict | None = None
+    bestsellerPageData: dict | None = None
+    input: str | None = None
+
+    def to_output(self) -> dict[str, Any]:
+        """Serialize to the flat dict output shape (keeps extras, aliases).
+
+        ``by_alias`` restores digit-prefixed keys like ``starsBreakdown.5star``;
+        ``exclude_none=False`` keeps unsourced keys present so consumers never
+        break on a missing field.
+        """
+        return self.model_dump(by_alias=True, exclude_none=False)
+
+
+# --------------------------------------------------------------------------- #
+# Error item (the failure model — pushed into the stream, not raised).        #
+# --------------------------------------------------------------------------- #
+
+ErrorCode = Literal[
+    "invalid_url",
+    "invalid_input",
+    "product_not_found",
+    "shortened_url_invalid",
+    "bestsellers_category_not_found",
+    "no_results_found",
+]
+
+
+class ErrorItem(BaseModel):
+    """A per-input failure emitted into the dataset instead of a normal item.
+
+    Consumers tell error items apart from products by the presence of ``error``.
+    ``invalid_input`` additionally terminates the run (enforced in later
+    milestones); all other codes are per-input and non-fatal.
+    """
+
+    model_config = ConfigDict(extra="allow")
+
+    error: ErrorCode
+    errorDescription: str | None = None
+    input: str | None = None
+    url: str | None = None

surfsense_backend/app/proprietary/platforms/amazon/scraper.py

@@ -0,0 +1,541 @@
+"""Orchestrate public Amazon product discovery and enrichment.
+
+The streaming core dispatches each start URL to a product, search, best-sellers,
+or shortened-link flow. Network and parsing concerns remain isolated in their
+own modules so markup changes and retry policy can be tested independently.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import AsyncIterator
+from typing import Any
+from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
+
+from .fetch import (
+    fetch_aod_html,
+    fetch_page,
+    fetch_seller_html,
+    gather_bounded,
+    get_location_session,
+    resolve_shortlink,
+    should_localize,
+)
+from .locale import accept_language_for, proxy_country_for
+from .parsers import (
+    parse_aod_offers,
+    parse_bestsellers_page,
+    parse_product,
+    parse_search_page,
+    parse_seller,
+)
+from .schemas import AmazonScrapeInput, ErrorItem, ProductItem
+from .url_resolver import ResolvedUrl, resolve_url
+
+__all__ = ["iter_products", "scrape_products"]
+
+_DETAIL_CONCURRENCY = 8
+_SEARCH_PAGE_LIMIT = 7
+_DEFAULT_ITEMS_PER_START_URL = 100
+
+
+def _error(
+    code: str, description: str, *, input_url: str | None, url: str | None = None
+) -> dict[str, Any]:
+    return ErrorItem(
+        error=code,
+        errorDescription=description,
+        input=input_url,
+        url=url or input_url,
+    ).model_dump()
+
+
+def _buybox_offer(fields: dict[str, Any]) -> dict[str, Any] | None:
+    """Synthesize a single offer from the PDP buy box.
+
+    Amazon serves the All-Offers-Display panel as a JS-only modal for many
+    (especially US) ASINs, so the AOD ajax endpoint 404s. Fall back to the
+    featured buy-box winner already parsed from the product page.
+    """
+    price = fields.get("price")
+    seller = fields.get("seller")
+    seller = seller if isinstance(seller, dict) else None
+    if not price and not (seller and seller.get("name")):
+        return None
+    return {
+        "position": 1,
+        "price": price,
+        "condition": fields.get("condition") or "New",
+        "delivery": fields.get("delivery"),
+        "seller": seller,
+        "isPinnedOffer": True,
+    }
+
+
+def _page_url(url: str, page: int) -> str:
+    parsed = urlparse(url)
+    query = dict(parse_qsl(parsed.query, keep_blank_values=True))
+    query["page"] = str(page)
+    return urlunparse(parsed._replace(query=urlencode(query)))
+
+
+async def _location_context(
+    resolved: ResolvedUrl, input_model: AmazonScrapeInput, route: str
+) -> tuple[dict[str, str] | None, str | None, str | None, str | None]:
+    if (
+        not input_model.zipCode
+        or not resolved.domain
+        or not should_localize(route, input_model.locationDeliverableRoutes)
+    ):
+        return None, None, None, None
+    session = await get_location_session(
+        resolved.domain,
+        zip_code=input_model.zipCode,
+        country_code=input_model.countryCode,
+        country=proxy_country_for(resolved.marketplace),
+        accept_language=accept_language_for(resolved.marketplace),
+    )
+    if session is None:
+        return None, None, None, None
+    return (
+        session.cookies,
+        session.proxy,
+        session.location_text,
+        session.country_code,
+    )
+
+
+async def _product_flow(
+    resolved: ResolvedUrl,
+    input_model: AmazonScrapeInput,
+    *,
+    provenance: dict[str, Any] | None = None,
+    allow_variants: bool = True,
+) -> AsyncIterator[dict[str, Any]]:
+    """Fetch and enrich one product detail page."""
+    asin = resolved.asin
+    if not asin or not resolved.domain:
+        yield _error(
+            "product_not_found",
+            "The product URL did not contain a valid ASIN.",
+            input_url=resolved.url,
+        )
+        return
+
+    cookies, proxy, location_text, loaded_country = await _location_context(
+        resolved, input_model, "PRODUCT"
+    )
+    country = proxy_country_for(resolved.marketplace)
+    accept_language = accept_language_for(resolved.marketplace)
+    response = await fetch_page(
+        resolved.url,
+        cookies=cookies,
+        proxy=proxy,
+        country=country,
+        accept_language=accept_language,
+    )
+    if response is None:
+        yield _error(
+            "product_not_found",
+            "The product page could not be loaded after retrying available proxy exits.",
+            input_url=resolved.url,
+        )
+        return
+    if response.status in {404, 410}:
+        yield _error(
+            "product_not_found",
+            "The product page was not found.",
+            input_url=resolved.url,
+        )
+        return
+    if response.status != 200:
+        yield _error(
+            "product_not_found",
+            f"The product page returned HTTP {response.status}.",
+            input_url=resolved.url,
+        )
+        return
+
+    fields = await asyncio.to_thread(
+        parse_product,
+        response.html,
+        asin=asin,
+        url=response.url,
+        domain=resolved.domain,
+    )
+    if not fields.get("title"):
+        yield _error(
+            "product_not_found",
+            "The response did not contain a recognizable product.",
+            input_url=resolved.url,
+            url=response.url,
+        )
+        return
+    fields.update(provenance or {})
+    fields["input"] = resolved.url
+    fields["unNormalizedProductUrl"] = resolved.url
+    fields["locationText"] = location_text
+    fields["loadedCountryCode"] = loaded_country
+
+    if input_model.maxOffers > 0:
+        offer_cookies, offer_proxy, _, _ = await _location_context(
+            resolved, input_model, "OFFERS"
+        )
+        offers_html = await fetch_aod_html(
+            asin,
+            resolved.domain,
+            cookies=offer_cookies or cookies,
+            proxy=offer_proxy or proxy,
+            country=country,
+            accept_language=accept_language,
+        )
+        offers = (
+            parse_aod_offers(offers_html, domain=resolved.domain) if offers_html else []
+        )
+        if not offers:
+            buybox = _buybox_offer(fields)
+            offers = [buybox] if buybox else []
+        if offers:
+            fields["offers"] = offers[: input_model.maxOffers]
+
+    if input_model.scrapeSellers:
+        seller_ids: list[str] = []
+        featured = fields.get("seller")
+        if isinstance(featured, dict) and featured.get("id"):
+            seller_ids.append(featured["id"])
+        for offer in fields.get("offers") or []:
+            seller = offer.get("seller") if isinstance(offer, dict) else None
+            if isinstance(seller, dict) and seller.get("id"):
+                seller_ids.append(seller["id"])
+        seller_ids = list(dict.fromkeys(seller_ids))
+
+        async def load_seller(seller_id: str) -> dict[str, Any] | None:
+            html = await fetch_seller_html(
+                seller_id,
+                resolved.domain,
+                cookies=cookies,
+                proxy=proxy,
+                country=country,
+                accept_language=accept_language,
+            )
+            return (
+                parse_seller(html, seller_id=seller_id, domain=resolved.domain)
+                if html
+                else None
+            )
+
+        sellers = await gather_bounded(
+            [
+                lambda seller_id=seller_id: load_seller(seller_id)
+                for seller_id in seller_ids
+            ],
+            concurrency=_DETAIL_CONCURRENCY,
+        )
+        by_id = {
+            seller["id"]: seller for seller in sellers if seller and seller.get("id")
+        }
+        if featured and featured.get("id") in by_id:
+            fields["seller"] = by_id[featured["id"]]
+        for offer in fields.get("offers") or []:
+            seller = offer.get("seller")
+            if isinstance(seller, dict) and seller.get("id") in by_id:
+                offer["seller"] = by_id[seller["id"]]
+
+    variant_asins = [
+        value for value in fields.get("variantAsins") or [] if value != asin
+    ]
+    variant_cap = (
+        min(len(variant_asins), input_model.maxProductVariantsAsSeparateResults)
+        if allow_variants
+        else 0
+    )
+    price_cap = (
+        len(variant_asins)
+        if allow_variants and input_model.scrapeProductVariantPrices
+        else 0
+    )
+    fetch_count = max(variant_cap, price_cap)
+    variant_items: list[dict[str, Any]] = []
+    if fetch_count:
+
+        async def load_variant(variant_asin: str) -> list[dict[str, Any]]:
+            variant_url = f"https://{resolved.domain}/dp/{variant_asin}"
+            variant_resolved = resolve_url(variant_url)
+            if variant_resolved is None:
+                return []
+            return [
+                item
+                async for item in _product_flow(
+                    variant_resolved, input_model, allow_variants=False
+                )
+            ]
+
+        batches = await gather_bounded(
+            [
+                lambda variant_asin=variant_asin: load_variant(variant_asin)
+                for variant_asin in variant_asins[:fetch_count]
+            ],
+            concurrency=_DETAIL_CONCURRENCY,
+        )
+        variant_items = [
+            item for batch in batches for item in batch if "error" not in item
+        ]
+        if input_model.scrapeProductVariantPrices:
+            prices = {
+                item.get("asin"): item.get("price")
+                for item in variant_items
+                if item.get("asin")
+            }
+            details = fields.get("variantDetails") or [
+                {"asin": variant_asin} for variant_asin in variant_asins
+            ]
+            for detail in details:
+                if detail.get("asin") in prices:
+                    detail["price"] = prices[detail["asin"]]
+            fields["variantDetails"] = details
+
+    yield ProductItem(**fields).to_output()
+    for item in variant_items[:variant_cap]:
+        item["originalAsin"] = asin
+        yield item
+
+
+async def _search_flow(
+    resolved: ResolvedUrl, input_model: AmazonScrapeInput
+) -> AsyncIterator[dict[str, Any]]:
+    """Page through search results and optionally fetch product details."""
+    if not resolved.domain:
+        yield _error(
+            "no_results_found", "The search domain was invalid.", input_url=resolved.url
+        )
+        return
+    cap = (
+        input_model.maxItemsPerStartUrl
+        if input_model.maxItemsPerStartUrl is not None
+        else _DEFAULT_ITEMS_PER_START_URL
+    )
+    max_pages = min(input_model.maxSearchPagesPerStartUrl, _SEARCH_PAGE_LIMIT)
+    cookies, proxy, _, _ = await _location_context(resolved, input_model, "SEARCH")
+    country = proxy_country_for(resolved.marketplace)
+    accept_language = accept_language_for(resolved.marketplace)
+    seen: set[str] = set()
+    emitted = 0
+    for page in range(1, max_pages + 1):
+        html_response = await fetch_page(
+            _page_url(resolved.url, page),
+            cookies=cookies,
+            proxy=proxy,
+            country=country,
+            accept_language=accept_language,
+        )
+        cards = (
+            await asyncio.to_thread(
+                parse_search_page,
+                html_response.html,
+                page=page,
+                domain=resolved.domain,
+            )
+            if html_response is not None and html_response.status == 200
+            else []
+        )
+        cards = [
+            card for card in cards if card.get("asin") and card["asin"] not in seen
+        ]
+        for card in cards:
+            seen.add(card["asin"])
+        if not cards:
+            if page == 1:
+                yield _error(
+                    "no_results_found",
+                    "The search page did not contain any products.",
+                    input_url=resolved.url,
+                )
+            return
+        cards = cards[: max(0, cap - emitted)]
+        if input_model.scrapeProductDetails is False:
+            for card in cards:
+                card["input"] = resolved.url
+                yield ProductItem(**card).to_output()
+                emitted += 1
+        else:
+
+            async def load_card(card: dict[str, Any]) -> list[dict[str, Any]]:
+                product = resolve_url(card["url"])
+                if product is None:
+                    return []
+                provenance = {"categoryPageData": card["categoryPageData"]}
+                return [
+                    item
+                    async for item in _product_flow(
+                        product, input_model, provenance=provenance
+                    )
+                ]
+
+            batches = await gather_bounded(
+                [lambda card=card: load_card(card) for card in cards],
+                concurrency=_DETAIL_CONCURRENCY,
+            )
+            for batch in batches:
+                for item in batch:
+                    if "error" not in item and emitted >= cap:
+                        return
+                    yield item
+                    if "error" not in item:
+                        emitted += 1
+        if emitted >= cap:
+            return
+
+
+async def _bestsellers_flow(
+    resolved: ResolvedUrl, input_model: AmazonScrapeInput
+) -> AsyncIterator[dict[str, Any]]:
+    """Page through a best-sellers category and emit ranked products."""
+    if not resolved.domain:
+        yield _error(
+            "bestsellers_category_not_found",
+            "The best-sellers domain was invalid.",
+            input_url=resolved.url,
+        )
+        return
+    cap = (
+        input_model.maxItemsPerStartUrl
+        if input_model.maxItemsPerStartUrl is not None
+        else _DEFAULT_ITEMS_PER_START_URL
+    )
+    country = proxy_country_for(resolved.marketplace)
+    accept_language = accept_language_for(resolved.marketplace)
+    seen: set[str] = set()
+    emitted = 0
+    for page in range(1, min(input_model.maxSearchPagesPerStartUrl, 2) + 1):
+        response = await fetch_page(
+            _page_url(resolved.url, page),
+            country=country,
+            accept_language=accept_language,
+        )
+        cards = (
+            await asyncio.to_thread(
+                parse_bestsellers_page,
+                response.html,
+                page=page,
+                domain=resolved.domain,
+            )
+            if response is not None and response.status == 200
+            else []
+        )
+        cards = [
+            card for card in cards if card.get("asin") and card["asin"] not in seen
+        ]
+        for card in cards:
+            seen.add(card["asin"])
+        if not cards:
+            if page == 1:
+                yield _error(
+                    "bestsellers_category_not_found",
+                    "The page did not contain a recognizable best-sellers category.",
+                    input_url=resolved.url,
+                )
+            return
+        cards = cards[: max(0, cap - emitted)]
+        if input_model.scrapeProductDetails is False:
+            for card in cards:
+                card["input"] = resolved.url
+                yield ProductItem(**card).to_output()
+                emitted += 1
+        else:
+
+            async def load_card(card: dict[str, Any]) -> list[dict[str, Any]]:
+                product = resolve_url(card["url"])
+                if product is None:
+                    return []
+                return [
+                    item
+                    async for item in _product_flow(
+                        product,
+                        input_model,
+                        provenance={"bestsellerPageData": card["bestsellerPageData"]},
+                    )
+                ]
+
+            for batch in await gather_bounded(
+                [lambda card=card: load_card(card) for card in cards],
+                concurrency=_DETAIL_CONCURRENCY,
+            ):
+                for item in batch:
+                    if "error" not in item and emitted >= cap:
+                        return
+                    yield item
+                    if "error" not in item:
+                        emitted += 1
+        if emitted >= cap:
+            return
+
+
+async def _shortened_flow(
+    resolved: ResolvedUrl, input_model: AmazonScrapeInput
+) -> AsyncIterator[dict[str, Any]]:
+    """Resolve a shortened link and dispatch its final Amazon URL."""
+    target = await resolve_shortlink(resolved.url)
+    final = resolve_url(target) if target else None
+    if final is None or final.kind == "shortened":
+        yield _error(
+            "shortened_url_invalid",
+            "The shortened link did not resolve to a supported Amazon page.",
+            input_url=resolved.url,
+            url=target,
+        )
+        return
+    async for item in _FLOWS[final.kind](final, input_model):
+        if "input" in item:
+            item["input"] = resolved.url
+        yield item
+
+
+_FLOWS = {
+    "product": _product_flow,
+    "search": _search_flow,
+    "bestsellers": _bestsellers_flow,
+    "shortened": _shortened_flow,
+}
+
+
+async def iter_products(
+    input_model: AmazonScrapeInput,
+) -> AsyncIterator[dict[str, Any]]:
+    """Yield product items for every start URL.
+
+    Each ``categoryOrProductUrls`` entry is classified and dispatched to its
+    per-kind flow. A URL that cannot be recognized as an Amazon product /
+    search / bestsellers / shortened link yields an ``invalid_url`` error item
+    (the failure model is in-stream error items, not exceptions).
+    """
+    for entry in input_model.categoryOrProductUrls:
+        url = entry.get("url") if isinstance(entry, dict) else None
+        resolved = resolve_url(url) if url else None
+        if resolved is None:
+            yield _error(
+                "invalid_url",
+                "Start URL was malformed or not a recognized Amazon URL.",
+                input_url=url,
+            )
+            continue
+        async for item in _FLOWS[resolved.kind](resolved, input_model):
+            yield item
+
+
+async def scrape_products(
+    input_model: AmazonScrapeInput, *, limit: int | None = None
+) -> list[dict[str, Any]]:
+    """Collect :func:`iter_products` into a list, honoring an optional ``limit``.
+
+    ``limit`` is a request-time policy guard (used by the route/capability), NOT
+    a ceiling in the streaming core.
+    """
+    from app.capabilities.core.progress import emit_progress
+
+    results: list[dict[str, Any]] = []
+    async for item in iter_products(input_model):
+        results.append(item)
+        emit_progress("scraping", current=len(results), total=limit, unit="product")
+        if limit is not None and len(results) >= limit:
+            break
+    return results

surfsense_backend/app/proprietary/platforms/amazon/url_resolver.py

@@ -0,0 +1,98 @@
+"""Classify an Amazon start URL and extract its identifiers.
+
+Covers the ``categoryOrProductUrls`` shapes accepted on input: product
+pages (``/dp/ASIN``, ``/gp/product/ASIN``), search / category pages (``/s?k=``,
+``/s?bbn=``, ``/s?rh=``), bestsellers pages (``/zgbs/`` / ``/gp/bestsellers/``),
+and shortened links (``a.co`` / ``amzn.to`` / ``amzn.eu``, which need a network
+redirect to resolve — classified here, resolved later in the fetch layer).
+
+Pure, no I/O. The ``marketplace`` suffix (``com``, ``de``, ``co.uk``, ...) is
+derived from the host so the fetch layer can default the proxy country and UI
+language without a separate geocoding step.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Literal
+from urllib.parse import parse_qs, urlparse
+
+ResolvedKind = Literal["product", "search", "bestsellers", "shortened"]
+
+_SHORTLINK_HOSTS = ("a.co", "amzn.to", "amzn.eu")
+
+# ASIN: a 10-char uppercase alphanumeric id, carried in the canonical product
+# path forms (``/dp/``, ``/gp/product/``, ``/product/``) or a ``?asin=`` query.
+_ASIN_PATH_RE = re.compile(r"/(?:dp|gp/product|product|d)/([A-Z0-9]{10})(?:[/?]|$)")
+_ASIN_QUERY_RE = re.compile(r"[?&]asin=([A-Z0-9]{10})", re.IGNORECASE)
+
+# Search/category pages carry their query in these params even without a /s path.
+_SEARCH_QUERY_KEYS = ("k", "bbn", "rh")
+
+
+@dataclass(frozen=True)
+class ResolvedUrl:
+    kind: ResolvedKind
+    url: str
+    asin: str | None = None
+    domain: str | None = None
+    marketplace: str | None = None  # TLD suffix, e.g. "com", "de", "co.uk"
+
+
+def extract_asin(url: str) -> str | None:
+    """Pull the 10-char ASIN out of an Amazon product URL, if present."""
+    match = _ASIN_PATH_RE.search(url)
+    if match:
+        return match.group(1)
+    match = _ASIN_QUERY_RE.search(url)
+    return match.group(1).upper() if match else None
+
+
+def _marketplace_from_host(host: str) -> str | None:
+    """The Amazon TLD suffix (``com``, ``de``, ``co.uk``, ...), or ``None``.
+
+    Strips common subdomain prefixes and the ``amazon`` label, returning the
+    remainder (e.g. ``www.amazon.co.jp`` -> ``co.jp``).
+    """
+    host = host.lower()
+    for prefix in ("www.", "smile."):
+        if host.startswith(prefix):
+            host = host[len(prefix) :]
+    if not host.startswith("amazon."):
+        return None
+    return host[len("amazon.") :] or None
+
+
+def resolve_url(url: str) -> ResolvedUrl | None:
+    """Classify an Amazon start URL into a scrape job, or ``None`` if unrecognized."""
+    parsed = urlparse(url)
+    host = (parsed.hostname or "").lower()
+    path = parsed.path or ""
+
+    # Shortened links resolve to their real URL later, in the fetch layer.
+    if host in _SHORTLINK_HOSTS:
+        return ResolvedUrl("shortened", url)
+
+    if "amazon." not in host:
+        return None
+
+    marketplace = _marketplace_from_host(host)
+
+    # Bestsellers before search: a /zgbs page has no /s path but is its own kind.
+    if "/zgbs/" in path or path.startswith("/gp/bestsellers"):
+        return ResolvedUrl("bestsellers", url, domain=host, marketplace=marketplace)
+
+    # Product: canonical /dp/ (etc.) path with an ASIN.
+    asin = extract_asin(url)
+    if asin is not None:
+        return ResolvedUrl(
+            "product", url, asin=asin, domain=host, marketplace=marketplace
+        )
+
+    # Search / category: the /s path, or any search-carrying query param.
+    query = parse_qs(parsed.query)
+    if path.startswith("/s") or any(k in query for k in _SEARCH_QUERY_KEYS):
+        return ResolvedUrl("search", url, domain=host, marketplace=marketplace)
+
+    return None

surfsense_backend/app/proprietary/platforms/google_search/README.md

@@ -27,22 +27,75 @@ traffic" wall, and the IPs that pass serve a JavaScript shell whose organic
 results only materialize after the page's JS runs. So `fetch.py` needs both a
 **non-blocked IP** and a **real browser render**, on the same IP:
 
-1. Reuse the last known-good sticky IP if it still passes a cheap re-precheck;
-   otherwise race prechecks on several fresh sticky IPs at once (DataImpulse
-   maps ports → sessions) and take the first that passes,
+1. Take a **warm sticky IP from the pool** (`_pool`): reuse an already-solved IP
+   under its soft per-IP concurrency cap — concurrent fetches fan out across the
+   pool's IPs (least-loaded first) instead of funneling onto one hot IP (which
+   Google re-walls). If the pool is below target, grow it: first try to **adopt**
+   an IP another process already solved (see the shared store below), else race
+   prechecks on several fresh sticky IPs at once (DataImpulse maps ports →
+   sessions) and solve on the first that passes. If the pool is full and every
+   IP is busy, wait briefly for a slot rather than burn a fresh solve,
 2. render on that IP using a **shared long-lived browser** (launched once per
    process, per layout; each fetch only opens a fresh context carrying its
-   vetted proxy) — during which the render clicks the AI Overview's "Show
-   more" clamp and the initially-served People-Also-Ask questions open (all
-   clicks fired first, then one shared wait) so their content lands in the DOM;
-3. retry on fresh IPs until one returns the results container.
+   vetted proxy). Renders drop image/font/media/stylesheet requests
+   (`disable_resources`; the parsed DOM is text/attributes only) — except AI
+   Mode, whose answer streams in. During the render it clicks the AI Overview's
+   "Show more" clamp and the initially-served People-Also-Ask questions open
+   (all clicks fired first, then one shared wait) so their content lands in the
+   DOM;
+3. on success (re)admit the IP to the pool; a walled render evicts it (and drops
+   its exemption fleet-wide). Retry until a render returns the results container
+   or the per-fetch deadline / IP budget runs out.
 
-A warm fetch (browser up, IP cached) runs ~8 s; the first fetch of a process
-also pays the ~5 s Chromium launch and a vetting round. Requires the browser
-tier (patchright Chromium via Scrapling's `AsyncStealthySession`) and a
-residential proxy — set `PROXY_PROVIDER` + `PROXY_URL` (see
-`.env`). Long-running callers can `await fetch.close_sessions()` on shutdown;
-scripts that exit anyway can skip it.
+**The reCAPTCHA wall.** A curl-vetted IP is only *half* the battle: the initial
+`/search` GET returns a 200 JS shell, but the shell's JS reloads `/search` with
+a `sei` token, which Google 302s into a reCAPTCHA-**Enterprise** `/sorry` page.
+This trips *every* real browser (the block is on the browser access pattern, not
+the IP — curl "passes" only because it never runs the reload, and so never gets
+results either). When a solver is configured (`CAPTCHA_SOLVING_ENABLED=TRUE` +
+`CAPTCHA_SOLVER_API_KEY`, see `app/utils/captcha/`), `captcha.py` runs inside the
+render's `page_action`: on landing at `/sorry` it harvests an Enterprise token
+(with the page's dynamic `data-s`) from the solver **egressing through the same
+sticky proxy**, injects it, and submits. Google then issues a
+`GOOGLE_ABUSE_EXEMPTION` cookie, which the fetch seam caches per proxy
+(`_exemption_jar`) and replays via `page_setup` so **one solve unlocks the
+sticky IP for all subsequent queries** on it. That solve is also **published to
+Redis** (`pool_store.py`), so any *other* worker process can adopt the same
+warm IP instead of re-solving — cost then tracks the shared pool size, not the
+worker count (best-effort: no Redis ⇒ each process just keeps its own pool).
+Without a solver configured the stealth tier is unchanged (and
+brand/navigational queries will 429-wall).
+
+Timings (measured): a fetch that must solve runs **~40–110 s and is entirely
+dominated by the solver** — the reCAPTCHA-Enterprise harvest alone measured
+**27–39 s on capsolver** (the configured default; AI-native, tighter spread) and
+37–100 s on 2captcha (pure vendor worker latency, not our overhead). NB the
+sub-10 s figures both vendors advertise are for the plain reCAPTCHA *demo*
+sitekey; Google's Enterprise `/sorry` wall is genuinely harder and slower for
+everyone. Every *subsequent* fetch on that sticky IP reuses the cached exemption
+and runs **~12–16 s** (two proxy navigations for Google's `sei` reload + ~3–4 s
+of block expansion; the vet precheck is skipped, `render_ms` is the whole
+render). The first fetch of a process also pays the ~5 s Chromium launch. So the
+one lever that actually moves the needle is **solving less often** (the pool +
+Redis-shared exemptions already amortize it) — or a faster solver.
+
+**Scale.** Per-process throughput ceiling = `GOOGLE_SEARCH_MAX_CONCURRENT_PAGES`
+÷ warm-render-secs ≈ **4 / 14 s ≈ 17 SERP/min** at the default. Because renders
+drop images/fonts/media, one Chromium holds well more than 4 text contexts, so
+raise `GOOGLE_SEARCH_MAX_CONCURRENT_PAGES` to trade RAM/CPU for throughput
+before adding processes (e.g. 16 → ~69/min/process ⇒ ~7 processes cover 500/min;
+the sim in `scripts/scale_google_search.py` reproduces these numbers offline).
+`GOOGLE_SEARCH_WARM_POOL_TARGET` sizes the shared warm-IP pool: steady-state
+paid solves ≈ pool size (not request count), and it must be large enough that
+fleet-wide per-IP load (local `GOOGLE_SEARCH_IP_MAX_CONCURRENCY` × processes)
+stays gentle enough that Google doesn't re-wall a pool IP. Requires the
+browser tier (patchright Chromium via
+Scrapling's `AsyncStealthySession`) and a residential proxy — set
+`PROXY_PROVIDER` + `PROXY_URL` (see `.env`; **do not** pin a `__cr.<country>`
+suffix — DataImpulse's per-country sub-pools are 429-walled even for the curl
+precheck, which starves vetting; `gl=` in the URL sets result locale instead).
+Long-running callers can `await fetch.close_sessions()` on shutdown; scripts
+that exit anyway can skip it.
 
 ## Scope
 
@@ -98,7 +151,9 @@ the progressive rollout.)
 | `schemas.py`       | Pydantic input/output models mirroring the Apify camelCase spec. `extra="allow"` on outputs keeps the contract open.      |
 | `scraper.py`       | Orchestrator. `iter_serps` dispatches each `queries` line to `_term_flow` / `_url_flow` (+ `_ai_mode_flow` per term).      |
 | `query_builder.py` | Pure: classify `queries` lines, fold advanced filters into search operators, resolve relative dates, build the URL.        |
-| `fetch.py`         | Proxy-vetted two-phase fetch: cheap precheck GET + headless render on a shared sticky IP, retrying across IPs.             |
+| `fetch.py`         | Proxy-vetted two-phase fetch: cheap precheck GET + headless render on a **warm sticky-IP pool** (spread across IPs, per-IP soft cap, grow-to-target), retrying across IPs. Caches per-IP reCAPTCHA exemptions. |
+| `pool_store.py`    | Best-effort Redis cache of solved-IP exemptions so a solve on one worker warms that IP for the whole fleet (adopt/publish/evict); silent no-op without Redis. |
+| `captcha.py`       | Async reCAPTCHA-Enterprise solve for the `/sorry` wall (via the `app.utils.captcha` solver seam — capsolver/2captcha, `enterprise`+`data-s`), run inside the render's `page_action`.  |
 | `parsers.py`       | Rendered SERP HTML → organic / text ads / product ads / related / People-Also-Ask / `resultsTotal` (degrades per-field). |
 
 ## Input semantics (matching Apify)

surfsense_backend/app/proprietary/platforms/google_search/captcha.py

@@ -0,0 +1,177 @@
+"""reCAPTCHA-Enterprise solve for Google's ``/sorry`` interstitial.
+
+Google walls a full browser's JS-driven ``sei`` reload by 302-ing it to
+``/sorry/index`` — a reCAPTCHA **Enterprise** challenge. A curl-vetted IP still
+hits this because the wall is on the *browser access pattern*, not the IP (curl
+never runs the JS reload, so it only ever sees the useless no-JS shell). The
+only way to keep rendering real SERPs on our own browser is to solve that
+challenge.
+
+This runs inside the render's async ``page_action``: when the page lands on
+``/sorry``, it harvests a token from the configured solver (egressing from the
+**same sticky proxy** as the browser, so the token is IP-bound), injects it, and
+submits the form. Google then issues a ``GOOGLE_ABUSE_EXEMPTION`` cookie that
+unlocks real results on that IP — which the fetch seam caches per proxy so one
+solve amortizes across subsequent fetches (see ``fetch._exemption_jar``).
+
+The actual vendor call goes through the shared, in-house solver seam
+(:mod:`app.utils.captcha.solvers`) with ``enterprise=True`` + the page's
+``data-s`` — this module just detects the wall, harvests via that seam egressing
+through the render's own sticky proxy, injects the token, and submits.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import logging
+import re
+import time
+from typing import Any
+
+from app.utils.captcha import CaptchaConfig, solvers
+
+logger = logging.getLogger(__name__)
+
+_LOG = "[google_search][captcha]"
+
+# The /sorry page embeds the widget sitekey and a dynamic `data-s` token that
+# the Enterprise challenge binds to; both are mandatory in the solve request.
+_SITEKEY_RE = re.compile(r'data-sitekey="([\w-]+)"')
+_DATA_S_RE = re.compile(r'data-s="([^"]+)"')
+
+# After submitting the solved /sorry form, Google 302s back to the SERP. We wait
+# only for the results container to attach — NOT for full network idle. A
+# rendered SERP fires telemetry/XHRs that keep the network busy long past the
+# point results are present, so a `networkidle` wait here burned up to its full
+# timeout on every solve for no benefit; the container's presence is the real
+# "we're through the wall" signal.
+_POST_SOLVE_WAIT_MS = 15000
+
+# Process-wide kill switch tripped on unrecoverable solver errors (no balance /
+# bad key). Persists across the per-fetch page_action instances of one run;
+# nothing short of a restart fixes balance/key, so keep hammering off.
+_latched = False
+
+
+def solver_latched() -> bool:
+    return _latched
+
+
+def reset_solver_latch() -> None:
+    """Clear the process latch (test seam / explicit re-enable)."""
+    global _latched
+    _latched = False
+
+
+def _latch(reason: str) -> None:
+    global _latched
+    _latched = True
+    logger.warning("%s solver latched OFF for this process: %s", _LOG, reason)
+
+
+def on_sorry(page: Any) -> bool:
+    """True when the render landed on Google's ``/sorry`` reCAPTCHA wall."""
+    return "/sorry/" in (getattr(page, "url", "") or "")
+
+
+# Set the token into the response textarea and submit the /sorry form. Google's
+# /sorry form posts to /sorry/index and, on a valid token, 302s back to the
+# `continue` URL (the SERP) while setting GOOGLE_ABUSE_EXEMPTION.
+_INJECT_JS = r"""
+(token) => {
+  let ta = document.getElementById('g-recaptcha-response');
+  if (!ta) {
+    ta = document.createElement('textarea');
+    ta.id = 'g-recaptcha-response';
+    ta.name = 'g-recaptcha-response';
+    ta.style.display = 'none';
+    (document.forms[0] || document.body).appendChild(ta);
+  }
+  ta.value = token; ta.innerHTML = token;
+  const form = ta.closest('form') || document.forms[0];
+  if (form) { try { form.requestSubmit ? form.requestSubmit() : form.submit(); } catch (e) { form.submit(); } }
+}
+"""
+
+# Organic-results containers; presence means the exemption unlocked the SERP.
+_RESULTS_SEL = "#search, #rso, div.tF2Cxc"
+
+
+async def solve_sorry(page: Any, proxy_url: str | None, cfg: CaptchaConfig) -> bool:
+    """Solve the ``/sorry`` challenge on ``page`` and submit; return solved.
+
+    Runs inside the async render. On success the page ends on the real SERP and
+    the context holds ``GOOGLE_ABUSE_EXEMPTION`` (the caller caches it). On any
+    failure returns ``False`` and leaves the page on ``/sorry`` so the fetch
+    loop moves to the next IP.
+    """
+    if solver_latched():
+        return False
+    try:
+        html = await page.content()
+    except Exception:
+        return False
+    sk = _SITEKEY_RE.search(html)
+    ds = _DATA_S_RE.search(html)
+    if not sk or not ds:
+        logger.warning("%s /sorry page missing sitekey/data-s; cannot solve", _LOG)
+        return False
+
+    page_url = getattr(page, "url", "") or ""
+    try:
+        user_agent = await page.evaluate("() => navigator.userAgent")
+    except Exception:
+        user_agent = None
+
+    logger.info("%s solving Enterprise reCAPTCHA site=%s", _LOG, sk.group(1)[:12])
+    _t0 = time.perf_counter()
+    try:
+        token = await asyncio.to_thread(
+            solvers.solve,
+            cfg,
+            challenge_type="v2",
+            sitekey=sk.group(1),
+            page_url=page_url,
+            proxy_url=proxy_url,
+            user_agent=user_agent,
+            enterprise=True,
+            data_s=ds.group(1),
+        )
+    except solvers.SolverError as e:
+        # Bad key / no balance / unsupported provider won't fix without a
+        # restart — latch so we stop hammering (and stop paying) this process.
+        _latch(repr(e))
+        return False
+    except Exception as e:  # never let a solver hiccup break the render
+        logger.warning("%s solve error: %s", _LOG, e)
+        return False
+    if not token:
+        return False
+
+    harvest_ms = (time.perf_counter() - _t0) * 1000
+    try:
+        await page.evaluate(_INJECT_JS, token)
+    except Exception as e:
+        logger.warning("%s token injection failed: %s", _LOG, e)
+        return False
+    with contextlib.suppress(Exception):
+        await page.wait_for_selector(_RESULTS_SEL, timeout=_POST_SOLVE_WAIT_MS)
+
+    solved = not on_sorry(page)
+    logger.info(
+        "%s solve %s harvest_ms=%.0f total_ms=%.0f",
+        _LOG,
+        "OK" if solved else "did not clear wall",
+        harvest_ms,
+        (time.perf_counter() - _t0) * 1000,
+    )
+    return solved
+
+
+async def exemption_cookies(page: Any) -> list[dict]:
+    """Snapshot the context's google.com cookies (for the per-proxy jar)."""
+    try:
+        return await page.context.cookies("https://www.google.com")
+    except Exception:
+        return []

surfsense_backend/app/proprietary/platforms/google_search/fetch.py

@@ -33,6 +33,7 @@ from __future__ import annotations
 import asyncio
 import contextlib
 import logging
+import os
 import random
 import sys
 import threading
@@ -41,6 +42,11 @@ from urllib.parse import urlsplit, urlunsplit
 
 from scrapling.fetchers import AsyncFetcher
 
+from app.proprietary.platforms.google_search import (
+    captcha as _captcha,
+    pool_store as _store,
+)
+from app.utils.captcha import captcha_enabled, get_captcha_config
 from app.utils.proxy import get_proxy_url
 
 try:  # browser tier is optional (needs `scrapling[fetchers]` browsers installed)
@@ -55,6 +61,23 @@ logger = logging.getLogger(__name__)
 CONSENT_COOKIES = {"CONSENT": "PENDING+987", "SOCS": "CAESHAgBEhIaAB"}
 _HEADERS = {"Accept-Language": "en-US,en;q=0.9"}
 
+# Context-cookie form of the consent cookies, injected before a render's
+# navigation (page_setup) when captcha solving is on, so the browser skips the
+# EU interstitial exactly like the curl precheck does.
+_CONSENT_CTX = [
+    {"name": k, "value": v, "domain": ".google.com", "path": "/"}
+    for k, v in CONSENT_COOKIES.items()
+]
+
+# GOOGLE_ABUSE_EXEMPTION cookies harvested per sticky proxy after a solve. A hit
+# lets a later render on the same IP skip straight to results (no re-solve),
+# amortizing the one paid solve across the sticky session.
+# ponytail: keyed by the full proxy URL (incl. sticky port). When that port's
+# exit IP rotates the cached cookie goes stale -> the next render re-hits
+# /sorry, re-solves, and overwrites the entry. Self-correcting; the sticky-port
+# space is small so the dict never grows unbounded in practice.
+_exemption_jar: dict[str, list[dict]] = {}
+
 # Gateways whose sticky sessions are selected by destination port. A random port
 # in this range pins one residential IP for the duration of a browser session.
 _STICKY_HOSTS = {"gw.dataimpulse.com"}
@@ -73,11 +96,37 @@ _VET_CONCURRENCY = 4
 # the budget in a couple of seconds.
 _WALLED_ROUND_BACKOFF_S = 3.0
 
-# The sticky IP that most recently served a real SERP; the strongest hint for
-# the next fetch. Re-vetted (cheap) before reuse, dropped on failure.
-# ponytail: a single slot, not a pool — concurrent fetches share (and race)
-# it; worst case a loser re-vets a fresh IP, which is the normal path anyway.
-_last_good_proxy: str | None = None
+# Warm sticky-IP pool. Concurrent fetches spread renders across the pool's IPs
+# instead of funneling every render onto one sticky IP (a single hot residential
+# IP is exactly what Google re-walls), and each pool IP is solved **once** — so
+# steady-state paid solves track the pool size, not the request count. Replaces
+# the old single "_last_good_proxy" slot, which both funneled load and forced a
+# fresh solve whenever it was raced to None.
+#
+# State is touched from two loops (fetch_serp_html on the request loop; the
+# exemption write in page_action on the browser loop), so a small lock guards
+# every mutation. No awaits are held under the lock — every critical section is
+# a handful of dict ops.
+_pool: dict[str, float] = {}  # warm proxy url -> last-used monotonic ts
+_pool_inflight: dict[str, int] = {}  # proxy url -> renders currently pinned to it
+_pool_pending = 0  # fresh vets/solves in flight (growing the pool toward target)
+_pool_lock = threading.Lock()
+
+# Target number of warm IPs to keep. Sized for throughput: it must cover
+# (target SERP/min) / (safe SERP/min per IP), so a fleet serving 500/min across
+# a handful of processes wants this in the low tens. Env-tunable without a code
+# change; per-IP concurrency is a soft cap (exceeded only when the pool is full
+# and every IP is busy, because reusing a warm IP always beats a fresh solve).
+_WARM_POOL_TARGET = int(os.getenv("GOOGLE_SEARCH_WARM_POOL_TARGET", "8"))
+_WARM_IP_MAX_CONCURRENCY = int(os.getenv("GOOGLE_SEARCH_IP_MAX_CONCURRENCY", "2"))
+# When the pool is full and every IP is at its cap, wait this long for a slot to
+# free instead of vetting+solving a fresh IP (this backpressure is what bounds
+# solves to ~pool size under a burst).
+_POOL_WAIT_S = 0.25
+# Absolute per-fetch budget. A cold solve is ~45 s and we may retry a couple of
+# walled IPs, but a fetch that can't get results within this window fails rather
+# than hanging a request forever under saturation.
+_FETCH_DEADLINE_S = float(os.getenv("GOOGLE_SEARCH_FETCH_DEADLINE_S", "180"))
 
 # A usable precheck responds in <1 s; anything slower is a dead/slow sticky IP
 # (seen hanging ~60 s). Abandon it on this deadline so a slow IP costs no more
@@ -172,6 +221,83 @@ async def _vet_fresh_ip(url: str, base: str) -> str | None:
     return winner
 
 
+def _pool_take() -> tuple[str, str | None]:
+    """Decide how the next render gets an IP. Returns one of:
+
+    * ``("reuse", url)`` — an existing warm IP under its soft concurrency cap
+      (its inflight count is already incremented; caller must ``_pool_settle``),
+    * ``("grow", None)`` — pool is below target, caller should vet a fresh IP
+      and solve on it (a pending slot is reserved),
+    * ``("wait", None)`` — pool is full and every IP is busy; caller should wait
+      briefly and retry rather than burn a fresh solve.
+
+    Reuse prefers the least-loaded, least-recently-used warm IP so concurrent
+    renders fan out across the pool instead of piling onto one IP.
+    """
+    global _pool_pending
+    with _pool_lock:
+        best: tuple[str, int, float] | None = None
+        for url, last in _pool.items():
+            n = _pool_inflight.get(url, 0)
+            if n < _WARM_IP_MAX_CONCURRENCY and (
+                best is None or (n, last) < (best[1], best[2])
+            ):
+                best = (url, n, last)
+        if best is not None:
+            _pool_inflight[best[0]] = _pool_inflight.get(best[0], 0) + 1
+            return ("reuse", best[0])
+        if len(_pool) + _pool_pending < _WARM_POOL_TARGET:
+            _pool_pending += 1
+            return ("grow", None)
+        return ("wait", None)
+
+
+def _pool_bind_fresh(url: str) -> None:
+    """Pin the first render onto a just-vetted fresh IP (grow path)."""
+    with _pool_lock:
+        _pool_inflight[url] = _pool_inflight.get(url, 0) + 1
+
+
+def _pool_adopt(url: str) -> None:
+    """Turn a reserved grow slot into an IP adopted from the shared store: it
+    joins the warm pool without a local solve, so release the pending reservation
+    and pin this render onto it."""
+    global _pool_pending
+    with _pool_lock:
+        _pool_pending = max(0, _pool_pending - 1)
+        _pool[url] = time.monotonic()
+        _pool_inflight[url] = _pool_inflight.get(url, 0) + 1
+
+
+def _pool_abort_grow() -> None:
+    """Release a reserved grow slot when the fresh vet found no usable IP."""
+    global _pool_pending
+    with _pool_lock:
+        _pool_pending = max(0, _pool_pending - 1)
+
+
+def _pool_settle(url: str, *, good: bool, grew: bool) -> None:
+    """Finish a render: drop its inflight hold, then admit or evict the IP.
+
+    A good render (re)admits the IP to the warm pool (refreshing its LRU stamp);
+    a walled render evicts it and drops its now-worthless cached exemption.
+    """
+    global _pool_pending
+    with _pool_lock:
+        n = _pool_inflight.get(url, 0) - 1
+        if n <= 0:
+            _pool_inflight.pop(url, None)
+        else:
+            _pool_inflight[url] = n
+        if grew:
+            _pool_pending = max(0, _pool_pending - 1)
+        if good:
+            _pool[url] = time.monotonic()
+        else:
+            _pool.pop(url, None)
+            _exemption_jar.pop(url, None)
+
+
 # People-also-ask answers only load when a question is expanded (clicked).
 # We expand just the initially-served questions (~4); each expansion appends
 # more questions we deliberately leave collapsed, or the loop never ends.
@@ -197,6 +323,7 @@ async def _expand_blocks(page):
     Both are free on pages without the block, and best-effort: a failed click
     just leaves that section collapsed rather than failing the render.
     """
+    _t0 = time.perf_counter()
     clicked = 0
     try:
         more = await page.query_selector(_AIO_SHOW_MORE_SEL)
@@ -218,9 +345,44 @@ async def _expand_blocks(page):
     if clicked:
         # One shared wait while all the answer XHRs land in parallel.
         await page.wait_for_timeout(_PAA_ANSWER_WAIT_MS)
+    logger.info(
+        "[google_search][perf] expand_ms=%.0f clicked=%d",
+        (time.perf_counter() - _t0) * 1000,
+        clicked,
+    )
     return page
 
 
+def _make_page_setup(proxy: str | None):
+    """Per-fetch ``page_setup``: seed consent + any cached exemption for this IP."""
+
+    async def page_setup(page):
+        jar = list(_CONSENT_CTX)
+        cached = _exemption_jar.get(proxy or "")
+        if cached:
+            jar += cached
+        with contextlib.suppress(Exception):
+            await page.context.add_cookies(jar)
+
+    return page_setup
+
+
+def _make_page_action(proxy: str | None, cfg):
+    """Per-fetch ``page_action``: solve the /sorry wall if hit, then expand blocks.
+
+    Overrides the session default (:func:`_expand_blocks`) so it must still call
+    it afterwards. On a successful solve the resulting ``GOOGLE_ABUSE_EXEMPTION``
+    cookie is cached against this sticky proxy for reuse.
+    """
+
+    async def page_action(page):
+        if _captcha.on_sorry(page) and await _captcha.solve_sorry(page, proxy, cfg):
+            _exemption_jar[proxy or ""] = await _captcha.exemption_cookies(page)
+        return await _expand_blocks(page)
+
+    return page_action
+
+
 # Firefox-on-Android UA to make Google serve its mobile lightweight layout.
 # ponytail: the engine underneath is patchright's *Chromium*, so this UA lies
 # about the engine — but empirically it's what gets the mobile layout served
@@ -279,9 +441,16 @@ _session_lock = asyncio.Lock()
 # second concurrent render raised RuntimeError('Maximum page limit (1)
 # reached'); the failure handler then closed the shared browser under the
 # sibling render (the TargetClosedError cascade seen in production when
-# several scrape runs overlap). The pool and this gate are sized together:
-# the gate queues excess renders instead of tripping the pool.
-_MAX_CONCURRENT_PAGES = 4
+# several scrape runs overlap). The pool and this gate are sized together
+# (the session's max_pages is set from this): the gate queues excess renders
+# instead of tripping the pool.
+#
+# This is THE throughput lever: per-process ceiling = this / warm-render-secs
+# (≈ 4/14 s ≈ 17 SERP/min at the default). Renders drop images/fonts/media
+# (`disable_resources`), so a single Chromium can hold well more than 4 text
+# contexts — raise this (env) to trade RAM/CPU for throughput before adding
+# processes. e.g. 16 → ~68/min/process, so ~8 processes cover 500/min.
+_MAX_CONCURRENT_PAGES = int(os.getenv("GOOGLE_SEARCH_MAX_CONCURRENT_PAGES", "4"))
 # Only ever awaited from coroutines running on the browser loop.
 _render_gate = asyncio.Semaphore(_MAX_CONCURRENT_PAGES)
 
@@ -350,7 +519,23 @@ async def _render_on_loop(url: str, proxy: str | None, mobile: bool):
         session = await _get_session(mobile)
         _inflight[session] = _inflight.get(session, 0) + 1
         try:
-            return await session.fetch(url, proxy=proxy)
+            kwargs: dict = {}
+            # Drop image/font/media/stylesheet requests: the SERP DOM we parse is
+            # text/attribute-only, so these are dead weight (a warm render moved
+            # ~2-7 MB per page) that also keeps `network_idle` from settling.
+            # AI Mode streams its answer into the DOM, so leave its resources on
+            # (dropping websockets/media can starve the stream) — detected by the
+            # udm=50 marker its URL always carries.
+            if "udm=50" not in url:
+                kwargs["disable_resources"] = True
+            # Only wire the solver when it's configured AND still viable this
+            # process; otherwise the session's default page_action (expand-only)
+            # runs and the stealth tier is unchanged.
+            if proxy and captcha_enabled() and not _captcha.solver_latched():
+                cfg = get_captcha_config()
+                kwargs["page_setup"] = _make_page_setup(proxy)
+                kwargs["page_action"] = _make_page_action(proxy, cfg)
+            return await session.fetch(url, proxy=proxy, **kwargs)
         finally:
             _inflight[session] -= 1
             if not _inflight[session]:
@@ -374,35 +559,61 @@ async def _render(url: str, proxy: str | None, mobile: bool = False):
 async def fetch_serp_html(url: str, *, mobile: bool = False) -> str | None:
     """Return fully-rendered SERP HTML for ``url``, or ``None`` if unobtainable.
 
-    Reuses the last known-good sticky IP when it still passes the cheap
-    precheck; otherwise races prechecks on fresh sticky IPs and renders on the
-    first that passes. Retries until a render returns real results or the IP
-    budget runs out. Requires the browser tier — without it we cannot get
-    JS-built results. ``mobile`` renders with a phone UA/viewport (the
-    ``mobileResults`` input).
+    Renders on a **warm sticky-IP pool**: it reuses an already-solved IP under
+    its soft concurrency cap (spreading concurrent renders across the pool),
+    grows the pool by vetting+solving a fresh IP when it's below target, and
+    applies backpressure (a short wait) when the pool is full and busy rather
+    than burning an extra solve. Retries walled IPs until real results land or
+    the per-fetch deadline / IP budget runs out. Requires the browser tier —
+    without it we cannot get JS-built results. ``mobile`` renders with a phone
+    UA/viewport (the ``mobileResults`` input).
     """
-    global _last_good_proxy
     if AsyncStealthySession is None:
         logger.error("[google_search] browser tier unavailable; cannot render SERPs")
         return None
 
     base = get_proxy_url()
+    if not base:
+        # No proxy configured: a single direct render, no pool bookkeeping.
+        with contextlib.suppress(Exception):
+            page = await _render(url, None, mobile=mobile)
+            html = page.html_content or ""
+            return html if (page.status == 200 and _has_results(html)) else None
+        return None
+
+    deadline = time.monotonic() + _FETCH_DEADLINE_S
     ips_tried = 0
-    while ips_tried < _MAX_IP_ATTEMPTS:
-        if base:
-            if _last_good_proxy and await _precheck(url, _last_good_proxy):
-                proxy = _last_good_proxy
+    while time.monotonic() < deadline and ips_tried < _MAX_IP_ATTEMPTS:
+        action, proxy = _pool_take()
+        if action == "wait":
+            # Pool full and every IP busy: wait for a warm slot instead of
+            # solving a fresh IP. The render gate drains continuously, so a slot
+            # frees shortly; this is the queue that bounds solves to ~pool size.
+            await asyncio.sleep(_POOL_WAIT_S)
+            continue
+        solved_fresh = False
+        vet_ms = 0.0
+        if action == "grow":
+            # Before paying for a solve, try to adopt an IP another process
+            # already solved (fleet-shared exemption). Only if none is available
+            # do we vet a fresh IP and solve on it ourselves.
+            adopted = await _store.adopt(set(_pool))
+            if adopted is not None:
+                proxy, cookies = adopted
+                _exemption_jar[proxy] = cookies
+                _pool_adopt(proxy)
             else:
-                _last_good_proxy = None
+                vt = time.perf_counter()
                 proxy = await _vet_fresh_ip(url, base)
+                vet_ms = (time.perf_counter() - vt) * 1000
                 ips_tried += _VET_CONCURRENCY
                 if proxy is None:
+                    _pool_abort_grow()
                     logger.debug("[google_search] vetting round: all IPs walled")
                     await asyncio.sleep(_WALLED_ROUND_BACKOFF_S)
                     continue
-        else:
-            proxy = None
-            ips_tried += 1
+                _pool_bind_fresh(proxy)
+                solved_fresh = True
         started = time.perf_counter()
         try:
             page = await _render(url, proxy, mobile=mobile)
@@ -411,23 +622,32 @@ async def fetch_serp_html(url: str, *, mobile: bool = False) -> str | None:
             # browser side is broken, so relaunch it rather than limp along.
             # repr(), not str(): e.g. NotImplementedError stringifies to "".
             logger.warning("[google_search] render failed: %r", e)
-            _last_good_proxy = None
+            _pool_settle(proxy, good=False, grew=solved_fresh)
+            await _store.evict(proxy)
             await _drop_session(mobile)
             continue
         fetch_ms = (time.perf_counter() - started) * 1000
         html = page.html_content or ""
         good = page.status == 200 and _has_results(html)
+        _pool_settle(proxy, good=good, grew=solved_fresh)
         logger.info(
-            "[google_search][perf] status=%s bytes=%d has_results=%s fetch_ms=%.0f reused_ip=%s",
+            "[google_search][perf] status=%s bytes=%d has_results=%s vet_ms=%.0f render_ms=%.0f from_pool=%s pool=%d",
             page.status,
             len(html),
             good,
+            vet_ms,
             fetch_ms,
-            proxy == _last_good_proxy,
+            not solved_fresh,
+            len(_pool),
         )
         if good:
-            _last_good_proxy = proxy
+            # Share a freshly-solved IP with the fleet; a walled IP is poison,
+            # so make sure no process keeps its stale exemption.
+            if solved_fresh and _exemption_jar.get(proxy):
+                await _store.publish(proxy, _exemption_jar[proxy])
             return html
-        _last_good_proxy = None
-    logger.warning("[google_search] exhausted %d IPs for %s", _MAX_IP_ATTEMPTS, url)
+        await _store.evict(proxy)
+    logger.warning(
+        "[google_search] gave up on %s (deadline/%d-IP budget)", url, _MAX_IP_ATTEMPTS
+    )
     return None

surfsense_backend/app/proprietary/platforms/google_search/pool_store.py

@@ -0,0 +1,121 @@
+"""Cross-process warm-IP exemption cache (Redis-backed, best-effort).
+
+The expensive unit is the reCAPTCHA solve. Within a process the warm pool
+(:mod:`fetch`) already solves each sticky IP once; this shares those solves
+across the whole worker fleet: a fresh solve on any process **publishes** the
+IP's ``GOOGLE_ABUSE_EXEMPTION`` cookies to Redis, and every other process can
+**adopt** that IP instead of solving it again. Without this, cost scales with
+the number of worker processes; with it, cost scales with the (shared) pool
+size.
+
+Best-effort by design: any Redis hiccup disables the layer and every process
+falls back to its own local pool — correctness is unchanged, we just pay more
+solves. Redis calls run in a worker thread (:func:`asyncio.to_thread`) so they
+never block the request loop and don't care which loop the caller is on.
+
+``ponytail:`` the shared store holds only exemptions (the costly artifact), not
+per-process render concurrency — global per-IP load is still governed by each
+process's local per-IP cap x the number of processes, so size the pool for the
+fleet (see ``GOOGLE_SEARCH_WARM_POOL_TARGET``). Full distributed inflight
+accounting is the upgrade path if a single shared IP ever gets overloaded.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import json
+import logging
+import os
+
+from app.config import config
+
+logger = logging.getLogger(__name__)
+
+_LOG = "[google_search][poolstore]"
+_PREFIX = "gsearch:warm:"
+# Google's GOOGLE_ABUSE_EXEMPTION outlives this comfortably; a conservative TTL
+# just means the fleet re-solves an idle IP occasionally (cheap) and never
+# serves a long-dead exemption. Env-tunable.
+_TTL_S = int(os.getenv("GOOGLE_SEARCH_EXEMPTION_TTL_S", "3600"))
+
+_client = None
+_disabled = False
+
+
+def _key(proxy: str) -> str:
+    return _PREFIX + hashlib.sha1(proxy.encode()).hexdigest()
+
+
+def _get_client():
+    """Lazily build a short-timeout Redis client, disabling on first failure."""
+    global _client, _disabled
+    if _disabled:
+        return None
+    if _client is not None:
+        return _client
+    try:
+        import redis
+
+        _client = redis.Redis.from_url(
+            config.REDIS_APP_URL, socket_timeout=1, socket_connect_timeout=1
+        )
+        _client.ping()
+    except Exception as e:
+        logger.info("%s Redis unavailable; cross-process sharing off: %s", _LOG, e)
+        _disabled = True
+        _client = None
+    return _client
+
+
+def _publish_sync(proxy: str, cookies: list[dict]) -> None:
+    c = _get_client()
+    if c is None:
+        return
+    try:
+        c.set(_key(proxy), json.dumps({"proxy": proxy, "cookies": cookies}), ex=_TTL_S)
+    except Exception as e:
+        logger.debug("%s publish failed: %s", _LOG, e)
+
+
+def _adopt_sync(exclude: set[str]) -> tuple[str, list[dict]] | None:
+    c = _get_client()
+    if c is None:
+        return None
+    try:
+        for key in c.scan_iter(match=_PREFIX + "*", count=100):
+            raw = c.get(key)
+            if not raw:
+                continue
+            d = json.loads(raw)
+            proxy = d.get("proxy")
+            if proxy and proxy not in exclude:
+                return proxy, d.get("cookies") or []
+    except Exception as e:
+        logger.debug("%s adopt failed: %s", _LOG, e)
+    return None
+
+
+def _evict_sync(proxy: str) -> None:
+    c = _get_client()
+    if c is None:
+        return
+    try:
+        c.delete(_key(proxy))
+    except Exception as e:
+        logger.debug("%s evict failed: %s", _LOG, e)
+
+
+async def publish(proxy: str, cookies: list[dict]) -> None:
+    """Share a freshly-solved IP's exemption with the fleet."""
+    await asyncio.to_thread(_publish_sync, proxy, cookies)
+
+
+async def adopt(exclude: set[str]) -> tuple[str, list[dict]] | None:
+    """Return a fleet-warm ``(proxy, cookies)`` not in ``exclude``, or ``None``."""
+    return await asyncio.to_thread(_adopt_sync, exclude)
+
+
+async def evict(proxy: str) -> None:
+    """Drop a poisoned (walled) IP so no process adopts its stale exemption."""
+    await asyncio.to_thread(_evict_sync, proxy)

surfsense_backend/app/proprietary/platforms/reddit/fetch.py

@@ -1,25 +1,30 @@
-"""Proxy-aware fetch seam for the Reddit scraper (no browser).
+"""Proxy-aware fetch seam for the Reddit scraper (browser-warm, HTTP-fetch).
 
 All network I/O flows through :func:`fetch_json` and always egresses through the
 residential proxy (a direct hit would expose and risk-block the server IP).
 
-Reddit deprecated *cold* unauthenticated ``.json`` (a bare anonymous GET now
-403s). The maintained anonymous recipe (proven live 2026-07-04, see
-``scripts/e2e_reddit_scraper.py`` step 0) is:
+Reddit shut anonymous ``.json`` (~May 2026) and, crucially, put a JavaScript
+challenge in front of minting the anonymous ``loid`` session cookie: a plain
+HTTP GET (even Chrome-impersonated) can no longer mint ``loid`` — it just 403s
+on the bot wall (confirmed live 2026-07-18, and yt-dlp #16877). So the recipe is
+now two-phase on ONE sticky exit IP:
 
-    warm one anonymous session cookie (``loid``) with a plain GET to
-    ``www.reddit.com/svc/shreddit/<slug>`` (``old.reddit.com`` fallback), then
-    GET ``www.reddit.com/<path>/.json?raw_json=1`` through that same
-    Chrome-impersonated, sticky-IP session. Which warm URL mints ``loid`` is
-    exit-IP dependent, so the order is a tiebreak — see :func:`warm_session`.
+    1. WARM (once per session): open a real patchright-Chromium stealth browser
+       (:func:`warm_session`) on a sticky proxy IP, load a public HTML page so
+       the JS challenge runs and mints the anonymous cookie jar (incl. ``loid``).
+    2. FETCH (many, fast): replay that minted jar through a plain-HTTP,
+       Chrome-impersonated ``FetcherSession`` pinned to the SAME sticky IP,
+       GETting ``www.reddit.com/<path>/.json?raw_json=1``. The ``.json`` body
+       comes back as raw JSON (no browser HTML wrapper) because this phase is
+       plain HTTP, keeping the fast/cheap fan-out — browser cost is paid ONCE
+       per warm, not per fetch.
 
-``loid`` is Reddit's equivalent of Google Maps' ``NID`` session cookie: an
-anonymous, logged-out id that unlocks the public API — no account, no browser.
-This module is a direct port of ``../youtube/innertube.py``'s rotate-on-block
-sticky-session pattern (``_RotatingSession`` + ``_current_session`` ContextVar +
-``open_proxy_holder``/``bind_proxy_holder``/``proxy_session``), with a
-Reddit-specific :func:`warm_session` bolted on and a ``.json``-shaped
-:func:`fetch_json` instead of an InnerTube POST.
+``loid`` binds to the exit IP, so both phases share one sticky proxy session id
+(see :meth:`_RotatingSession._open`); a 403/blocked IP rotates to a fresh sticky
+id + country and re-warms. This module is a port of ``../youtube/innertube.py``'s
+rotate-on-block sticky-session pattern (``_RotatingSession`` + ``_current_session``
+ContextVar + ``open_proxy_holder``/``bind_proxy_holder``/``proxy_session``), with
+the browser warm bolted on and a ``.json``-shaped :func:`fetch_json`.
 """
 
 from __future__ import annotations
@@ -29,15 +34,22 @@ import json
 import logging
 import random
 import time
+import uuid
 from contextlib import asynccontextmanager, suppress
 from contextvars import ContextVar
 from datetime import UTC, datetime
 from typing import Any
 from urllib.parse import urlencode
 
-from scrapling.fetchers import AsyncFetcher, FetcherSession
+from scrapling.fetchers import AsyncFetcher, AsyncStealthySession, FetcherSession
 
-from app.utils.proxy import get_proxy_url
+from app.utils.proxy import get_proxy_url, get_sticky_proxy_url
+
+# Shared cross-country rotation walk (also used by the TikTok sibling). Kept under
+# the historical private names this module and its tests reference.
+from app.utils.proxy.rotation import (
+    country_for_rotation as _country_for_rotation,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -68,7 +80,16 @@ _current_session: ContextVar[_RotatingSession | None] = ContextVar(
 # different handling per status (spec section 3).
 _ROTATE_STATUS = 403
 _BACKOFF_STATUS = 429
-_MAX_ROTATIONS = 3
+# Rotating an IP is cheap (close + reopen one keep-alive connection through the
+# gateway + a 2-request warm ≈ a few seconds) and each rotation also walks to the
+# next country pool, so we spend rotations liberally: neither a dirty IP nor a
+# wholly-blocked country pool should fail a job. 8 ≥ len(_FALLBACK_COUNTRIES), so
+# a job tries every country at least once before giving up. Worst case (a genuine
+# global block) costs _MAX_ROTATIONS bounded warm attempts before
+# RedditAccessBlockedError.
+# ponytail: 8 caps that worst case at ~30s; raise if every pool gets dirty at
+# once, lower if a real global block is wasting time.
+_MAX_ROTATIONS = 8
 _MAX_BACKOFFS = 4
 _BACKOFF_BASE_S = 5.0
 
@@ -96,29 +117,49 @@ _HEADERS = {"Accept-Language": "en-US,en;q=0.9"}
 # (the caller filters on ``includeNSFW`` downstream). Mirrors the probe.
 _OVER18_COOKIES = {"over18": "1"}
 
-# ``svc/shreddit`` needs *a* path to render; any always-public subreddit mints
-# ``loid`` just the same. ``r/popular`` always exists, so it's a safe default
-# regardless of which target this session ends up serving.
-_WARM_SLUG = "r/popular"
-_SHREDDIT_URL = (
-    "https://www.reddit.com/svc/shreddit/{slug}"
-    "?render-mode=partial&seeker-session=false"
-)
-_OLD_REDDIT_URL = "https://old.reddit.com/"
+# The browser warm loads *a* public HTML page so Reddit's JS challenge runs and
+# mints the anonymous jar; any always-public subreddit works. ``r/popular``
+# always exists, so it's a safe default regardless of this session's target.
+_WARM_HTML_URL = "https://www.reddit.com/r/popular/"
 _LOID_COOKIE = "loid"
 
+# The stealth browser warm (cold start + the page's own ``load``) lands in
+# ~6-15s once the solve_cloudflare/network_idle dead-waits are dropped (see
+# warm_session), so 30s is a generous ceiling that still bounds a dead exit; the
+# fast HTTP ``.json`` fetches keep the tight _REQUEST_TIMEOUT_S above.
+_WARM_TIMEOUT_MS = 30_000
+
+# Bound concurrent browser warms so a wide fan-out (up to _FANOUT_CONCURRENCY=16
+# workers, each warming once) can't spawn 16 Chromiums at once and OOM the box.
+# Warms are staggered, not serialized: a freed slot starts the next worker's warm
+# while already-warmed workers fetch over plain HTTP.
+# ponytail: 4 is tuned for a ~2GB container; raise on a bigger box, lower if warm
+# spikes push memory. Ceiling: a burst of >4 cold workers queues behind this.
+_WARM_CONCURRENCY = 4
+_warm_slots = asyncio.Semaphore(_WARM_CONCURRENCY)
+
 
 def now_iso() -> str:
     """UTC timestamp in the millisecond ISO shape used by scraper output."""
     return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
 
 
-def _response_cookie_names(page: Any) -> set[str]:
-    """Cookie names set by a response (best-effort across scrapling shapes)."""
+def _browser_cookie_jar(page: Any) -> dict[str, str]:
+    """Extract a ``{name: value}`` jar from a browser Response (best-effort).
+
+    Scrapling's browser Response exposes cookies as a list of ``{name, value,
+    ...}`` dicts (patchright/playwright shape); tolerate a plain ``{name: value}``
+    dict too so a shape change or a fake in tests still works.
+    """
     cookies = getattr(page, "cookies", None)
     if isinstance(cookies, dict):
-        return set(cookies.keys())
-    return set()
+        return {str(k): str(v) for k, v in cookies.items()}
+    jar: dict[str, str] = {}
+    if isinstance(cookies, (list, tuple)):
+        for item in cookies:
+            if isinstance(item, dict) and "name" in item and "value" in item:
+                jar[str(item["name"])] = str(item["value"])
+    return jar
 
 
 def _parse_json(page: Any) -> Any | None:
@@ -155,11 +196,13 @@ class _RotatingSession:
     """Owns one live ``FetcherSession`` (sticky IP) and can swap it for a fresh one.
 
     ``rotate()`` closes the current keep-alive connection and opens a new one, so
-    the rotating gateway hands out a different residential exit IP. Because the
-    ``loid`` cookie binds to the exit IP, ``rotate()`` also drops the warmed
-    state — the next fetch re-warms on the new IP. Used sequentially within a
-    single flow (never shared across concurrent tasks), so no locking is needed.
-    ``session`` is ``None`` only when no proxy is configured.
+    the rotating gateway hands out a different residential exit IP — walking to
+    the next country pool (see :func:`_country_for_rotation`) so a wholly-blocked
+    pool can't fail the flow. Because the ``loid`` cookie binds to the exit IP,
+    ``rotate()`` also drops the warmed state — the next fetch re-warms on the new
+    IP. Used sequentially within a single flow (never shared across concurrent
+    tasks), so no locking is needed. ``session`` is ``None`` only when no proxy
+    is configured.
     """
 
     def __init__(self) -> None:
@@ -167,22 +210,45 @@ class _RotatingSession:
         self.session: Any | None = None
         self.rotations = 0
         self.warmed = False
+        self.country = ""
+        # Sticky proxy URL shared by the browser warm AND the HTTP fetch session
+        # so both egress the SAME exit IP (``loid`` binds to that IP). The minted
+        # cookie jar is replayed on every ``.json`` fetch.
+        self.proxy: str | None = None
+        self.cookies: dict[str, str] = {}
         self._last_at = 0.0
 
     async def _open(self) -> None:
-        proxy = get_proxy_url()
         self.warmed = False
-        if proxy is None:
+        self.cookies = {}
+        self.country = _country_for_rotation(self.rotations)
+        # A fresh vendor session id per (re)open pins a distinct sticky exit IP,
+        # so rotating drops the dirty IP AND its dead cookie jar in one step.
+        session_id = uuid.uuid4().hex[:12]
+        self.proxy = get_sticky_proxy_url(session_id, self.country)
+        if self.proxy is None:
             self._cm = self.session = None
             return
         self._cm = FetcherSession(
-            proxy=proxy,
+            proxy=self.proxy,
             stealthy_headers=True,
             impersonate="chrome",
             timeout=_REQUEST_TIMEOUT_S,
         )
         self.session = await self._cm.__aenter__()
 
+    async def warm(self) -> bool:
+        """Browser-mint the anonymous cookie jar on this session's sticky IP.
+
+        Returns ``True`` and stores the jar (:attr:`cookies`) when ``loid`` was
+        minted, else ``False`` (caller rotates the IP and retries).
+        """
+        jar = await warm_session(self.proxy)
+        if jar:
+            self.cookies = jar
+            return True
+        return False
+
     async def close(self) -> None:
         if self._cm is not None:
             with suppress(Exception):  # best-effort teardown
@@ -194,7 +260,11 @@ class _RotatingSession:
         await self.close()
         self.rotations += 1
         await self._open()
-        logger.info("[reddit] rotated proxy session (rotation #%d)", self.rotations)
+        logger.info(
+            "[reddit] rotated proxy session (rotation #%d, country=%s)",
+            self.rotations,
+            self.country,
+        )
         return self.session
 
     async def pace(self) -> None:
@@ -237,49 +307,58 @@ async def proxy_session():
         await holder.close()
 
 
-async def warm_session(session: Any, *, slug: str = _WARM_SLUG) -> bool:
-    """Mint an anonymous ``loid`` cookie on a freshly opened session.
+async def warm_session(proxy: str | None) -> dict[str, str] | None:
+    """Browser-mint the anonymous cookie jar (incl. ``loid``) on a sticky IP.
 
-    Returns ``True`` when a ``loid`` was issued (the session can now reach
-    ``.json``), else ``False`` (caller rotates the IP and retries).
+    Reddit gates ``loid`` minting behind a JS challenge that a plain-HTTP client
+    can't clear, so this spins up patchright-Chromium ONCE per session on the
+    given sticky proxy IP, loads a public HTML page so the challenge runs, and
+    returns the minted ``{name: value}`` cookie jar. Returns ``None`` when
+    ``loid`` wasn't minted (a dirty exit IP), so the caller rotates to a fresh
+    sticky IP and retries. Proven live 2026-07-18 (see e2e step 0).
 
-    Tries ``svc/shreddit`` first, then ``old.reddit`` (yt-dlp's primary) as
-    the fallback. Live probes 2026-07-04 saw both directions across exit IPs,
-    but a live run 2026-07-06 had ``old.reddit`` 403 on 12/12 fresh IPs while
-    ``shreddit`` minted every time — Reddit appears to have shut old.reddit to
-    anonymous traffic, so shreddit-first saves one guaranteed-403 round trip
-    per warm-up. The fallback is what actually matters — it preserves
-    correctness if a given IP (or Reddit) flips back the other way. That cost
-    is amortized: ``fan_out`` reuses one warmed session per worker across many
-    jobs, so warm-up runs once per worker, not per fetch.
+    Takes the sticky ``proxy`` URL (never the fetch session) so the browser
+    egresses the SAME exit IP the HTTP ``.json`` fetches will replay the jar on.
+    A module-level semaphore bounds concurrent browser launches under fan-out.
 
-    ponytail: sequential two-source warm burns 1 wasted request on ~half of new
-    sessions. A parallel warm (gather both, take whichever mints) removes the
-    latency but always spends 2 requests; not worth it while warm-up is
-    once-per-worker. Revisit only if session churn (not reuse) dominates.
-
-    Takes an already-open ``session`` (never constructs one) so tests can drive
-    warm/rotate deterministically with a fake session, exactly like the youtube
-    sibling's fetch-resilience tests.
+    What actually clears Reddit's wall is the stealth browser + ``google_search``
+    referer, NOT captcha solving. ``solve_cloudflare`` and ``network_idle`` were
+    each ~60s of dead wait per warm (Reddit's wall isn't Cloudflare, so the solver
+    hunts a challenge that never appears; and its long-lived connections never go
+    network-idle) — dropping both took the warm from ~133s to ~10s with ``loid``
+    still minting 3/3 across us/gb/de exits (measured 2026-07-18). A rare miss
+    still self-heals: :func:`fetch_json` rotates to a fresh sticky IP and re-warms.
     """
-    seen: set[str] = set()
-    with suppress(Exception):
-        page = await session.get(_SHREDDIT_URL.format(slug=slug), headers=_HEADERS)
-        seen |= _response_cookie_names(page)
-    if _LOID_COOKIE in seen:
-        return True
-
-    # Fallback: mints loid on exit IPs where shreddit 403s instead.
-    with suppress(Exception):
-        page = await session.get(_OLD_REDDIT_URL, headers=_HEADERS)
-        seen |= _response_cookie_names(page)
-    return _LOID_COOKIE in seen
+    if proxy is None:
+        return None
+    async with _warm_slots:
+        try:
+            async with AsyncStealthySession(
+                headless=True,
+                google_search=True,
+                network_idle=False,
+                proxy=proxy,
+                timeout=_WARM_TIMEOUT_MS,
+            ) as sess:
+                page = await sess.fetch(_WARM_HTML_URL)
+                jar = _browser_cookie_jar(page)
+                return jar if _LOID_COOKIE in jar else None
+        except Exception as e:  # a browser crash must not abort the flow
+            logger.warning("[reddit] browser warm failed: %s", e)
+            return None
 
 
-async def _get_page(session: Any, url: str) -> Any:
-    """GET through the warmed sticky session, or a one-shot proxied fetch."""
+async def _get_page(session: Any, url: str, cookies: dict[str, str]) -> Any:
+    """GET through the warmed sticky session, or a one-shot proxied fetch.
+
+    ``cookies`` is the browser-minted jar; ``over18`` is merged so NSFW listings
+    aren't blanked. The one-shot path (no bound session) has no minted jar and is
+    a best-effort fallback only — it can't clear the JS challenge.
+    """
     if session is not None:
-        return await session.get(url, headers=_HEADERS, cookies=_OVER18_COOKIES)
+        return await session.get(
+            url, headers=_HEADERS, cookies={**cookies, **_OVER18_COOKIES}
+        )
     return await AsyncFetcher.get(
         url,
         headers=_HEADERS,
@@ -313,7 +392,7 @@ async def fetch_json(path: str, params: dict[str, Any] | None = None) -> Any | N
         session = holder.session
         try:
             if session is not None and not holder.warmed:
-                warmed_ok = await warm_session(session)
+                warmed_ok = await holder.warm()
                 holder.warmed = True  # attempted; don't re-warm this IP
                 if not warmed_ok:
                     if attempt < _MAX_ROTATIONS:
@@ -325,7 +404,7 @@ async def fetch_json(path: str, params: dict[str, Any] | None = None) -> Any | N
                     )
 
             await holder.pace()
-            page = await _get_page(session, url)
+            page = await _get_page(session, url, holder.cookies)
             status = page.status
 
             if status == 200:

surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py

@@ -10,6 +10,7 @@ from __future__ import annotations
 
 from collections.abc import AsyncIterator
 from typing import Any
+from urllib.parse import quote
 
 from .extraction.timestamps import now_iso
 from .flows import FetchCommentsFn, FetchFn, FetchListingFn, FetchUsersFn
@@ -31,15 +32,18 @@ from .targets.types import TikTokTarget
 
 _PROFILE_URL = "https://www.tiktok.com/@{name}"
 _HASHTAG_URL = "https://www.tiktok.com/tag/{tag}"
+_SEARCH_URL = "https://www.tiktok.com/search?q={query}"
 _EXPLORE_URL = "https://www.tiktok.com/explore"
 
 
 def _resolve_targets(input_model: TikTokScrapeInput) -> list[TikTokTarget]:
-    """Build the target list from the URL/profile/hashtag sources.
+    """Build the target list from every input source.
 
-    A raw ``tiktok.com/search?...`` URL passed explicitly in
-    ``startUrls``/``postURLs`` still resolves here and keeps its native listing
-    routing; there is no keyword-search shortcut.
+    ``searchQueries`` map to the same ``tiktok.com/search?q=`` targets that a raw
+    search URL in ``startUrls``/``postURLs`` resolves to, so both share the
+    listing flow's parse/dedupe/cap/empty-ErrorItem contract (the anonymous
+    search feed often withholds results, degrading to one honest ErrorItem rather
+    than a silent empty).
     """
     targets: list[TikTokTarget] = []
     for entry in input_model.startUrls:
@@ -55,6 +59,10 @@ def _resolve_targets(input_model: TikTokScrapeInput) -> list[TikTokTarget]:
         targets.append(TikTokTarget("profile", name, _PROFILE_URL.format(name=name)))
     for tag in input_model.hashtags:
         targets.append(TikTokTarget("hashtag", tag, _HASHTAG_URL.format(tag=tag)))
+    for query in input_model.searchQueries:
+        resolved = resolve_target(_SEARCH_URL.format(query=quote(query)))
+        if resolved is not None:
+            targets.append(resolved)
     return targets
 
 

surfsense_backend/app/proprietary/platforms/tiktok/session/client.py

@@ -26,7 +26,12 @@ logger = logging.getLogger(__name__)
 # 403 => IP blocked; rotate and re-warm. 429 => rate limited; back off same IP.
 _ROTATE_STATUS = 403
 _BACKOFF_STATUS = 429
-_MAX_ROTATIONS = 3
+# Each rotation walks to the next country pool (see session/proxy.py). The bare
+# worldwide pool never mints ``ttwid`` (proven live 2026-07-17), so warming
+# relies on reaching a good country pool — budget enough rotations to try every
+# one at least once (>= len(rotation FALLBACK_COUNTRIES)). Rotating is cheap
+# (reopen one keep-alive connection + a 1-request warm), so spend liberally.
+_MAX_ROTATIONS = 8
 _MAX_BACKOFFS = 4
 _BACKOFF_BASE_S = 5.0
 

surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py

@@ -10,10 +10,11 @@ signs correctly for whatever version TikTok ships.
 The pure response-shape parsing lives in :func:`items_from_response`; this module
 is the untested browser-I/O glue (covered by the e2e smoke, not unit tests).
 
-Needs a residential proxy; datacenter IPs get empty bodies and 429s. The profile
-feed returns an empty 200 to headless sessions, so :func:`fetch_item_list` goes
-headful only when ``CRAWL_HEADED_XVFB_ENABLED`` promises an Xvfb display — else it
-stays headless and degrades to an ``ErrorItem`` instead of crashing on launch.
+Needs a residential proxy; datacenter IPs get empty bodies and 429s. TikTok also
+withholds the feed (empty 200, or a redirect to /login) from the provider's
+unpinned worldwide pool, so :func:`_fetch_with_rotation` walks country-pinned
+exits until a draw is non-empty — the same escape the Reddit/ttwid warm path uses
+(see :mod:`app.utils.proxy.rotation`).
 """
 
 from __future__ import annotations
@@ -22,6 +23,7 @@ import asyncio
 import logging
 from collections.abc import Callable
 from typing import Any
+from urllib.parse import urlsplit
 
 from scrapling.fetchers import StealthyFetcher
 
@@ -30,7 +32,8 @@ from app.proprietary.web_crawler.stealth import (
     build_stealthy_kwargs,
     get_stealth_config,
 )
-from app.utils.proxy import get_proxy_url
+from app.utils.proxy import get_geo_proxy_url
+from app.utils.proxy.rotation import country_for_rotation
 
 from ..extraction import (
     comments_from_response,
@@ -224,6 +227,20 @@ def _build_page_action(
     return page_action
 
 
+def _primer_url(url: str) -> str:
+    """A tiny same-origin URL (``/robots.txt``) for the initial navigation.
+
+    Scrapling's outer ``page.goto`` waits for the ``load`` event, which TikTok's
+    SPA feed pages never fire — so navigating straight to the target burns the
+    full 30s timeout every fetch. Priming on ``robots.txt`` (a plain-text 200 that
+    fires ``load`` at once, same origin so referer/cookies stay coherent) lets the
+    ``page_action`` then reach the real target with ``domcontentloaded`` — ~4x
+    faster on profile feeds with no loss of capture.
+    """
+    parts = urlsplit(url)
+    return f"{parts.scheme}://{parts.netloc}/robots.txt"
+
+
 def _fetch_sync(
     url: str,
     target_count: int,
@@ -231,15 +248,16 @@ def _fetch_sync(
     extract: ExtractFn,
     interact: InteractFn,
     *,
+    proxy: str | None,
     headless: bool = True,
 ) -> list[dict[str, Any]]:
     collected: list[dict[str, Any]] = []
     kwargs = build_stealthy_kwargs(get_stealth_config())
     StealthyFetcher.fetch(
-        url,
+        _primer_url(url),
         headless=headless,
         network_idle=False,
-        proxy=get_proxy_url(),
+        proxy=proxy,
         page_action=_build_page_action(
             collected, url, target_count, markers, extract, interact
         ),
@@ -248,40 +266,68 @@ def _fetch_sync(
     return collected[:target_count]
 
 
-async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, Any]]:
-    """Return up to ``target_count`` itemStructs from a listing page's XHRs.
+async def _fetch_with_rotation(
+    page_url: str,
+    target_count: int,
+    markers: tuple[str, ...],
+    extract: ExtractFn,
+    interact: InteractFn,
+    *,
+    headless: bool = True,
+) -> list[dict[str, Any]]:
+    """Capture matching XHRs, walking exit *countries* until a draw is non-empty.
 
-    Headful when ``CRAWL_HEADED_XVFB_ENABLED`` promises a display (the profile feed
-    is empty to headless sessions); headless otherwise so launch never fails.
-    Retries an empty draw up to ``TIKTOK_LISTING_MAX_ATTEMPTS`` for a fresh exit IP.
+    TikTok withholds ``item_list``/``comment`` bodies from DataImpulse's unpinned
+    worldwide pool (the pool ``PROXY_URL`` uses for SERP health) but serves them on
+    country-pinned exits, so each empty draw retries on a fresh country rather than
+    the same flagged pool (see :mod:`app.utils.proxy.rotation`). Non-geo providers
+    ignore the country and re-draw their one URL, so this stays a safe no-op there.
     """
-    headless = not config.CRAWL_HEADED_XVFB_ENABLED
     attempts = max(1, config.TIKTOK_LISTING_MAX_ATTEMPTS)
-    for attempt in range(1, attempts + 1):
+    for attempt in range(attempts):
+        proxy = get_geo_proxy_url(country_for_rotation(attempt))
         items = await asyncio.to_thread(
             _fetch_sync,
             page_url,
             target_count,
-            _ITEM_LIST_MARKERS,
-            items_from_response,
-            _scroll_page,
+            markers,
+            extract,
+            interact,
+            proxy=proxy,
             headless=headless,
         )
-        if items or attempt == attempts:
+        if items or attempt == attempts - 1:
             return items
         logger.info(
-            "[tiktok] empty item_list for %s (attempt %d/%d); retrying on a fresh exit IP",
+            "[tiktok] empty %s for %s (attempt %d/%d); retrying on a fresh exit country",
+            markers[0],
             page_url,
-            attempt,
+            attempt + 1,
             attempts,
         )
     return []
 
 
+async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, Any]]:
+    """Return up to ``target_count`` itemStructs from a listing page's XHRs.
+
+    Headful when ``CRAWL_HEADED_XVFB_ENABLED`` promises a display, headless
+    otherwise so launch never fails. Retries empty draws across a spread of exit
+    countries (``TIKTOK_LISTING_MAX_ATTEMPTS``) to escape a TikTok-flagged pool.
+    """
+    return await _fetch_with_rotation(
+        page_url,
+        target_count,
+        _ITEM_LIST_MARKERS,
+        items_from_response,
+        _scroll_page,
+        headless=not config.CRAWL_HEADED_XVFB_ENABLED,
+    )
+
+
 async def fetch_user_search(page_url: str, target_count: int) -> list[dict[str, Any]]:
     """Return up to ``target_count`` ``user_info`` records from a user-search page."""
-    return await asyncio.to_thread(
-        _fetch_sync,
+    return await _fetch_with_rotation(
         page_url,
         target_count,
         _USER_SEARCH_MARKERS,
@@ -292,8 +338,7 @@ async def fetch_user_search(page_url: str, target_count: int) -> list[dict[str,
 
 async def fetch_comments(page_url: str, target_count: int) -> list[dict[str, Any]]:
     """Return up to ``target_count`` raw comment records from a video page's XHRs."""
-    return await asyncio.to_thread(
-        _fetch_sync,
+    return await _fetch_with_rotation(
         page_url,
         target_count,
         _COMMENT_MARKERS,
@@ -304,8 +349,7 @@ async def fetch_comments(page_url: str, target_count: int) -> list[dict[str, Any
 
 async def fetch_trending(page_url: str, target_count: int) -> list[dict[str, Any]]:
     """Return up to ``target_count`` trending itemStructs from the Explore feed."""
-    return await asyncio.to_thread(
-        _fetch_sync,
+    return await _fetch_with_rotation(
         page_url,
         target_count,
         _EXPLORE_MARKERS,

surfsense_backend/app/proprietary/platforms/tiktok/session/proxy.py

@@ -18,7 +18,8 @@ from typing import Any
 
 from scrapling.fetchers import FetcherSession
 
-from app.utils.proxy import get_proxy_url
+from app.utils.proxy import get_geo_proxy_url
+from app.utils.proxy.rotation import country_for_rotation
 
 logger = logging.getLogger(__name__)
 
@@ -37,9 +38,13 @@ _current_session: ContextVar[_RotatingSession | None] = ContextVar(
 class _RotatingSession:
     """Owns one live ``FetcherSession`` (sticky IP); ``rotate()`` swaps the IP.
 
-    Used sequentially within a single flow (never shared across concurrent
-    tasks), so no locking is needed. ``session`` is ``None`` only when no proxy
-    is configured.
+    Each open walks to the next country pool (see
+    :func:`app.utils.proxy.rotation.country_for_rotation`): TikTok withholds the
+    anonymous ``ttwid`` cookie from the provider's default worldwide pool but
+    mints it on country-pinned exits, so re-drawing within the same pool never
+    warms — spreading rotations across country pools does. Used sequentially
+    within a single flow (never shared across concurrent tasks), so no locking is
+    needed. ``session`` is ``None`` only when no proxy is configured.
     """
 
     def __init__(self) -> None:
@@ -47,11 +52,13 @@ class _RotatingSession:
         self.session: Any | None = None
         self.rotations = 0
         self.warmed = False
+        self.country = ""
         self._last_at = 0.0
 
     async def _open(self) -> None:
-        proxy = get_proxy_url()
         self.warmed = False
+        self.country = country_for_rotation(self.rotations)
+        proxy = get_geo_proxy_url(self.country)
         if proxy is None:
             self._cm = self.session = None
             return
@@ -74,7 +81,11 @@ class _RotatingSession:
         await self.close()
         self.rotations += 1
         await self._open()
-        logger.info("[tiktok] rotated proxy session (rotation #%d)", self.rotations)
+        logger.info(
+            "[tiktok] rotated proxy session (rotation #%d, country=%s)",
+            self.rotations,
+            self.country,
+        )
         return self.session
 
     async def pace(self) -> None:

surfsense_backend/app/proprietary/web_crawler/captcha.py

@@ -6,10 +6,10 @@
 """Captcha detection + token-injection ``page_action`` (Phase 3d).
 
 This is the **bypass logic** (hence proprietary); the generic, vendor-agnostic
-config lives in the Apache-2.0 ``app/utils/captcha/`` package. ``captchatools``
-is the multi-vendor solver registry — we only detect the challenge, harvest a
-token egressing from **the crawl's own proxy IP** (token IP-binding), inject it,
-and submit.
+config and the vendor API clients live in the Apache-2.0 ``app/utils/captcha/``
+package (:mod:`app.utils.captcha.solvers`). We only detect the challenge,
+harvest a token through that seam egressing from **the crawl's own proxy IP**
+(token IP-binding), inject it, and submit.
 
 Why a closure cell: Scrapling runs ``page_action`` after navigation but
 **swallows its exceptions and discards its return value** (see
@@ -29,7 +29,6 @@ import re
 from collections.abc import Callable
 from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeout
 from typing import Any
-from urllib.parse import urlsplit
 
 from app.utils.captcha import CaptchaConfig
 
@@ -118,26 +117,6 @@ def detect_challenge(page: Any, cfg: CaptchaConfig) -> tuple[str, str] | None:
 # --- proxy / harvest -------------------------------------------------------
 
 
-def proxy_url_to_captchatools(proxy_url: str | None) -> str | None:
-    """Reformat ``http://user:pass@host:port`` -> ``host:port:user:pass``.
-
-    captchatools wants the colon-delimited form. Returns ``None`` for a missing
-    or unparseable proxy so the solver harvests proxyless (the token may then be
-    IP-mismatched, but that's better than crashing the fetch).
-    """
-    if not proxy_url:
-        return None
-    try:
-        p = urlsplit(proxy_url)
-        if not p.hostname or not p.port:
-            return None
-        if p.username and p.password:
-            return f"{p.hostname}:{p.port}:{p.username}:{p.password}"
-        return f"{p.hostname}:{p.port}"
-    except Exception:
-        return None
-
-
 def _harvest_token(
     cfg: CaptchaConfig,
     challenge_type: str,
@@ -146,26 +125,21 @@ def _harvest_token(
     proxy: str | None,
     user_agent: str | None,
 ) -> str | None:
-    """Harvest a token from the solver. Raises on solver errors so the caller
-    can latch on the unrecoverable ones; returns ``None`` on an empty token.
+    """Harvest a token from the in-house solver seam. Raises on solver errors so
+    the caller can latch on the unrecoverable ones; returns ``None`` on an empty
+    token. ``proxy`` is a raw ``http://user:pass@host:port`` URL (the seam
+    reformats it per vendor).
     """
-    import captchatools
+    from app.utils.captcha import solvers
 
-    harvester = captchatools.new_harvester(
-        api_key=cfg.api_key,
-        solving_site=cfg.solving_site,
+    return solvers.solve(
+        cfg,
+        challenge_type=challenge_type,
         sitekey=sitekey,
-        captcha_url=page_url,
-        captcha_type=challenge_type,
-        min_score=cfg.v3_min_score,
-        action=cfg.v3_action,
-    )
-    token = harvester.get_token(
-        proxy=proxy,
-        proxy_type="HTTP",
+        page_url=page_url,
+        proxy_url=proxy,
         user_agent=user_agent,
     )
-    return token or None
 
 
 # --- injection -------------------------------------------------------------
@@ -247,7 +221,9 @@ def build_captcha_page_action(
     if not cfg.enabled or solver_latched():
         return None
 
-    captcha_proxy = proxy_url_to_captchatools(proxy_url)
+    # Raw URL; the solver seam reformats it per vendor and binds the token to
+    # this exit IP.
+    captcha_proxy = proxy_url
 
     def page_action(page: Any) -> Any:
         # Hard cap: never exceed the per-URL budget even across proxy-retry
@@ -320,10 +296,13 @@ def build_captcha_page_action(
 
 
 def _is_unrecoverable(exc: Exception) -> bool:
-    """True for solver errors that must latch solving off (no balance / bad key).
+    """True for solver errors that must latch solving off.
 
-    Matched by class name to avoid a hard import of ``captchatools.exceptions``
-    at module load (the dep is optional / lazily imported).
+    Covers the in-house seam's typed errors (``SolverBalanceError`` /
+    ``SolverAuthError`` / ``SolverUnsupportedError``) plus legacy/no-balance shapes.
+    Matched by class name so no solver module must be imported here.
     """
     name = type(exc).__name__.lower()
-    return "nobalance" in name or "wrongapikey" in name or "apikey" in name
+    return any(
+        k in name for k in ("balance", "apikey", "auth", "unsupported", "wronguser")
+    )

surfsense_backend/app/routes/__init__.py

@@ -1,6 +1,7 @@
 from fastapi import APIRouter, Depends
 
 # Import verb namespaces for their registration side effects before the door builds.
+import app.capabilities.amazon
 import app.capabilities.google_maps
 import app.capabilities.google_search
 import app.capabilities.indeed

surfsense_backend/app/routes/model_connections_routes.py

@@ -1,4 +1,5 @@
 import logging
+from datetime import UTC, datetime
 
 from fastapi import APIRouter, Depends, HTTPException
 from sqlalchemy import select, update
@@ -16,11 +17,13 @@ from app.db import (
     Permission,
     Workspace,
     get_async_session,
+    has_permission,
 )
 from app.schemas import (
     ConnectionCreate,
     ConnectionRead,
     ConnectionUpdate,
+    LlmSetupStatusRead,
     ModelCreate,
     ModelPreviewRead,
     ModelProviderRead,
@@ -43,7 +46,7 @@ from app.services.model_connection_service import (
 )
 from app.services.provider_registry import REGISTRY
 from app.users import get_auth_context, require_session_context
-from app.utils.rbac import check_permission
+from app.utils.rbac import check_permission, get_user_permissions
 
 router = APIRouter()
 logger = logging.getLogger(__name__)
@@ -355,7 +358,7 @@ async def list_connections(
             session,
             auth,
             workspace_id,
-            Permission.LLM_CONFIGS_CREATE.value,
+            Permission.LLM_CONFIGS_READ.value,
             "You don't have permission to view model connections in this workspace",
         )
         stmt = stmt.where(Connection.workspace_id == workspace_id)
@@ -760,7 +763,7 @@ async def get_model_roles(
         session,
         auth,
         workspace_id,
-        Permission.LLM_CONFIGS_CREATE.value,
+        Permission.LLM_CONFIGS_READ.value,
         "You don't have permission to view model roles in this workspace",
     )
     workspace = await _clear_invalid_roles(session, workspace_id)
@@ -831,3 +834,127 @@ async def update_model_roles(
         vision_model_id=workspace.vision_model_id,
         image_gen_model_id=workspace.image_gen_model_id,
     )
+
+
+def _global_catalog_has_usable_chat() -> bool:
+    """True when the operator global catalog exposes a usable chat model.
+
+    Checks usability (enabled connection + enabled, chat-capable model), not
+    mere presence of ``global_llm_config.yaml`` — an empty or malformed file,
+    or an OpenRouter-only config whose startup fetch failed, yields no models
+    and must fall through to onboarding.
+    """
+    enabled_connection_ids = {
+        conn["id"] for conn in config.GLOBAL_CONNECTIONS if conn.get("enabled", True)
+    }
+    return any(
+        model.get("connection_id") in enabled_connection_ids
+        and model.get("enabled", True)
+        and has_capability(model, "chat")
+        for model in config.GLOBAL_MODELS
+    )
+
+
+async def _workspace_has_enabled_chat_model(
+    session: AsyncSession, workspace_id: int
+) -> bool:
+    result = await session.execute(
+        select(Connection)
+        .options(selectinload(Connection.models))
+        .where(
+            Connection.workspace_id == workspace_id,
+            Connection.enabled,
+        )
+    )
+    return any(
+        model.enabled and has_capability(model, "chat")
+        for conn in result.scalars().all()
+        for model in conn.models
+    )
+
+
+async def compute_llm_setup_status(
+    session: AsyncSession,
+    auth: AuthContext,
+    workspace_id: int,
+) -> LlmSetupStatusRead:
+    """Single source of truth for whether a workspace can chat.
+
+    "Needs onboarding" is derived, never persisted: a workspace is ``ready``
+    exactly when a usable chat model resolves for it (operator global catalog,
+    a valid pinned role, or an enabled chat-capable model in Auto mode).
+    """
+    await check_permission(
+        session,
+        auth,
+        workspace_id,
+        Permission.LLM_CONFIGS_READ.value,
+        "You don't have permission to view LLM setup status in this workspace",
+    )
+    permissions = await get_user_permissions(session, auth.user.id, workspace_id)
+    can_configure = has_permission(permissions, Permission.LLM_CONFIGS_CREATE.value)
+
+    global_usable = _global_catalog_has_usable_chat()
+    if config.GLOBAL_LLM_CONFIG_FILE_EXISTS and global_usable:
+        # Global readiness is never stamped: it is not this workspace's own setup.
+        return LlmSetupStatusRead(
+            status="ready",
+            source="global_config",
+            can_configure=can_configure,
+            stage="ready",
+        )
+
+    # Heal dangling role pins first: a chat_model_id pointing at a deleted or
+    # disabled model collapses to 0 (Auto) here, so the checks below see the
+    # real state.
+    workspace = await _clear_invalid_roles(session, workspace_id)
+    await session.commit()
+    await session.refresh(workspace)
+
+    async def _stamp_own_setup() -> None:
+        # Record first own-model readiness once; the guard makes concurrent
+        # stamps idempotent. (Writing on this GET is precedented above.)
+        if workspace.llm_setup_completed_at is None:
+            workspace.llm_setup_completed_at = datetime.now(UTC)
+            await session.commit()
+
+    chat_model_id = workspace.chat_model_id or 0
+    if chat_model_id != 0:
+        # Survived _clear_invalid_roles => valid, enabled, chat-capable.
+        source = "global_config" if chat_model_id < 0 else "models"
+        if source == "models":
+            await _stamp_own_setup()
+        return LlmSetupStatusRead(
+            status="ready", source=source, can_configure=can_configure, stage="ready"
+        )
+
+    if global_usable:
+        return LlmSetupStatusRead(
+            status="ready",
+            source="global_config",
+            can_configure=can_configure,
+            stage="ready",
+        )
+    if await _workspace_has_enabled_chat_model(session, workspace_id):
+        await _stamp_own_setup()
+        return LlmSetupStatusRead(
+            status="ready", source="models", can_configure=can_configure, stage="ready"
+        )
+
+    # A set timestamp => self-configured before (recovery); NULL => never (first-run).
+    stage = "recovery" if workspace.llm_setup_completed_at else "initial_setup"
+    return LlmSetupStatusRead(
+        status="needs_setup", source="none", can_configure=can_configure, stage=stage
+    )
+
+
+@router.get(
+    "/workspaces/{workspace_id}/llm-setup-status",
+    response_model=LlmSetupStatusRead,
+)
+async def llm_setup_status(
+    workspace_id: int,
+    session: AsyncSession = Depends(get_async_session),
+    auth: AuthContext = Depends(get_auth_context),
+):
+    return await compute_llm_setup_status(session, auth, workspace_id)

surfsense_backend/app/routes/workspaces_routes.py

@@ -14,6 +14,7 @@ from app.db import (
     get_async_session,
     get_default_roles_config,
 )
+from app.routes.model_connections_routes import compute_llm_setup_status
 from app.schemas import (
     WorkspaceApiAccessUpdate,
     WorkspaceCreate,
@@ -93,7 +94,12 @@ async def create_workspace(
 
         await session.commit()
         await session.refresh(db_workspace)
-        return db_workspace
+
+        response = WorkspaceRead.model_validate(db_workspace)
+        response.llm_setup = await compute_llm_setup_status(
+            session, auth, db_workspace.id
+        )
+        return response
     except HTTPException:
         raise
     except Exception as e:

surfsense_backend/app/schemas/__init__.py

@@ -43,6 +43,7 @@ from .model_connections import (
     ConnectionCreate,
     ConnectionRead,
     ConnectionUpdate,
+    LlmSetupStatusRead,
     ModelCreate,
     ModelPreviewRead,
     ModelProviderRead,
@@ -185,13 +186,14 @@ __all__ = [
     "InviteInfoResponse",
     "InviteRead",
     "InviteUpdate",
+    # Auth schemas
+    "LlmSetupStatusRead",
     # Log schemas
     "LogBase",
     "LogCreate",
     "LogFilter",
     "LogRead",
     "LogUpdate",
-    # Auth schemas
     "LogoutAllResponse",
     "LogoutRequest",
     "LogoutResponse",

surfsense_backend/app/schemas/model_connections.py

@@ -1,6 +1,6 @@
 import uuid
 from datetime import datetime
-from typing import Any
+from typing import Any, Literal
 
 from pydantic import BaseModel, ConfigDict, Field
 
@@ -146,3 +146,18 @@ class ModelRolesUpdate(BaseModel):
     chat_model_id: int | None = None
     vision_model_id: int | None = None
     image_gen_model_id: int | None = None
+
+
+class LlmSetupStatusRead(BaseModel):
+    """Server-authoritative verdict for the per-workspace LLM onboarding gate.
+
+    ``status`` is the only thing the frontend gate acts on; ``source`` is
+    informational and ``can_configure`` selects the onboarding vs. blocked
+    screen for members who cannot manage models. ``stage`` refines a
+    ``needs_setup`` verdict into first-run (``initial_setup``) vs. recovery.
+    """
+
+    status: Literal["ready", "needs_setup"]
+    source: Literal["global_config", "models", "none"]
+    can_configure: bool
+    stage: Literal["initial_setup", "recovery", "ready"]

surfsense_backend/app/schemas/workspace.py

@@ -4,6 +4,7 @@ from datetime import datetime
 from pydantic import BaseModel, ConfigDict
 
 from .base import IDModel, TimestampModel
+from .model_connections import LlmSetupStatusRead
 
 
 class WorkspaceBase(BaseModel):
@@ -35,6 +36,9 @@ class WorkspaceRead(WorkspaceBase, IDModel, TimestampModel):
     api_access_enabled: bool = False
     qna_custom_instructions: str | None = None
     shared_memory_md: str | None = None
+    # Populated only by create_workspace so the client can route straight to
+    # onboarding vs. new-chat on the first hop. Null everywhere else.
+    llm_setup: LlmSetupStatusRead | None = None
 
     model_config = ConfigDict(from_attributes=True)
 

surfsense_backend/app/utils/captcha/__init__.py

@@ -1,15 +1,16 @@
 """App-wide captcha-solver configuration (Apache-2.0, generic glue).
 
-This package holds only the **generic, vendor-agnostic** config resolution for
-captcha solving — mirroring ``app/utils/proxy/`` (which stays Apache-2.0 even
-though the crawler that consumes it is proprietary). The actual bypass logic
-(challenge detection + token injection ``page_action``) lives in the separately
-licensed ``app/proprietary/web_crawler/captcha.py``.
+This package holds the **generic, vendor-agnostic** config resolution for
+captcha solving (:class:`CaptchaConfig`) plus the in-house vendor API clients
+(:mod:`app.utils.captcha.solvers`) — mirroring ``app/utils/proxy/`` (which stays
+Apache-2.0 even though the crawler that consumes it is proprietary). The actual
+bypass logic (challenge detection + token injection ``page_action``) lives in
+the separately licensed ``app/proprietary/web_crawler/captcha.py`` and
+``app/proprietary/platforms/google_search/captcha.py``.
 
-``captchatools`` is itself the multi-vendor registry (capmonster / 2captcha /
-anticaptcha / capsolver / captchaai), so there is no provider hierarchy here —
-just one env-driven :class:`CaptchaConfig` and a cheap ``captcha_enabled()``
-gate callers check before constructing anything (mirrors
+``solvers.solve()`` dispatches on ``CAPTCHA_SOLVER_PROVIDER``; only 2captcha is
+wired today, more vendors are added progressively. ``captcha_enabled()`` is the
+cheap gate callers check before constructing anything (mirrors
 ``WebCrawlCreditService.billing_enabled()``).
 """
 

surfsense_backend/app/utils/captcha/solvers.py

@@ -0,0 +1,315 @@
+"""In-house captcha-solver clients (vendor API glue, Apache-2.0).
+
+Replaces the unmaintained ``captchatools`` registry. Each provider is a small
+function that submits a challenge to the vendor and polls for the token;
+:func:`solve` dispatches on the app-wide ``CAPTCHA_SOLVER_PROVIDER``.
+
+Two providers are wired: **2captcha** (legacy ``in.php``/``res.php``) and
+**capsolver** (AI-native ``createTask``/``getTaskResult``, materially faster on
+the reCAPTCHA-*Enterprise* ``/sorry`` wall). Both express the Enterprise pieces
+Google needs — the widget sitekey plus the page's dynamic ``data-s`` token
+(something ``captchatools`` could not). More vendors (anticaptcha / capmonster)
+are added progressively as new entries in :data:`_PROVIDERS`; until then an
+unconfigured provider raises :class:`SolverUnsupportedError` so callers latch off
+cleanly instead of leaking the API key to the wrong service.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from urllib.parse import urlsplit
+
+import requests
+
+from app.utils.captcha.config import CaptchaConfig
+
+logger = logging.getLogger(__name__)
+
+_LOG = "[captcha][solver]"
+
+
+class SolverError(Exception):
+    """Base class for solver failures the caller may want to latch on."""
+
+
+class SolverAuthError(SolverError):
+    """Bad / unknown API key — unrecoverable without a config change."""
+
+
+class SolverBalanceError(SolverError):
+    """Solver account is out of balance — unrecoverable this process."""
+
+
+class SolverUnsupportedError(SolverError):
+    """Configured provider has no in-house client yet — unrecoverable."""
+
+
+# --- 2captcha (legacy in.php/res.php) --------------------------------------
+
+_2CAP_IN = "http://2captcha.com/in.php"
+_2CAP_RES = "http://2captcha.com/res.php"
+# Historical captchatools soft_id, kept so solves stay attributed on 2captcha.
+_2CAP_SOFT_ID = 4782723
+
+
+def proxy_login_form(proxy_url: str | None) -> str | None:
+    """``http://user:pass@host:port`` -> ``user:pass@host:port`` (no scheme).
+
+    The vendor APIs want the proxy without a scheme prefix (2captcha rejects a
+    scheme with ``ERROR_PROXY_FORMAT``). Returns ``None`` for a missing or
+    unparseable proxy so the solve goes proxyless rather than crashing the fetch
+    (the token may then be IP-mismatched, but that fails cleanly).
+    """
+    if not proxy_url:
+        return None
+    try:
+        p = urlsplit(proxy_url)
+        if not p.hostname or not p.port:
+            return None
+        if p.username and p.password:
+            return f"{p.username}:{p.password}@{p.hostname}:{p.port}"
+        return f"{p.hostname}:{p.port}"
+    except Exception:
+        return None
+
+
+def _raise_2cap(err: str) -> None:
+    """Map a 2captcha error string to a typed exception (or just log a soft one)."""
+    up = err.upper()
+    if "ZERO_BALANCE" in up:
+        raise SolverBalanceError(err)
+    if "WRONG_USER_KEY" in up or "KEY_DOES_NOT_EXIST" in up:
+        raise SolverAuthError(err)
+    logger.warning("%s 2captcha error: %s", _LOG, err)
+
+
+def _twocaptcha(
+    cfg: CaptchaConfig,
+    *,
+    challenge_type: str,
+    sitekey: str,
+    page_url: str,
+    proxy_url: str | None,
+    user_agent: str | None,
+    enterprise: bool,
+    data_s: str | None,
+) -> str | None:
+    payload: dict = {"key": cfg.api_key, "json": 1, "soft_id": _2CAP_SOFT_ID}
+    if challenge_type in ("hcaptcha", "hcap"):
+        payload |= {"method": "hcaptcha", "sitekey": sitekey, "pageurl": page_url}
+    elif challenge_type == "v3":
+        payload |= {
+            "method": "userrecaptcha",
+            "version": "v3",
+            "googlekey": sitekey,
+            "pageurl": page_url,
+            "action": cfg.v3_action,
+            "min_score": cfg.v3_min_score,
+        }
+    else:  # v2, optionally the Enterprise variant (adds enterprise=1 + data-s)
+        payload |= {
+            "method": "userrecaptcha",
+            "googlekey": sitekey,
+            "pageurl": page_url,
+        }
+        if enterprise:
+            payload["enterprise"] = 1
+        if data_s:
+            payload["data-s"] = data_s
+
+    proxy = proxy_login_form(proxy_url)
+    if proxy:
+        payload["proxy"] = proxy
+        payload["proxytype"] = "HTTP"
+    if user_agent:
+        payload["userAgent"] = user_agent
+
+    try:
+        submit = requests.post(_2CAP_IN, data=payload, timeout=30).json()
+    except requests.RequestException as e:
+        logger.warning("%s 2captcha submit request failed: %s", _LOG, e)
+        return None
+    if submit.get("status") != 1:
+        _raise_2cap(str(submit.get("request", "")))
+        return None
+
+    task_id = submit["request"]
+    deadline = time.monotonic() + cfg.timeout_s
+    while time.monotonic() < deadline:
+        time.sleep(5)
+        try:
+            got = requests.get(
+                f"{_2CAP_RES}?key={cfg.api_key}&action=get&id={task_id}&json=1",
+                timeout=30,
+            ).json()
+        except requests.RequestException:
+            continue
+        if got.get("status") == 1:
+            return got["request"] or None
+        req = str(got.get("request", ""))
+        if req != "CAPCHA_NOT_READY":
+            _raise_2cap(req)
+            return None
+    logger.warning("%s 2captcha solve timed out after %ss", _LOG, cfg.timeout_s)
+    return None
+
+
+# --- capsolver (createTask / getTaskResult) --------------------------------
+
+_CAPSOLVER_CREATE = "https://api.capsolver.com/createTask"
+_CAPSOLVER_RESULT = "https://api.capsolver.com/getTaskResult"
+
+
+def capsolver_proxy(proxy_url: str | None) -> str | None:
+    """``http://user:pass@host:port`` -> ``http:host:port:user:pass``.
+
+    CapSolver's ``proxy`` field is a single colon-delimited
+    ``scheme:host:port[:user:pass]`` string (NOT a URL). Returns ``None`` for a
+    missing/unparseable proxy so the caller falls back to a proxyless task.
+    """
+    if not proxy_url:
+        return None
+    try:
+        p = urlsplit(proxy_url)
+        if not p.hostname or not p.port:
+            return None
+        scheme = p.scheme or "http"
+        if p.username and p.password:
+            return f"{scheme}:{p.hostname}:{p.port}:{p.username}:{p.password}"
+        return f"{scheme}:{p.hostname}:{p.port}"
+    except Exception:
+        return None
+
+
+def _raise_capsolver(code: str, desc: str) -> None:
+    """Map a CapSolver errorCode to a typed exception (or log a soft one)."""
+    up = (code or "").upper()
+    if "ZERO_BALANCE" in up or "INSUFFICIENT" in up:
+        raise SolverBalanceError(f"{code}: {desc}")
+    if "KEY" in up:  # ERROR_KEY_DENIED_ACCESS / ERROR_KEY_DOES_NOT_EXIST
+        raise SolverAuthError(f"{code}: {desc}")
+    logger.warning("%s capsolver error: %s %s", _LOG, code, desc)
+
+
+def _capsolver(
+    cfg: CaptchaConfig,
+    *,
+    challenge_type: str,
+    sitekey: str,
+    page_url: str,
+    proxy_url: str | None,
+    user_agent: str | None,
+    enterprise: bool,
+    data_s: str | None,
+) -> str | None:
+    # We always egress through our own sticky proxy, so the proxied task types
+    # (no "ProxyLess" casing ambiguity) are the hot path; the proxyless variants
+    # are only the fallback when no proxy was threaded through.
+    proxy = capsolver_proxy(proxy_url)
+    has_proxy = proxy is not None
+    task: dict = {"websiteURL": page_url, "websiteKey": sitekey}
+    if challenge_type in ("hcaptcha", "hcap"):
+        task["type"] = "HCaptchaTask" if has_proxy else "HCaptchaTaskProxyLess"
+    elif challenge_type == "v3":
+        base = "ReCaptchaV3EnterpriseTask" if enterprise else "ReCaptchaV3Task"
+        task["type"] = base if has_proxy else base + "ProxyLess"
+        task["pageAction"] = cfg.v3_action
+        task["minScore"] = cfg.v3_min_score
+    else:  # v2, optionally the Enterprise variant (Google /sorry)
+        base = "ReCaptchaV2EnterpriseTask" if enterprise else "ReCaptchaV2Task"
+        task["type"] = base if has_proxy else base + "ProxyLess"
+        if data_s:
+            # Enterprise carries the /anchor `s` under enterprisePayload; the
+            # normal-v2 field is recaptchaDataSValue.
+            task["enterprisePayload" if enterprise else "recaptchaDataSValue"] = (
+                {"s": data_s} if enterprise else data_s
+            )
+    if has_proxy:
+        task["proxy"] = proxy
+    if user_agent:
+        task["userAgent"] = user_agent
+
+    try:
+        created = requests.post(
+            _CAPSOLVER_CREATE,
+            json={"clientKey": cfg.api_key, "task": task},
+            timeout=30,
+        ).json()
+    except requests.RequestException as e:
+        logger.warning("%s capsolver createTask failed: %s", _LOG, e)
+        return None
+    if created.get("errorId"):
+        _raise_capsolver(
+            str(created.get("errorCode", "")), str(created.get("errorDescription", ""))
+        )
+        return None
+    task_id = created.get("taskId")
+    if not task_id:
+        return None
+
+    deadline = time.monotonic() + cfg.timeout_s
+    while time.monotonic() < deadline:
+        time.sleep(2)  # AI solver is fast; poll tighter than 2captcha's 5 s
+        try:
+            got = requests.post(
+                _CAPSOLVER_RESULT,
+                json={"clientKey": cfg.api_key, "taskId": task_id},
+                timeout=30,
+            ).json()
+        except requests.RequestException:
+            continue
+        if got.get("errorId"):
+            _raise_capsolver(
+                str(got.get("errorCode", "")), str(got.get("errorDescription", ""))
+            )
+            return None
+        if got.get("status") == "ready":
+            return (got.get("solution") or {}).get("gRecaptchaResponse") or None
+    logger.warning("%s capsolver solve timed out after %ss", _LOG, cfg.timeout_s)
+    return None
+
+
+# provider name (CAPTCHA_SOLVER_PROVIDER) -> client. Add vendors here.
+_PROVIDERS = {"2captcha": _twocaptcha, "capsolver": _capsolver}
+
+
+def supported_providers() -> list[str]:
+    return sorted(_PROVIDERS)
+
+
+def solve(
+    cfg: CaptchaConfig,
+    *,
+    challenge_type: str,
+    sitekey: str,
+    page_url: str,
+    proxy_url: str | None = None,
+    user_agent: str | None = None,
+    enterprise: bool = False,
+    data_s: str | None = None,
+) -> str | None:
+    """Harvest a token from the configured solver, or ``None`` on soft failure.
+
+    ``challenge_type`` is ``v2`` / ``v3`` / ``hcaptcha``; set ``enterprise`` +
+    ``data_s`` for reCAPTCHA-v2-Enterprise pages (e.g. Google ``/sorry``). The
+    solve egresses through ``proxy_url`` so the token is bound to the crawl's
+    own exit IP. Raises a :class:`SolverError` subclass on unrecoverable errors
+    (bad key / no balance / unsupported provider) so callers can latch.
+    """
+    client = _PROVIDERS.get((cfg.solving_site or "").lower())
+    if client is None:
+        raise SolverUnsupportedError(
+            f"captcha provider {cfg.solving_site!r} has no in-house client "
+            f"(supported: {supported_providers()})"
+        )
+    return client(
+        cfg,
+        challenge_type=challenge_type,
+        sitekey=sitekey,
+        page_url=page_url,
+        proxy_url=proxy_url,
+        user_agent=user_agent,
+        enterprise=enterprise,
+        data_s=data_s,
+    )

surfsense_backend/app/utils/crawl/contacts.py

@@ -1,6 +1,6 @@
 """Pure contact/social-signal extraction from raw HTML (Apache-2.0, generic).
 
-Lead-gen / competitive-intelligence crawls need the emails, phone numbers, and
+Lead-gen / company-research crawls need the emails, phone numbers, and
 social profiles a site publishes — which almost always live in the footer, the
 contact page, or the privacy/terms pages. Trafilatura's main-content extraction
 deliberately drops that boilerplate, so these signals must be pulled from the

surfsense_backend/app/utils/proxy/__init__.py

@@ -15,6 +15,16 @@ def get_proxy_url() -> str | None:
     return get_active_provider().get_proxy_url()
 
 
+def get_geo_proxy_url(country: str | None = None) -> str | None:
+    """Proxy URL pinned to an exit country when the provider supports it."""
+    return get_active_provider().get_geo_proxy_url(country)
+
+
+def get_sticky_proxy_url(session_id: str, country: str | None = None) -> str | None:
+    """Proxy URL pinned to a stable vendor session when supported."""
+    return get_active_provider().get_sticky_proxy_url(session_id, country)
+
+
 def get_playwright_proxy() -> dict[str, str] | None:
     """Playwright-style proxy dict, or ``None`` when not configured."""
     return get_active_provider().get_playwright_proxy()
@@ -41,9 +51,11 @@ def get_residential_proxy_url() -> str | None:
 __all__ = [
     "ProxyProvider",
     "get_active_provider",
+    "get_geo_proxy_url",
     "get_playwright_proxy",
     "get_proxy_url",
     "get_requests_proxies",
     "get_residential_proxy_url",
+    "get_sticky_proxy_url",
     "is_pool_backed",
 ]

surfsense_backend/app/utils/proxy/base.py

@@ -67,6 +67,24 @@ class ProxyProvider(ABC):
             return None
         return {"http": proxy_url, "https": proxy_url}
 
+    def get_geo_proxy_url(self, country: str | None = None) -> str | None:
+        """Return a proxy URL pinned to ``country`` when supported.
+
+        Providers without vendor-specific country routing safely fall back to
+        their ordinary proxy URL.
+        """
+        return self.get_proxy_url()
+
+    def get_sticky_proxy_url(
+        self, session_id: str, country: str | None = None
+    ) -> str | None:
+        """Return a proxy URL pinned to ``session_id`` when supported.
+
+        Providers without vendor-specific session routing safely fall back to
+        their ordinary proxy URL.
+        """
+        return self.get_geo_proxy_url(country)
+
     def get_location(self) -> str:
         """Return the proxy's configured exit region (e.g. ``"us"``), or ``""``.
 

surfsense_backend/app/utils/proxy/providers/dataimpulse.py

@@ -16,15 +16,13 @@ Example URL::
 
     http://<token>__cr.us:<password>@gw.dataimpulse.com:823
 
-ponytail: sticky sessions (a stable exit IP across requests) are another
-username suffix (``__sid.<id>``) — the lever the Reddit scraper's README flags as
-a TODO for its ``loid``-per-IP flow. Not built yet: Reddit isn't wired to a
-route, so there's no caller to thread a session id through. Add a
-``get_sticky_proxy_url(session_id)`` here (rewriting the username) when it lands.
+Sticky sessions use the vendor's ``__sid.<id>`` username suffix, allowing a
+caller to keep cookies and requests on the same exit IP.
 """
 
 import logging
-from urllib.parse import urlsplit
+import re
+from urllib.parse import quote, urlsplit, urlunsplit
 
 from app.config import Config
 from app.utils.proxy.base import ProxyProvider
@@ -34,6 +32,22 @@ logger = logging.getLogger(__name__)
 # DataImpulse encodes country routing as a "__cr.<country>" username suffix; the
 # country token runs until the next "__" param (e.g. "__sid") or the end.
 _COUNTRY_MARKER = "__cr."
+_COUNTRY_RE = re.compile(r"__cr\.[A-Za-z]{2,}")
+_SESSION_RE = re.compile(r"__sid\.[A-Za-z0-9_-]+")
+
+
+def _safe_session_id(session_id: str) -> str:
+    safe_id = re.sub(r"[^A-Za-z0-9_-]", "-", session_id).strip("-")
+    if not safe_id:
+        raise ValueError("session_id must contain at least one letter or digit")
+    return safe_id
+
+
+def _safe_country(country: str | None) -> str | None:
+    if country is None:
+        return None
+    safe_country = re.sub(r"[^a-z]", "", country.lower())
+    return safe_country or None
 
 
 class DataImpulseProvider(ProxyProvider):
@@ -54,3 +68,41 @@ class DataImpulseProvider(ProxyProvider):
         if _COUNTRY_MARKER not in username:
             return ""
         return username.split(_COUNTRY_MARKER, 1)[1].split("__", 1)[0].lower()
+
+    def _rewrite_proxy_url(
+        self, *, country: str | None = None, session_id: str | None = None
+    ) -> str | None:
+        url = self.get_proxy_url()
+        if not url:
+            return None
+        parts = urlsplit(url)
+        username = _SESSION_RE.sub("", _COUNTRY_RE.sub("", parts.username or ""))
+        safe_country = _safe_country(country)
+        if safe_country is not None:
+            username += f"__cr.{safe_country}"
+        elif _COUNTRY_MARKER in (parts.username or ""):
+            username += f"__cr.{self.get_location()}"
+        if session_id is not None:
+            username += f"__sid.{_safe_session_id(session_id)}"
+        userinfo = quote(username, safe="%")
+        if parts.password is not None:
+            userinfo += f":{quote(parts.password, safe='%')}"
+        host = parts.hostname or ""
+        netloc = f"{userinfo}@{host}"
+        if parts.port:
+            netloc += f":{parts.port}"
+        return urlunsplit(
+            (parts.scheme, netloc, parts.path, parts.query, parts.fragment)
+        )
+
+    def get_geo_proxy_url(self, country: str | None = None) -> str | None:
+        """Return the configured URL with a country-routing suffix when requested."""
+        if not _safe_country(country):
+            return self.get_proxy_url()
+        return self._rewrite_proxy_url(country=country)
+
+    def get_sticky_proxy_url(
+        self, session_id: str, country: str | None = None
+    ) -> str | None:
+        """Return the configured URL with deterministic country/session suffixes."""
+        return self._rewrite_proxy_url(country=country, session_id=session_id)

surfsense_backend/app/utils/proxy/rotation.py

@@ -0,0 +1,34 @@
+"""Cross-country exit rotation for warm-session scrapers.
+
+Some targets (Reddit's ``loid``, TikTok's ``ttwid``) silently withhold their
+anonymous session cookie from the provider's *default worldwide* pool but hand
+it out freely on **country-pinned** exits (proven live: a bare-pool homepage
+hit returns 200 with an empty cookie jar, while a us/gb/de/nl-pinned hit mints
+the cookie every time). A warm-on-block flow that only re-draws from the same
+worldwide pool therefore burns every rotation on cookie-less IPs and fails.
+
+Walking a spread of country pools instead lets the flow escape a wholly-blocked
+pool. The provider's configured country leads (so an operator's choice is
+honoured first); the fallbacks are large, reliable residential pools. Non-geo
+providers (e.g. the custom single-URL provider) ignore the country and re-draw
+their one URL, so this is a harmless no-op there.
+"""
+
+from __future__ import annotations
+
+from app.utils.proxy.registry import get_active_provider
+
+# Walk order after the configured country. Ordered by pool size / reliability.
+FALLBACK_COUNTRIES = ("us", "gb", "de", "ca", "nl", "fr")
+
+
+def rotation_countries() -> tuple[str, ...]:
+    """Ordered, de-duplicated exit countries with the configured one leading."""
+    lead = get_active_provider().get_location()
+    return tuple(dict.fromkeys(c for c in (lead, *FALLBACK_COUNTRIES) if c))
+
+
+def country_for_rotation(n: int) -> str:
+    """Exit country for rotation index ``n`` (cycles the list, wrapping around)."""
+    countries = rotation_countries()
+    return countries[n % len(countries)]

surfsense_backend/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "surf-new-backend"
-version = "0.0.32"
+version = "0.0.34"
 description = "SurfSense Backend"
 requires-python = ">=3.12"
 dependencies = [
@@ -44,7 +44,6 @@ dependencies = [
     "azure-storage-blob>=12.23.0",
     "fake-useragent>=2.2.0",
     "trafilatura>=2.0.0",
-    "captchatools>=1.5.0",
     "fastapi-users[oauth,sqlalchemy]>=15.0.3",
     "chonkie[all]>=1.5.0",
     "langgraph-checkpoint-postgres>=3.0.2",
@@ -86,7 +85,7 @@ dependencies = [
     "opentelemetry-instrumentation-logging>=0.61b0",
     "python-telegram-bot>=22.7",
     "croniter>=2.0.0",
-    "scrapling[fetchers]>=0.4.9",
+    "scrapling[fetchers]>=0.4.11",
 ]
 
 [project.optional-dependencies]

surfsense_backend/scripts/e2e_amazon_scraper.py

@@ -0,0 +1,84 @@
+"""Manual end-to-end check for the public Amazon scraper.
+
+Run from the backend directory:
+
+    uv run python scripts/e2e_amazon_scraper.py
+    uv run python scripts/e2e_amazon_scraper.py --refresh-fixtures
+
+The script requires live network access and the configured residential proxy.
+The optional flag replaces the product and search parser fixtures with current
+live responses. The script is intentionally excluded from pytest.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import sys
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+_BACKEND_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(_BACKEND_ROOT))
+for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"):
+    if _candidate.exists():
+        load_dotenv(_candidate)
+        break
+
+from app.proprietary.platforms.amazon import (  # noqa: E402
+    AmazonScrapeInput,
+    scrape_products,
+)
+from app.proprietary.platforms.amazon.fetch import fetch_page  # noqa: E402
+
+_PRODUCT_URL = "https://www.amazon.com/dp/B09V3KXJPB"
+_SEARCH_URL = "https://www.amazon.com/s?k=wireless+mouse"
+_FIXTURE_DIR = _BACKEND_ROOT / "tests" / "unit" / "platforms" / "amazon" / "fixtures"
+
+
+def _check(label: str, passed: bool) -> bool:
+    print(f"[{'PASS' if passed else 'FAIL'}] {label}")
+    return passed
+
+
+async def main() -> int:
+    product_items = await scrape_products(
+        AmazonScrapeInput(categoryOrProductUrls=[{"url": _PRODUCT_URL}]),
+        limit=1,
+    )
+    product = product_items[0] if product_items else {}
+    print(json.dumps(product, indent=2, ensure_ascii=False)[:3000])
+    product_ok = _check(
+        "product detail has identity and title",
+        bool(product.get("asin") and product.get("title")),
+    )
+
+    search_items = await scrape_products(
+        AmazonScrapeInput(
+            categoryOrProductUrls=[{"url": _SEARCH_URL}],
+            maxItemsPerStartUrl=3,
+            scrapeProductDetails=False,
+        )
+    )
+    search_ok = _check(
+        "search returns product cards",
+        bool(search_items)
+        and all(
+            item.get("asin") and item.get("categoryPageData") for item in search_items
+        ),
+    )
+    fixture_ok = True
+    if "--refresh-fixtures" in sys.argv:
+        _FIXTURE_DIR.mkdir(parents=True, exist_ok=True)
+        for name, url in (("product.html", _PRODUCT_URL), ("search.html", _SEARCH_URL)):
+            response = await fetch_page(url)
+            saved = response is not None and response.status == 200
+            if saved:
+                (_FIXTURE_DIR / name).write_text(response.html, encoding="utf-8")
+            fixture_ok &= _check(f"refreshed {name}", saved)
+    return 0 if product_ok and search_ok and fixture_ok else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(asyncio.run(main()))

surfsense_backend/scripts/e2e_google_search.py

@@ -28,9 +28,12 @@ sys.path.insert(0, str(_ROOT))
 load_dotenv(_ROOT / ".env")
 
 logging.basicConfig(level=logging.WARNING)
-logging.getLogger("app.proprietary.platforms.google_search.scraper").setLevel(
-    logging.INFO
-)
+for _name in (
+    "app.proprietary.platforms.google_search.scraper",
+    "app.proprietary.platforms.google_search.fetch",
+    "app.proprietary.platforms.google_search.captcha",
+):
+    logging.getLogger(_name).setLevel(logging.INFO)
 
 from app.proprietary.platforms.google_search import (  # noqa: E402
     GoogleSearchScrapeInput,
@@ -146,6 +149,9 @@ async def run(
 
 _CASES = {
     "plain": lambda: run("plain query", queries="python asyncio tutorial"),
+    # Prod incident 2026-07-17: single-word brand/navigational queries were
+    # exhausting all 24 IPs (precheck 200 but the browser render 429-walled).
+    "brand": lambda: run("brand query (notebooklm, prod repro)", queries="notebooklm"),
     "site": lambda: run("site: filter", queries="machine learning", site="arxiv.org"),
     "ads": lambda: run("text ads", queries="car insurance quotes", expect_ads=True),
     "products": lambda: run(

surfsense_backend/scripts/e2e_instagram_scraper.py

@@ -51,8 +51,14 @@ from app.proprietary.platforms.instagram.fetch import (  # noqa: E402
 )
 from app.proprietary.platforms.instagram.url_resolver import resolve_url  # noqa: E402
 
-_PROFILE = "natgeo"
-_SEARCH_TERM = "national geographic"
+# Canonical public targets. Override from the CLI to test any real-world case:
+#   python scripts/e2e_instagram_scraper.py <profile> [search term]
+# Note: web_profile_info intermittently 400s for *business/creator* accounts
+# (IG server bug on the ig_business_category_subvertical schema); a regular
+# public account is the reliable smoke target.
+_DEFAULT_PROFILE = "natgeo"
+_PROFILE = sys.argv[1] if len(sys.argv) > 1 else _DEFAULT_PROFILE
+_SEARCH_TERM = sys.argv[2] if len(sys.argv) > 2 else "national geographic"
 
 _FIXTURE_DIR = _BACKEND_ROOT / "tests" / "unit" / "platforms" / "instagram" / "fixtures"
 
@@ -179,6 +185,12 @@ async def step5_search() -> bool:
 
 async def step6_dump_fixtures(post_url: str | None) -> bool:
     _hr("STEP 6 — dump trimmed, anonymized fixtures for offline tests")
+    if _PROFILE != _DEFAULT_PROFILE:
+        return _check(
+            "dumped fixtures",
+            True,
+            f"skipped (custom profile {_PROFILE!r} would clobber committed fixtures)",
+        )
     profile = await fetch_json("api/v1/users/web_profile_info/", {"username": _PROFILE})
     _FIXTURE_DIR.mkdir(parents=True, exist_ok=True)
     wrote = []

surfsense_backend/scripts/e2e_reddit_scraper.py

@@ -9,10 +9,10 @@ This is NOT a pytest test (it needs live network + a residential/custom proxy).
 It:
 
   Step 0 — go/no-go probe (folds in the old scripts/reddit_probe.py): open a
-      proxy session, warm a ``loid`` (svc/shreddit first, old.reddit fallback),
-      then do sequential ``.json`` fetches on the SAME sticky IP and assert each
-      returns a Reddit Listing. If this fails the whole approach is invalid —
-      later steps are skipped.
+      proxy session, browser-warm a ``loid`` cookie jar on the sticky exit IP,
+      then do sequential plain-HTTP ``.json`` fetches on the SAME sticky IP
+      (replaying the jar) and assert each returns a Reddit Listing. If this fails
+      the whole approach is invalid — later steps are skipped.
   Step 1 — scrape a discovered post URL (post + a few comments).
   Step 2 — scrape a subreddit listing.
   Step 3 — run a search query.
@@ -96,8 +96,11 @@ async def step0_probe() -> bool:
             return _check(
                 "proxy configured", False, "no proxy -> set PROXY_PROVIDER + creds"
             )
-        minted = await warm_session(holder.session)
+        jar = await warm_session(holder.proxy)
+        if jar:
+            holder.cookies = jar
         holder.warmed = True  # don't let fetch_json re-warm; we just warmed it
+        minted = bool(jar)
         _check("loid warm-up minted a session", minted)
         oks: list[bool] = []
         for path in (f"r/{_SUBREDDIT}/hot", "r/programming/new", f"r/{_SUBREDDIT}/hot"):

surfsense_backend/scripts/e2e_tiktok_scrape.py

@@ -39,6 +39,12 @@ from urllib.parse import urlsplit
 from dotenv import load_dotenv
 
 # --- bootstrap: load .env and put the backend root on sys.path before app.* ---
+# Scraped captions carry arbitrary Unicode (emoji, CJK, decorative glyphs); the
+# Windows cp1252 console can't encode it and would abort the whole run on a single
+# character. Emit UTF-8 and replace anything un-encodable instead of crashing.
+if hasattr(sys.stdout, "reconfigure"):
+    sys.stdout.reconfigure(encoding="utf-8", errors="replace")
+
 _BACKEND_ROOT = Path(__file__).resolve().parent.parent
 sys.path.insert(0, str(_BACKEND_ROOT))
 for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"):
@@ -50,7 +56,7 @@ _FIXTURES = _BACKEND_ROOT / "tests" / "fixtures" / "tiktok"
 
 # Evergreen public targets: a regular high-volume creator, a broad hashtag, and
 # a common search term.
-_PROFILE = "nasa"
+_PROFILE = "tiktok"
 _HASHTAG = "food"
 _SEARCH = "meal prep"
 _COUNT = 5

surfsense_backend/scripts/scale_google_search.py

@@ -0,0 +1,244 @@
+"""Zero-cost concurrency simulation for the Google Search fetch seam.
+
+Drives many concurrent ``fetch_serp_html`` calls with the network I/O stubbed
+out (no live Google, no proxy spend, no captcha solves) so we can measure the
+*structure* of the scraper under load without paying for it:
+
+* **throughput** — completions/sec at steady state (the per-process ceiling is
+  the render gate divided by warm-render latency; nothing above it is real),
+* **latency** — p50/p95/p99 wait as the gate queues excess renders,
+* **solves** — how many paid solves the run would cost (the number we most want
+  the sticky-IP pool to shrink), and
+* **per-IP peak concurrency** — the funneling metric: how many renders pile onto
+  a single sticky IP at once (a hot IP is what Google re-walls).
+
+The stub keeps the *real* ``fetch_serp_html`` loop intact (sticky reuse,
+exemption skip-precheck, the render gate, inflight accounting) and only replaces
+``_precheck`` / ``_get_session`` / ``get_proxy_url``. Timings are compressed
+(warm≈0.1 s vs ~14 s live) and extrapolated to real seconds via the warm-render
+ratio, since the system is gate-bounded sleeps + a semaphore (delays scale
+linearly).
+
+    .venv/Scripts/python.exe scripts/scale_google_search.py --count 400 --rate 60
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import sys
+import threading
+import time
+from pathlib import Path
+
+if hasattr(sys.stdout, "reconfigure"):
+    sys.stdout.reconfigure(encoding="utf-8")
+
+_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(_ROOT))
+
+from app.proprietary.platforms.google_search import fetch  # noqa: E402
+
+# Live-measured render latencies (README "Timings"); used only to translate the
+# compressed sim clock back to real seconds for the human-readable report.
+_REAL_WARM_S = 14.0
+_REAL_SOLVE_S = 45.0
+
+_RESULTS_HTML = '<div id="rso">ok</div>'  # trips fetch._has_results
+
+
+class _Metrics:
+    """Browser-loop-side counters (fetch runs on two loops → guard with a lock)."""
+
+    def __init__(self) -> None:
+        self.lock = threading.Lock()
+        self.solves = 0
+        self.renders = 0
+        self.ip_hits: dict[str, int] = {}
+        self.ip_now: dict[str, int] = {}
+        self.ip_peak: dict[str, int] = {}
+
+    def enter(self, proxy: str, solved: bool) -> None:
+        with self.lock:
+            self.renders += 1
+            if solved:
+                self.solves += 1
+            self.ip_hits[proxy] = self.ip_hits.get(proxy, 0) + 1
+            n = self.ip_now.get(proxy, 0) + 1
+            self.ip_now[proxy] = n
+            self.ip_peak[proxy] = max(self.ip_peak.get(proxy, 0), n)
+
+    def exit(self, proxy: str) -> None:
+        with self.lock:
+            self.ip_now[proxy] = self.ip_now.get(proxy, 1) - 1
+
+
+class _FakePage:
+    def __init__(self) -> None:
+        self.status = 200
+        self.html_content = _RESULTS_HTML
+
+
+class _FakeSession:
+    """Stands in for AsyncStealthySession: sleeps like a render, simulates the
+    solve on a cold IP by seeding fetch._exemption_jar (what page_action does)."""
+
+    def __init__(self, metrics: _Metrics, warm_s: float, solve_s: float) -> None:
+        self.m = metrics
+        self.warm_s = warm_s
+        self.solve_s = solve_s
+
+    async def fetch(self, url, proxy=None, **kwargs):
+        key = proxy or ""
+        # Cold IP (no cached exemption) pays the solve and warms the jar; a warm
+        # IP just pays the render. Mirrors _make_page_action's cache write.
+        cold = key not in fetch._exemption_jar
+        self.m.enter(key, solved=cold)
+        try:
+            if cold:
+                await asyncio.sleep(self.solve_s)
+                fetch._exemption_jar[key] = [{"name": "GOOGLE_ABUSE_EXEMPTION"}]
+            await asyncio.sleep(self.warm_s)
+            return _FakePage()
+        finally:
+            self.m.exit(key)
+
+
+def _install_stubs(metrics: _Metrics, warm_s: float, solve_s: float, precheck_s: float):
+    base = "http://u:p@gw.dataimpulse.com:823"  # hostname in fetch._STICKY_HOSTS
+    session = _FakeSession(metrics, warm_s, solve_s)
+
+    async def fake_precheck(url, proxy):
+        await asyncio.sleep(precheck_s)
+        return True
+
+    async def fake_get_session(mobile):
+        return session
+
+    fetch.get_proxy_url = lambda: base
+    fetch._precheck = fake_precheck
+    fetch._get_session = fake_get_session
+    fetch.captcha_enabled = lambda: True
+    fetch.get_captcha_config = lambda: object()
+    fetch._captcha.solver_latched = lambda: False
+    # The pool's backpressure poll is negligible vs a real 14 s render; keep it
+    # proportional under the compressed sim clock so waiters don't idle.
+    fetch._POOL_WAIT_S = max(warm_s * 0.05, 0.001)
+
+    # Isolate the sim to the LOCAL pool: the cross-process store (Redis) has its
+    # own unit test and would otherwise add ping latency / cross-run bleed here.
+    async def _no_adopt(exclude):
+        return None
+
+    async def _noop(*a, **k):
+        return None
+
+    fetch._store.adopt = _no_adopt
+    fetch._store.publish = _noop
+    fetch._store.evict = _noop
+
+
+def _reset_state() -> None:
+    fetch._exemption_jar.clear()
+    for name in ("_pool", "_pool_inflight"):
+        obj = getattr(fetch, name, None)
+        if isinstance(obj, dict):
+            obj.clear()
+    if hasattr(fetch, "_pool_pending"):
+        fetch._pool_pending = 0
+    if hasattr(fetch, "_last_good_proxy"):  # pre-pool revision
+        fetch._last_good_proxy = None
+
+
+async def _drive(count: int, rate: float) -> list[float]:
+    """Fire ``count`` fetches at ``rate``/sec (steady arrival), return latencies."""
+    lat: list[float] = []
+    interval = 1.0 / rate if rate > 0 else 0.0
+
+    async def one() -> None:
+        t0 = time.perf_counter()
+        await fetch.fetch_serp_html("https://www.google.com/search?q=notebooklm")
+        lat.append(time.perf_counter() - t0)
+
+    tasks: list[asyncio.Task] = []
+    for _ in range(count):
+        tasks.append(asyncio.create_task(one()))
+        if interval:
+            await asyncio.sleep(interval)
+    await asyncio.gather(*tasks)
+    return lat
+
+
+def _pct(xs: list[float], p: float) -> float:
+    if not xs:
+        return 0.0
+    xs = sorted(xs)
+    return xs[min(len(xs) - 1, int(p / 100 * len(xs)))]
+
+
+async def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--count", type=int, default=400)
+    ap.add_argument("--rate", type=float, default=60.0, help="arrivals/sec (sim clock)")
+    ap.add_argument(
+        "--warm", type=float, default=0.10, help="warm render seconds (sim)"
+    )
+    ap.add_argument("--solve", type=float, default=0.45, help="solve seconds (sim)")
+    ap.add_argument("--precheck", type=float, default=0.01)
+    args = ap.parse_args()
+
+    metrics = _Metrics()
+    _install_stubs(metrics, args.warm, args.solve, args.precheck)
+    _reset_state()
+
+    scale = _REAL_WARM_S / args.warm  # sim seconds → real seconds
+    gate = fetch._MAX_CONCURRENT_PAGES
+    pool = getattr(fetch, "_WARM_POOL_TARGET", None)
+
+    wall0 = time.perf_counter()
+    lat = await _drive(args.count, args.rate)
+    wall = time.perf_counter() - wall0
+    await fetch._in_browser_loop(asyncio.sleep(0))  # let last exit() settle
+
+    hot_peak = max(metrics.ip_peak.values()) if metrics.ip_peak else 0
+    top_hits = max(metrics.ip_hits.values()) if metrics.ip_hits else 0
+    top_share = 100.0 * top_hits / metrics.renders if metrics.renders else 0.0
+    thru_sim = args.count / wall if wall else 0.0
+    thru_real = thru_sim / scale
+
+    print("\n=== Google Search scale simulation ===")
+    print(
+        f"  gate (_MAX_CONCURRENT_PAGES) = {gate}"
+        + (f", warm-pool target = {pool}" if pool else "  (single-slot sticky IP)")
+    )
+    print(f"  requests={args.count} arrival_rate={args.rate}/s (sim)")
+    print("  --- structure (scale-free) ---")
+    print(
+        f"  paid solves           = {metrics.solves}  (want ~pool size, not ~requests)"
+    )
+    print(f"  distinct sticky IPs   = {len(metrics.ip_hits)}")
+    print(
+        f"  busiest IP carried    = {top_hits}/{metrics.renders} renders "
+        f"({top_share:.0f}%)  (funneling; want ~even spread)"
+    )
+    print(f"  peak renders on 1 IP  = {hot_peak}  (concurrency; capped by gate)")
+    print(
+        f"  --- throughput / latency (extrapolated to live @ warm={_REAL_WARM_S}s) ---"
+    )
+    print(f"  throughput  = {thru_real * 60:.0f} SERP/min  ({thru_real:.2f}/s)")
+    print(
+        f"  latency p50 = {_pct(lat, 50) * scale:6.1f}s   p95 = {_pct(lat, 95) * scale:6.1f}s   "
+        f"p99 = {_pct(lat, 99) * scale:6.1f}s"
+    )
+    print(
+        f"  ceiling (gate/warm) = {gate / _REAL_WARM_S * 60:.0f} SERP/min per process"
+    )
+    need = 500 / (gate / _REAL_WARM_S * 60)
+    print(
+        f"  -> to sustain 500 SERP/min you need ~{need:.0f} such processes "
+        f"(or a larger gate)\n"
+    )
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

surfsense_backend/scripts/stress_google_search.py

@@ -0,0 +1,215 @@
+"""Live stress test for the Google Search scraper (REAL solves + proxy spend).
+
+Simulates a burst of concurrent "users" hitting the real ``scrape_serps``
+pipeline with a mix of single-query and multi-query requests. Unlike the offline
+sim (``scale_google_search.py``), this exercises the actual DataImpulse IPs,
+CapSolver solves, and the shared browser — so it surfaces what the sim can't:
+real warm-vs-cold latency, whether the warm-IP pool amortizes solves across real
+IPs without Google re-walling them, and browser stability under concurrency.
+
+    .venv/Scripts/python.exe scripts/stress_google_search.py --users 30 --gate 8
+
+It COSTS money (each cold IP = one CapSolver solve, bounded by the pool target)
+and proxy bandwidth. Metrics are tallied from the live perf/captcha logs.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import logging
+import os
+import random
+import re
+import sys
+import time
+from pathlib import Path
+
+if hasattr(sys.stdout, "reconfigure"):
+    sys.stdout.reconfigure(encoding="utf-8")
+
+_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(_ROOT))
+
+from dotenv import load_dotenv  # noqa: E402
+
+load_dotenv(_ROOT / ".env")
+
+# Brand/navigational terms reliably trip the reCAPTCHA wall (a solve/pool grow);
+# informational terms are the lighter, often-warm path. A realistic mix.
+_BRAND = ["notebooklm", "figma", "notion", "linear app", "vercel", "perplexity ai"]
+_INFO = [
+    "python asyncio tutorial",
+    "best mechanical keyboard 2026",
+    "kubernetes vs docker",
+    "typescript generics explained",
+    "how to make sourdough bread",
+    "rust ownership model",
+]
+
+
+class _LogTally(logging.Handler):
+    """Counts solves / renders / walls / pool-reuse from the live log stream."""
+
+    _RENDER = re.compile(r"has_results=(\w+).*from_pool=(\w+) pool=(\d+)")
+    _SOLVE = re.compile(r"\[captcha\] solve (OK|did not)")
+
+    def __init__(self) -> None:
+        super().__init__(level=logging.INFO)
+        self.solves_ok = 0
+        self.solves_fail = 0
+        self.renders_ok = 0
+        self.renders_walled = 0
+        self.reuse = 0
+        self.grow = 0
+        self.max_pool = 0
+
+    def emit(self, record: logging.LogRecord) -> None:
+        msg = record.getMessage()
+        m = self._RENDER.search(msg)
+        if m:
+            ok, from_pool, pool = m.group(1), m.group(2), int(m.group(3))
+            if ok == "True":
+                self.renders_ok += 1
+            else:
+                self.renders_walled += 1
+            if from_pool == "True":
+                self.reuse += 1
+            else:
+                self.grow += 1
+            self.max_pool = max(self.max_pool, pool)
+            return
+        s = self._SOLVE.search(msg)
+        if s:
+            if s.group(1) == "OK":
+                self.solves_ok += 1
+            else:
+                self.solves_fail += 1
+
+
+def _make_request(rng: random.Random, multi_ratio: float) -> tuple[str, int]:
+    """Build one user's request: a single term or a 2-3 term multi-query.
+    Returns (newline-joined queries, expected SERP count)."""
+    pool = _BRAND + _INFO
+    if rng.random() < multi_ratio:
+        n = rng.randint(2, 3)
+        terms = rng.sample(pool, n)
+        return "\n".join(terms), n
+    return rng.choice(pool), 1
+
+
+async def _user(uid: int, queries: str, want: int, results: list[dict]) -> None:
+    from app.proprietary.platforms.google_search import (
+        GoogleSearchScrapeInput,
+        scrape_serps,
+    )
+
+    t0 = time.perf_counter()
+    inp = GoogleSearchScrapeInput(queries=queries, countryCode="us", languageCode="en")
+    try:
+        items = await scrape_serps(inp, limit=want)
+        got = len(items)
+        err = None
+    except Exception as e:  # a user request should never crash the whole run
+        got, err = 0, repr(e)
+    results.append(
+        {
+            "uid": uid,
+            "want": want,
+            "got": got,
+            "secs": time.perf_counter() - t0,
+            "err": err,
+        }
+    )
+
+
+async def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--users", type=int, default=30, help="concurrent requests")
+    ap.add_argument("--gate", type=int, default=8, help="MAX_CONCURRENT_PAGES for run")
+    ap.add_argument("--multi-ratio", type=float, default=0.5)
+    ap.add_argument("--ramp", type=float, default=0.3, help="secs between user starts")
+    ap.add_argument("--seed", type=int, default=7)
+    args = ap.parse_args()
+
+    # Must be set BEFORE fetch.py is imported (the gate is read at import).
+    os.environ["GOOGLE_SEARCH_MAX_CONCURRENT_PAGES"] = str(args.gate)
+
+    logging.basicConfig(level=logging.WARNING)
+    tally = _LogTally()
+    for name in (
+        "app.proprietary.platforms.google_search.fetch",
+        "app.proprietary.platforms.google_search.captcha",
+    ):
+        lg = logging.getLogger(name)
+        lg.setLevel(logging.INFO)
+        lg.addHandler(tally)
+
+    from app.proprietary.platforms.google_search.fetch import (
+        _WARM_POOL_TARGET,
+        close_sessions,
+    )
+
+    rng = random.Random(args.seed)
+    plans = [_make_request(rng, args.multi_ratio) for _ in range(args.users)]
+    total_serps = sum(w for _, w in plans)
+    n_multi = sum(1 for _, w in plans if w > 1)
+
+    print(
+        f"\n=== LIVE stress: {args.users} users "
+        f"({args.users - n_multi} single / {n_multi} multi), "
+        f"~{total_serps} SERPs, gate={args.gate}, pool_target={_WARM_POOL_TARGET} ==="
+    )
+    print("  (real solves + proxy spend; warming up...)\n")
+
+    results: list[dict] = []
+    wall0 = time.perf_counter()
+    tasks = []
+    for uid, (queries, want) in enumerate(plans):
+        tasks.append(asyncio.create_task(_user(uid, queries, want, results)))
+        await asyncio.sleep(args.ramp)  # gentle ramp, not a thundering herd
+    await asyncio.gather(*tasks)
+    wall = time.perf_counter() - wall0
+
+    await close_sessions()
+
+    lat = sorted(r["secs"] for r in results)
+    got_serps = sum(r["got"] for r in results)
+    fails = [r for r in results if r["err"]]
+    short = [r for r in results if not r["err"] and r["got"] < r["want"]]
+
+    def pct(p: float) -> float:
+        return lat[min(len(lat) - 1, int(p / 100 * len(lat)))] if lat else 0.0
+
+    print("=== results ===")
+    print(f"  wall time            = {wall:.0f}s")
+    print(f"  requests ok/failed   = {len(results) - len(fails)}/{len(fails)}")
+    print(
+        f"  SERPs got/expected   = {got_serps}/{total_serps}"
+        + (f"  ({len(short)} short)" if short else "")
+    )
+    print(f"  throughput           = {got_serps / wall * 60:.0f} SERP/min")
+    print(
+        f"  request latency p50={pct(50):.0f}s  p95={pct(95):.0f}s  "
+        f"max={lat[-1] if lat else 0:.0f}s"
+    )
+    print("  --- pipeline (from live logs) ---")
+    print(
+        f"  paid solves ok/fail  = {tally.solves_ok}/{tally.solves_fail}  "
+        f"(bounded by pool target {_WARM_POOL_TARGET})"
+    )
+    print(f"  renders ok/walled    = {tally.renders_ok}/{tally.renders_walled}")
+    print(
+        f"  pool reuse/grow      = {tally.reuse}/{tally.grow}  "
+        f"(reuse share {100 * tally.reuse / max(1, tally.reuse + tally.grow):.0f}%)"
+    )
+    print(f"  peak pool size       = {tally.max_pool}")
+    if fails:
+        print("  --- failures ---")
+        for r in fails[:5]:
+            print(f"    user{r['uid']}: {r['err']}")
+    print()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py

@@ -27,6 +27,7 @@ pytestmark = pytest.mark.unit
 # specialist is a deliberate product change and must be reflected here.
 _EXPECTED_SUBAGENTS = frozenset(
     {
+        "amazon",
         "deliverables",
         "dropbox",
         "google_drive",

surfsense_backend/tests/unit/capabilities/amazon/__init__.py

@@ -0,0 +1 @@
+

surfsense_backend/tests/unit/capabilities/amazon/scrape/__init__.py

@@ -0,0 +1 @@
+

surfsense_backend/tests/unit/capabilities/amazon/scrape/test_executor.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+from app.capabilities.amazon.scrape.executor import build_scrape_executor
+from app.capabilities.amazon.scrape.schemas import MAX_AMAZON_RESULTS, ScrapeInput
+from app.proprietary.platforms.amazon import AmazonScrapeInput
+
+
+class _FakeScraper:
+    def __init__(self) -> None:
+        self.calls: list[tuple[AmazonScrapeInput, int | None]] = []
+
+    async def __call__(
+        self, input_model: AmazonScrapeInput, *, limit: int | None = None
+    ) -> list[dict]:
+        self.calls.append((input_model, limit))
+        return [{"asin": "B09V3KXJPB", "title": "Product"}]
+
+
+async def test_executor_maps_agent_input_to_scraper_input():
+    scraper = _FakeScraper()
+    execute = build_scrape_executor(scraper)
+
+    output = await execute(
+        ScrapeInput(
+            search_terms=["wireless mouse"],
+            urls=["https://www.amazon.com/dp/B09V3KXJPB"],
+            max_items=5,
+            max_offers=2,
+            include_sellers=True,
+            zip_code="10001",
+            country_code="US",
+        )
+    )
+
+    assert output.items[0].asin == "B09V3KXJPB"
+    input_model, limit = scraper.calls[0]
+    assert input_model.categoryOrProductUrls == [
+        {"url": "https://www.amazon.com/dp/B09V3KXJPB"},
+        {"url": "https://www.amazon.com/s?k=wireless+mouse"},
+    ]
+    assert input_model.maxItemsPerStartUrl == 5
+    assert input_model.maxOffers == 2
+    assert input_model.scrapeSellers is True
+    assert input_model.zipCode == "10001"
+    assert limit == MAX_AMAZON_RESULTS

surfsense_backend/tests/unit/capabilities/amazon/scrape/test_schemas.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+import pytest
+from pydantic import ValidationError
+
+from app.capabilities.amazon.scrape.schemas import (
+    MAX_AMAZON_RESULTS,
+    ScrapeInput,
+    ScrapeOutput,
+)
+
+
+def test_estimated_units_cover_search_and_direct_variant_fanout():
+    payload = ScrapeInput(
+        search_terms=["mouse", "keyboard"],
+        urls=["https://www.amazon.com/dp/B09V3KXJPB"],
+        max_items=20,
+        max_variants=3,
+    )
+
+    assert payload.estimated_units == 44
+
+
+def test_estimated_units_respect_hard_run_ceiling():
+    payload = ScrapeInput(search_terms=["x"] * 20, max_items=100)
+    assert payload.estimated_units == MAX_AMAZON_RESULTS
+
+
+def test_at_least_one_source_is_required():
+    with pytest.raises(ValidationError):
+        ScrapeInput()
+
+
+def test_error_items_are_not_billable():
+    output = ScrapeOutput(
+        items=[
+            {"asin": "B09V3KXJPB", "title": "Product"},
+            {"error": "product_not_found", "errorDescription": "Missing"},
+        ]
+    )
+
+    assert output.billable_units == 1

surfsense_backend/tests/unit/capabilities/amazon/test_registry.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+from app.capabilities.amazon.scrape.schemas import ScrapeInput, ScrapeOutput
+from app.capabilities.core import BillingUnit
+from app.capabilities.core.store import get_capability
+
+
+def test_amazon_scrape_is_registered_and_metered():
+    capability = get_capability("amazon.scrape")
+
+    assert capability.input_schema is ScrapeInput
+    assert capability.output_schema is ScrapeOutput
+    assert capability.billing_unit is BillingUnit.AMAZON_PRODUCT

surfsense_backend/tests/unit/capabilities/google_maps/test_registry.py

@@ -7,15 +7,15 @@ import pytest
 from app.capabilities import (
     google_maps,  # noqa: F401  — importing the namespace registers its verbs
 )
+from app.capabilities.core import BillingUnit
 from app.capabilities.core.store import get_capability
-from app.capabilities.core.types import BillingUnit
 from app.capabilities.google_maps.reviews.schemas import ReviewsInput, ReviewsOutput
 from app.capabilities.google_maps.scrape.schemas import ScrapeInput, ScrapeOutput
 
 pytestmark = pytest.mark.unit
 
 
-def test_google_maps_scrape_is_registered_and_billed_per_place():
+def test_google_maps_scrape_is_registered_and_billable():
     cap = get_capability("google_maps.scrape")
 
     assert cap.name == "google_maps.scrape"
@@ -24,7 +24,7 @@ def test_google_maps_scrape_is_registered_and_billed_per_place():
     assert cap.billing_unit is BillingUnit.GOOGLE_MAPS_PLACE
 
 
-def test_google_maps_reviews_is_registered_and_billed_per_review():
+def test_google_maps_reviews_is_registered_and_billable():
     cap = get_capability("google_maps.reviews")
 
     assert cap.name == "google_maps.reviews"

surfsense_backend/tests/unit/capabilities/reddit/test_registry.py

@@ -7,14 +7,14 @@ import pytest
 from app.capabilities import (
     reddit,  # noqa: F401  — importing the namespace registers its verbs
 )
+from app.capabilities.core import BillingUnit
 from app.capabilities.core.store import get_capability
-from app.capabilities.core.types import BillingUnit
 from app.capabilities.reddit.scrape.schemas import ScrapeInput, ScrapeOutput
 
 pytestmark = pytest.mark.unit
 
 
-def test_reddit_scrape_is_registered_and_billed_per_item():
+def test_reddit_scrape_is_registered_and_billable():
     cap = get_capability("reddit.scrape")
 
     assert cap.name == "reddit.scrape"

surfsense_backend/tests/unit/capabilities/youtube/test_registry.py

@@ -7,15 +7,15 @@ import pytest
 from app.capabilities import (
     youtube,  # noqa: F401  — importing the namespace registers its verbs
 )
+from app.capabilities.core import BillingUnit
 from app.capabilities.core.store import get_capability
-from app.capabilities.core.types import BillingUnit
 from app.capabilities.youtube.comments.schemas import CommentsInput, CommentsOutput
 from app.capabilities.youtube.scrape.schemas import ScrapeInput, ScrapeOutput
 
 pytestmark = pytest.mark.unit
 
 
-def test_youtube_scrape_is_registered_and_billed_per_video():
+def test_youtube_scrape_is_registered_and_billable():
     cap = get_capability("youtube.scrape")
 
     assert cap.name == "youtube.scrape"
@@ -24,7 +24,7 @@ def test_youtube_scrape_is_registered_and_billed_per_video():
     assert cap.billing_unit is BillingUnit.YOUTUBE_VIDEO
 
 
-def test_youtube_comments_is_registered_and_billed_per_comment():
+def test_youtube_comments_is_registered_and_billable():
     cap = get_capability("youtube.comments")
 
     assert cap.name == "youtube.comments"

surfsense_backend/tests/unit/platforms/amazon/fixtures/bestsellers.html

@@ -0,0 +1,17 @@
+<!doctype html>
+<html lang="en">
+  <body>
+    <div id="gridItemRoot">
+      <span class="zg-bdg-text">#1</span>
+      <a href="/dp/B09V3KXJPB"><span class="_cDEzb_p13n-sc-css-line-clamp-3_g3dy1">Example Wireless Headphones</span></a>
+      <span class="a-price"><span class="a-offscreen">$49.99</span></span>
+      <span class="a-icon-alt">4.6 out of 5 stars</span>
+      <img src="https://images.example/headphones.jpg" alt="Example Wireless Headphones">
+    </div>
+    <div id="gridItemRoot">
+      <span class="zg-bdg-text">#2</span>
+      <a href="/dp/B09V3KXJP1"><span class="_cDEzb_p13n-sc-css-line-clamp-3_g3dy1">Example Earbuds</span></a>
+      <span class="a-price"><span class="a-offscreen">$29.99</span></span>
+    </div>
+  </body>
+</html>

surfsense_backend/tests/unit/platforms/amazon/fixtures/blocked.html

@@ -0,0 +1,9 @@
+<!doctype html>
+<html lang="en">
+  <head><title>Robot Check</title></head>
+  <body>
+    <form action="/errors/validateCaptcha">
+      <p>Enter the characters you see below</p>
+    </form>
+  </body>
+</html>

surfsense_backend/tests/unit/platforms/amazon/fixtures/offers.html

@@ -0,0 +1,17 @@
+<!doctype html>
+<html lang="en">
+  <body>
+    <div id="aod-offer">
+      <div id="aod-pinned-offer"></div>
+      <div id="aod-offer-heading">New</div>
+      <span class="a-price"><span class="a-offscreen">$47.50</span></span>
+      <div id="aod-offer-soldBy"><a href="/sp?seller=SELLER123">Example Retailer</a></div>
+      <div id="mir-layout-DELIVERY_BLOCK">FREE delivery tomorrow</div>
+    </div>
+    <div id="aod-offer">
+      <div id="aod-offer-heading">Used - Very Good</div>
+      <span class="a-price"><span class="a-offscreen">$39.00</span></span>
+      <div id="aod-offer-soldBy"><a href="/sp?seller=SELLER456">Second Seller</a></div>
+    </div>
+  </body>
+</html>

surfsense_backend/tests/unit/platforms/amazon/fixtures/product.html

@@ -0,0 +1,51 @@
+<!doctype html>
+<html lang="en">
+  <body>
+    <div id="wayfinding-breadcrumbs_feature_div"><ul><li><a href="/b?node=172282">Electronics</a></li><li><a href="/b?node=667846011">Audio</a></li></ul></div>
+    <span id="productTitle">Example Wireless Headphones</span>
+    <a id="bylineInfo" href="/stores/Example">Visit the Example Store</a>
+    <a id="sellerProfileTriggerId" href="/sp?seller=SELLER123">Example Retailer</a>
+    <div id="corePrice_feature_div">
+      <span class="priceToPay"><span class="a-offscreen">$49.99</span></span>
+      <span class="basisPrice"><span class="a-offscreen">$79.99</span></span>
+    </div>
+    <div id="averageCustomerReviews">
+      <span id="acrPopover">4.6 out of 5 stars</span>
+      <span id="acrCustomerReviewText">1,234 ratings</span>
+      <a id="acrCustomerReviewLink" href="/product-reviews/B09V3KXJPB">Reviews</a>
+    </div>
+    <table id="histogramTable">
+      <tr><td class="a-text-left">5 star</td><td class="a-text-right">80%</td></tr>
+      <tr><td class="a-text-left">4 star</td><td class="a-text-right">12%</td></tr>
+    </table>
+    <div id="availability">In Stock</div>
+    <div id="social-proofing-faceout-title-tk_bought">2K+ bought in past month</div>
+    <div id="feature-bullets"><ul><li><span class="a-list-item">Long battery life</span></li><li><span class="a-list-item">Noise cancelling</span></li></ul></div>
+    <div id="productDescription">Detailed product description.</div>
+    <img id="landingImage" src="https://images.example/thumb.jpg" data-old-hires="https://images.example/full.jpg" alt="Example Wireless Headphones">
+    <div id="aplus"><p>Premium materials and comfort.</p><img src="https://images.example/aplus.jpg" alt="Premium materials and comfort"></div>
+    <div id="brandStory_feature_div">Designed for everyday listening.</div>
+    <div id="detailBulletsWrapper_feature_div"><ul><li>Best Sellers Rank: #12 in Electronics #3 in Headphones</li></ul></div>
+    <table id="productDetails_techSpec_section_1"><tr><th>Model</th><td>WH-100</td></tr></table>
+    <div id="productOverview_feature_div"><table><tr><td>Color</td><td>Black</td></tr></table></div>
+    <div id="acBadge_feature_div">Amazon's Choice</div>
+    <script>
+      window.twister = {
+        "dimensionValuesDisplayData": {
+          "B09V3KXJPB": ["Black"],
+          "B09V3KXJP1": ["Blue"]
+        },
+        "variationValues": {"color_name": ["Black", "Blue"]}
+      };
+    </script>
+    <div data-hook="review" id="R1">
+      <span class="a-profile-name">Alex</span>
+      <a data-hook="review-title" href="/gp/customer-reviews/R1">Excellent sound</a>
+      <i data-hook="review-star-rating">5.0 out of 5 stars</i>
+      <span data-hook="review-date">Reviewed on July 1, 2026</span>
+      <span data-hook="review-body">Clear and comfortable.</span>
+      <span data-hook="avp-badge">Verified Purchase</span>
+      <span data-hook="helpful-vote-statement">12 people found this helpful</span>
+    </div>
+  </body>
+</html>

surfsense_backend/tests/unit/platforms/amazon/fixtures/search.html

@@ -0,0 +1,19 @@
+<!doctype html>
+<html lang="en">
+  <body>
+    <div data-component-type="s-search-result" data-asin="B09V3KXJPB" class="s-result-item">
+      <h2><a href="/dp/B09V3KXJPB"><span>Example Wireless Headphones</span></a></h2>
+      <span class="a-price"><span class="a-offscreen">$49.99</span></span>
+      <span class="a-icon-alt">4.6 out of 5 stars</span>
+      <span class="s-underline-text">1,234</span>
+      <img class="s-image" src="https://images.example/headphones.jpg" alt="Example Wireless Headphones">
+      <span class="a-badge-text">Best Seller</span>
+    </div>
+    <div data-component-type="s-search-result" data-asin="B09V3KXJP1" class="s-result-item">
+      <span>Sponsored</span>
+      <h2><a href="/dp/B09V3KXJP1"><span>Example Earbuds</span></a></h2>
+      <span class="a-price"><span class="a-offscreen">$29.99</span></span>
+      <span class="a-icon-alt">4.2 out of 5 stars</span>
+    </div>
+  </body>
+</html>

surfsense_backend/tests/unit/platforms/amazon/fixtures/seller.html

@@ -0,0 +1,11 @@
+<!doctype html>
+<html lang="en">
+  <body>
+    <h1 id="sellerName">Example Retailer</h1>
+    <div id="seller-feedback-summary">1,875 ratings</div>
+    <table id="feedback-summary-table">
+      <tr><td><span class="a-icon-alt">4.8 out of 5 stars</span></td></tr>
+      <tr><td>1,875 ratings</td></tr>
+    </table>
+  </body>
+</html>

surfsense_backend/tests/unit/platforms/amazon/test_flows.py

@@ -0,0 +1,220 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from app.proprietary.platforms.amazon import AmazonScrapeInput, scrape_products, scraper
+from app.proprietary.platforms.amazon.fetch import FetchResult
+from app.proprietary.platforms.amazon.url_resolver import resolve_url
+
+_FIXTURES = Path(__file__).parent / "fixtures"
+
+
+def _fixture(name: str) -> str:
+    return (_FIXTURES / name).read_text(encoding="utf-8")
+
+
+def _response(url: str, html: str, status: int = 200) -> FetchResult:
+    return FetchResult(status=status, html=html, url=url, cookies={})
+
+
+async def test_product_flow_emits_parsed_item(monkeypatch):
+    async def fetch_page(url: str, **_kwargs):
+        return _response(url, _fixture("product.html"))
+
+    monkeypatch.setattr(scraper, "fetch_page", fetch_page)
+    items = await scrape_products(
+        AmazonScrapeInput(
+            categoryOrProductUrls=[{"url": "https://www.amazon.com/dp/B09V3KXJPB"}]
+        )
+    )
+
+    assert len(items) == 1
+    assert items[0]["asin"] == "B09V3KXJPB"
+    assert items[0]["title"] == "Example Wireless Headphones"
+
+
+async def test_product_flow_maps_not_found_to_error_item(monkeypatch):
+    async def fetch_page(url: str, **_kwargs):
+        return _response(url, "", status=404)
+
+    monkeypatch.setattr(scraper, "fetch_page", fetch_page)
+    items = await scrape_products(
+        AmazonScrapeInput(
+            categoryOrProductUrls=[{"url": "https://www.amazon.com/dp/B09V3KXJPB"}]
+        )
+    )
+
+    assert items[0]["error"] == "product_not_found"
+
+
+async def test_search_flow_honors_cap_and_stops_on_empty_page(monkeypatch):
+    calls: list[str] = []
+
+    async def fetch_page(url: str, **_kwargs):
+        calls.append(url)
+        html = _fixture("search.html") if "page=1" in url else "<html></html>"
+        return _response(url, html)
+
+    monkeypatch.setattr(scraper, "fetch_page", fetch_page)
+    items = await scrape_products(
+        AmazonScrapeInput(
+            categoryOrProductUrls=[{"url": "https://www.amazon.com/s?k=headphones"}],
+            maxItemsPerStartUrl=1,
+            maxSearchPagesPerStartUrl=5,
+            scrapeProductDetails=False,
+        )
+    )
+
+    assert len(items) == 1
+    assert items[0]["categoryPageData"]["position"] == 1
+    assert len(calls) == 1
+
+
+async def test_search_flow_threads_marketplace_locale_to_fetch(monkeypatch):
+    calls: list[dict[str, object]] = []
+
+    async def fetch_page(url: str, **kwargs):
+        calls.append({"url": url, **kwargs})
+        return _response(url, _fixture("search.html"))
+
+    monkeypatch.setattr(scraper, "fetch_page", fetch_page)
+    items = await scrape_products(
+        AmazonScrapeInput(
+            categoryOrProductUrls=[{"url": "https://www.amazon.co.uk/s?k=headphones"}],
+            maxItemsPerStartUrl=1,
+            scrapeProductDetails=False,
+        )
+    )
+
+    assert len(items) == 1
+    assert calls[0]["country"] == "gb"
+    assert calls[0]["accept_language"] == "en-GB"
+
+
+async def test_search_flow_returns_no_results_error(monkeypatch):
+    async def fetch_page(url: str, **_kwargs):
+        return _response(url, "<html></html>")
+
+    monkeypatch.setattr(scraper, "fetch_page", fetch_page)
+    items = await scrape_products(
+        AmazonScrapeInput(
+            categoryOrProductUrls=[{"url": "https://www.amazon.com/s?k=missing"}],
+            scrapeProductDetails=False,
+        )
+    )
+
+    assert items[0]["error"] == "no_results_found"
+
+
+async def test_product_flow_enriches_offers_and_seller(monkeypatch):
+    async def fetch_page(url: str, **_kwargs):
+        return _response(url, _fixture("product.html"))
+
+    async def fetch_aod_html(*_args, **_kwargs):
+        return _fixture("offers.html")
+
+    async def fetch_seller_html(*_args, **_kwargs):
+        return _fixture("seller.html")
+
+    monkeypatch.setattr(scraper, "fetch_page", fetch_page)
+    monkeypatch.setattr(scraper, "fetch_aod_html", fetch_aod_html)
+    monkeypatch.setattr(scraper, "fetch_seller_html", fetch_seller_html)
+    items = await scrape_products(
+        AmazonScrapeInput(
+            categoryOrProductUrls=[{"url": "https://www.amazon.com/dp/B09V3KXJPB"}],
+            maxOffers=1,
+            scrapeSellers=True,
+        )
+    )
+
+    assert len(items[0]["offers"]) == 1
+    assert items[0]["offers"][0]["seller"]["averageRating"] == 4.8
+    assert items[0]["seller"]["reviewsCount"] == 1875
+
+
+async def test_bestsellers_card_only_flow(monkeypatch):
+    async def fetch_page(url: str, **_kwargs):
+        return _response(url, _fixture("bestsellers.html"))
+
+    monkeypatch.setattr(scraper, "fetch_page", fetch_page)
+    items = await scrape_products(
+        AmazonScrapeInput(
+            categoryOrProductUrls=[
+                {
+                    "url": (
+                        "https://www.amazon.com/Best-Sellers-Electronics/"
+                        "zgbs/electronics"
+                    )
+                }
+            ],
+            maxItemsPerStartUrl=2,
+            scrapeProductDetails=False,
+        )
+    )
+
+    assert [item["bestsellerPageData"]["rank"] for item in items] == [1, 2]
+
+
+async def test_shortlink_resolves_and_dispatches(monkeypatch):
+    async def resolve(_url: str):
+        return "https://www.amazon.com/dp/B09V3KXJPB"
+
+    async def fetch_page(url: str, **_kwargs):
+        return _response(url, _fixture("product.html"))
+
+    monkeypatch.setattr(scraper, "resolve_shortlink", resolve)
+    monkeypatch.setattr(scraper, "fetch_page", fetch_page)
+    items = await scrape_products(
+        AmazonScrapeInput(categoryOrProductUrls=[{"url": "https://a.co/d/example"}])
+    )
+
+    assert items[0]["asin"] == "B09V3KXJPB"
+    assert items[0]["input"] == "https://a.co/d/example"
+
+
+async def test_product_flow_expands_variants_and_attaches_prices(monkeypatch):
+    async def fetch_page(url: str, **_kwargs):
+        return _response(url, _fixture("product.html"))
+
+    monkeypatch.setattr(scraper, "fetch_page", fetch_page)
+    items = await scrape_products(
+        AmazonScrapeInput(
+            categoryOrProductUrls=[{"url": "https://www.amazon.com/dp/B09V3KXJPB"}],
+            maxProductVariantsAsSeparateResults=1,
+            scrapeProductVariantPrices=True,
+        )
+    )
+
+    assert [item["asin"] for item in items] == ["B09V3KXJPB", "B09V3KXJP1"]
+    assert items[1]["originalAsin"] == "B09V3KXJPB"
+    variant = next(
+        detail
+        for detail in items[0]["variantDetails"]
+        if detail["asin"] == "B09V3KXJP1"
+    )
+    assert variant["price"]["value"] == 49.99
+
+
+async def test_route_outside_location_allowlist_skips_session_mint(monkeypatch):
+    calls = 0
+
+    async def get_location_session(*_args, **_kwargs):
+        nonlocal calls
+        calls += 1
+        return None
+
+    monkeypatch.setattr(scraper, "get_location_session", get_location_session)
+    resolved = resolve_url("https://www.amazon.com/dp/B09V3KXJPB")
+    assert resolved is not None
+    context = await scraper._location_context(
+        resolved,
+        AmazonScrapeInput(
+            categoryOrProductUrls=[{"url": resolved.url}],
+            zipCode="10001",
+            locationDeliverableRoutes=["OFFERS"],
+        ),
+        "PRODUCT",
+    )
+
+    assert context == (None, None, None, None)
+    assert calls == 0

surfsense_backend/tests/unit/platforms/amazon/test_locale.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from app.proprietary.platforms.amazon.locale import (
+    accept_language_for,
+    proxy_country_for,
+)
+
+
+def test_proxy_country_for_confirmed_marketplaces():
+    assert proxy_country_for("com") == "us"
+    assert proxy_country_for("co.uk") == "gb"
+    assert proxy_country_for("de") == "de"
+    assert proxy_country_for("fr") == "fr"
+    assert proxy_country_for("it") == "it"
+    assert proxy_country_for("es") == "es"
+
+
+def test_proxy_country_for_unknown_marketplace_falls_back():
+    assert proxy_country_for("co.jp") is None
+    assert proxy_country_for(None) is None
+
+
+def test_accept_language_for_marketplaces():
+    assert accept_language_for("co.uk") == "en-GB"
+    assert accept_language_for("de") == "de-DE"
+    assert accept_language_for("fr") == "fr-FR"
+    assert accept_language_for("it") == "it-IT"
+    assert accept_language_for("es") == "es-ES"
+
+
+def test_accept_language_for_unknown_marketplace_defaults_to_us_english():
+    assert accept_language_for("co.jp") == "en-US"
+    assert accept_language_for(None) == "en-US"

surfsense_backend/tests/unit/platforms/amazon/test_parsers.py

@@ -0,0 +1,176 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from app.proprietary.platforms.amazon import fetch
+from app.proprietary.platforms.amazon.fetch import (
+    FetchResult,
+    get_location_session,
+    is_blocked,
+    should_localize,
+)
+from app.proprietary.platforms.amazon.parsers import (
+    _float,
+    parse_aod_offers,
+    parse_bestsellers_page,
+    parse_product,
+    parse_search_page,
+    parse_seller,
+)
+
+_FIXTURES = Path(__file__).parent / "fixtures"
+
+
+def _fixture(name: str) -> str:
+    return (_FIXTURES / name).read_text(encoding="utf-8")
+
+
+def test_product_parser_extracts_core_public_fields():
+    item = parse_product(
+        _fixture("product.html"),
+        asin="B09V3KXJPB",
+        url="https://www.amazon.com/dp/B09V3KXJPB",
+    )
+
+    assert item["title"] == "Example Wireless Headphones"
+    assert item["price"] == {"value": 49.99, "currency": "USD"}
+    assert item["listPrice"] == {"value": 79.99, "currency": "USD"}
+    assert item["stars"] == 4.6
+    assert item["reviewsCount"] == 1234
+    assert item["inStock"] is True
+    assert item["features"] == ["Long battery life", "Noise cancelling"]
+    assert item["starsBreakdown"]["5star"] == 0.8
+    assert "B09V3KXJP1" in item["variantAsins"]
+    assert item["variantAttributes"][0]["name"] == "color_name"
+    assert item["productPageReviews"][0]["id"] == "R1"
+
+
+def test_block_detection_handles_status_and_markup():
+    blocked = _fixture("blocked.html")
+    assert is_blocked(blocked, 200)
+    assert is_blocked("", 503)
+    assert not is_blocked(_fixture("product.html"), 200)
+
+
+def test_block_detection_handles_waf_header_and_body_markers():
+    assert is_blocked(
+        "<html>ordinary status</html>",
+        200,
+        {"X-Amzn-Waf-Action": "challenge"},
+    )
+    assert is_blocked(
+        "<script src='https://token.awswaf.com/challenge.js'></script>", 200
+    )
+    assert is_blocked(
+        '<meta http-equiv="refresh" content="5; URL=/s?bm-verify=x">', 200
+    )
+
+
+def test_float_handles_us_and_eu_price_formats():
+    assert _float("1.234,56 €") == 1234.56
+    assert _float("12,99 €") == 12.99
+    assert _float("$1,234.56") == 1234.56
+    assert _float("1234") == 1234.0
+
+
+def test_block_retry_proxy_uses_fresh_country_session(monkeypatch):
+    geo_calls: list[str | None] = []
+    sticky_calls: list[tuple[str, str | None]] = []
+
+    def get_geo_proxy_url(country: str | None = None):
+        geo_calls.append(country)
+        return f"http://geo-{country}"
+
+    def get_sticky_proxy_url(session_id: str, country: str | None = None):
+        sticky_calls.append((session_id, country))
+        return f"http://sticky-{country}-{session_id}"
+
+    monkeypatch.setattr(fetch, "get_geo_proxy_url", get_geo_proxy_url)
+    monkeypatch.setattr(fetch, "get_sticky_proxy_url", get_sticky_proxy_url)
+
+    first = fetch._selected_proxy(None, "fr", 1, "https://www.amazon.fr/s?k=x")
+    second = fetch._selected_proxy(None, "fr", 2, "https://www.amazon.fr/s?k=x")
+
+    assert first == "http://geo-fr"
+    assert second.startswith("http://sticky-fr-amazon-fr-2-")
+    assert geo_calls == ["fr"]
+    assert sticky_calls[0][1] == "fr"
+
+
+def test_search_parser_extracts_cards_and_provenance():
+    cards = parse_search_page(_fixture("search.html"), page=2)
+
+    assert [card["asin"] for card in cards] == ["B09V3KXJPB", "B09V3KXJP1"]
+    assert cards[0]["categoryPageData"] == {
+        "position": 1,
+        "page": 2,
+        "isSponsored": False,
+        "isBestSeller": True,
+    }
+    assert cards[1]["categoryPageData"]["isSponsored"] is True
+
+
+def test_offer_and_seller_parsers():
+    offers = parse_aod_offers(_fixture("offers.html"))
+    seller = parse_seller(_fixture("seller.html"), seller_id="SELLER123")
+
+    assert len(offers) == 2
+    assert offers[0]["price"]["value"] == 47.5
+    assert offers[0]["seller"]["id"] == "SELLER123"
+    assert offers[0]["isPinnedOffer"] is True
+    assert seller["name"] == "Example Retailer"
+    assert seller["averageRating"] == 4.8
+    assert seller["reviewsCount"] == 1875
+
+
+def test_bestsellers_parser_extracts_ranked_products():
+    items = parse_bestsellers_page(_fixture("bestsellers.html"))
+
+    assert [item["asin"] for item in items] == ["B09V3KXJPB", "B09V3KXJP1"]
+    assert [item["bestsellerPageData"]["rank"] for item in items] == [1, 2]
+
+
+def test_location_route_filter_is_explicit():
+    routes = ["PRODUCT", "OFFERS"]
+    assert should_localize("product", routes)
+    assert not should_localize("search", routes)
+
+
+async def test_location_session_mints_once_and_reuses_cache(monkeypatch):
+    fetch._location_sessions.clear()
+    calls: list[str] = []
+
+    async def sticky_proxy(_session_id: str, _country: str | None = None):
+        return "http://sticky-proxy"
+
+    async def fetch_page(url: str, **_kwargs):
+        calls.append(url)
+        if len(calls) == 1:
+            return FetchResult(
+                status=200,
+                html='<input name="anti-csrftoken-a2z" value="token-123">',
+                url=url,
+                cookies={"session-id": "session", "ubid-main": "ubid"},
+            )
+        return FetchResult(
+            status=200,
+            html='{"isValidAddress":1}',
+            url=url,
+            cookies={"session-id": "session"},
+        )
+
+    monkeypatch.setattr(fetch, "_sticky_proxy", sticky_proxy)
+    monkeypatch.setattr(fetch, "fetch_page", fetch_page)
+
+    first = await get_location_session(
+        "www.amazon.com", zip_code="10001", country_code="US"
+    )
+    second = await get_location_session(
+        "www.amazon.com", zip_code="10001", country_code="US"
+    )
+
+    assert first is second
+    assert first is not None
+    assert first.proxy == "http://sticky-proxy"
+    assert first.country_code == "US"
+    assert len(calls) == 2

surfsense_backend/tests/unit/platforms/amazon/test_proxy.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+import pytest
+
+from app.config import Config
+from app.utils.proxy.providers.dataimpulse import DataImpulseProvider
+
+
+def test_dataimpulse_sticky_url_is_deterministic(monkeypatch):
+    monkeypatch.setattr(
+        Config,
+        "PROXY_URL",
+        "http://token__cr.us:secret@gw.dataimpulse.com:823",
+    )
+    provider = DataImpulseProvider()
+
+    first = provider.get_sticky_proxy_url("location-123")
+    second = provider.get_sticky_proxy_url("location-123")
+
+    assert first == second
+    assert "token__cr.us__sid.location-123" in first
+    assert first.endswith("@gw.dataimpulse.com:823")
+
+
+def test_dataimpulse_sticky_url_replaces_existing_session(monkeypatch):
+    monkeypatch.setattr(
+        Config,
+        "PROXY_URL",
+        "http://token__cr.us__sid.old:secret@gw.dataimpulse.com:823",
+    )
+
+    result = DataImpulseProvider().get_sticky_proxy_url("new")
+
+    assert "__sid.old" not in result
+    assert result.count("__sid.new") == 1
+
+
+def test_dataimpulse_sticky_url_rewrites_country(monkeypatch):
+    monkeypatch.setattr(
+        Config,
+        "PROXY_URL",
+        "http://token__cr.us:secret@gw.dataimpulse.com:823",
+    )
+
+    result = DataImpulseProvider().get_sticky_proxy_url("new", "gb")
+
+    assert "token__cr.gb__sid.new" in result
+    assert "__cr.us" not in result
+
+
+def test_dataimpulse_sticky_url_rejects_empty_session(monkeypatch):
+    monkeypatch.setattr(
+        Config,
+        "PROXY_URL",
+        "http://token:secret@gw.dataimpulse.com:823",
+    )
+
+    with pytest.raises(ValueError):
+        DataImpulseProvider().get_sticky_proxy_url("...")

surfsense_backend/tests/unit/platforms/amazon/test_skeleton.py

@@ -0,0 +1,177 @@
+"""Offline contract tests for the Amazon Product Scraper.
+
+Deterministic (no network, no live Amazon HTML): asserts the input defaults, the
+full output-item serialization contract, the error-item shape, and URL
+classification. The live fetch / parse flows are exercised by the e2e script in
+later milestones, not here.
+"""
+
+from __future__ import annotations
+
+from app.proprietary.platforms.amazon import (
+    AmazonScrapeInput,
+    ErrorItem,
+    ProductItem,
+    scrape_products,
+)
+from app.proprietary.platforms.amazon.url_resolver import extract_asin, resolve_url
+
+# A complete input payload should validate without removing optional fields.
+_SPEC_INPUT = {
+    "categoryOrProductUrls": [{"url": "https://www.amazon.com/s?k=keyboard"}],
+    "maxItemsPerStartUrl": 100,
+    "language": "en",
+    "proxyCountry": "AUTO_SELECT_PROXY_COUNTRY",
+    "maxSearchPagesPerStartUrl": 9999,
+    "maxOffers": 0,
+    "scrapeSellers": False,
+    "useCaptchaSolver": False,
+    "scrapeProductVariantPrices": False,
+    "scrapeProductDetails": True,
+    "countryCode": "US",
+    "zipCode": "10001",
+    "locationDeliverableRoutes": ["PRODUCT", "SEARCH", "OFFERS"],
+}
+
+
+def test_input_has_no_auth_fields():
+    # Public, anonymous only: no auth-shaped field may exist on the input surface.
+    forbidden = {"username", "password", "token", "login", "auth", "credentials"}
+    assert forbidden.isdisjoint(AmazonScrapeInput.model_fields)
+
+
+def test_scrape_input_defaults_match_contract():
+    inp = AmazonScrapeInput(
+        categoryOrProductUrls=[{"url": "https://www.amazon.com/dp/B0"}]
+    )
+    assert inp.maxItemsPerStartUrl is None
+    assert inp.maxSearchPagesPerStartUrl == 9999
+    assert inp.maxProductVariantsAsSeparateResults == 0
+    assert inp.maxOffers == 0
+    assert inp.proxyCountry == "AUTO_SELECT_PROXY_COUNTRY"
+    assert inp.language is None
+    assert inp.countryCode is None
+    assert inp.zipCode is None
+    assert inp.locationDeliverableRoutes == ["PRODUCT", "SEARCH", "OFFERS"]
+    assert inp.scrapeSellers is False
+    assert inp.useCaptchaSolver is False
+    assert inp.scrapeProductVariantPrices is False
+    assert inp.scrapeProductDetails is True
+
+
+def test_complete_payload_validates():
+    inp = AmazonScrapeInput(**_SPEC_INPUT)
+    assert inp.maxItemsPerStartUrl == 100
+    assert inp.countryCode == "US"
+
+
+def test_input_allows_extra_inert_fields():
+    # extra="allow": unknown add-ons (e.g. upstream proxy config) are accepted.
+    inp = AmazonScrapeInput(
+        categoryOrProductUrls=[{"url": "https://www.amazon.com/dp/B0"}],
+        proxyConfiguration={"useResidentialProxy": True},
+        someFutureAddon=123,
+    )
+    assert inp.model_dump().get("someFutureAddon") == 123
+
+
+def test_output_item_serializes_full_shape():
+    item = ProductItem(asin="B08EXAMPLE01").to_output()
+    assert item["asin"] == "B08EXAMPLE01"
+    # Unsourced scalars are still present as None (consumers never KeyError).
+    assert item["title"] is None
+    assert item["monthlyPurchaseVolume"] is None
+    # List fields default to [].
+    assert item["features"] == []
+    assert item["offers"] == []
+    assert item["productPageReviews"] == []
+    # Typed-but-unset nested objects are None.
+    assert item["price"] is None
+    assert item["seller"] is None
+
+
+def test_stars_breakdown_round_trips_digit_keys():
+    item = ProductItem(
+        stars=4.8, starsBreakdown={"5star": 0.86, "1star": 0.01}
+    ).to_output()
+    # by_alias serialization restores Amazon's digit-prefixed keys.
+    assert item["starsBreakdown"]["5star"] == 0.86
+    assert item["starsBreakdown"]["1star"] == 0.01
+    assert item["starsBreakdown"]["4star"] is None
+
+
+def test_error_item_shape():
+    err = ErrorItem(
+        error="product_not_found",
+        errorDescription="Loaded a 404 page.",
+        input="https://www.amazon.com/dp/B0XXXXXXXX",
+        url="https://www.amazon.com/dp/B0XXXXXXXX",
+    ).model_dump()
+    assert set(err) >= {"error", "errorDescription", "input", "url"}
+    assert err["error"] == "product_not_found"
+
+
+def test_extract_asin():
+    assert extract_asin("https://www.amazon.com/dp/B09X7MPX8L") == "B09X7MPX8L"
+    assert extract_asin("https://www.amazon.com/gp/product/B09X7MPX8L/ref=x") == (
+        "B09X7MPX8L"
+    )
+    assert extract_asin("https://www.amazon.com/s?k=keyboard") is None
+
+
+def test_resolve_url_classifies_all_kinds():
+    product = resolve_url("https://www.amazon.com/dp/B0EXAMPLE1")
+    assert product is not None
+    assert product.kind == "product"
+    assert product.asin == "B0EXAMPLE1"
+    assert product.marketplace == "com"
+
+    search = resolve_url("https://www.amazon.com/s?k=keyboard")
+    assert search is not None and search.kind == "search"
+
+    # A category URL with no /s path but a bbn/rh query still classifies as search.
+    category = resolve_url(
+        "https://www.amazon.de/s?i=specialty-aps&bbn=16225007011&rh=n%3A16225007011"
+    )
+    assert category is not None
+    assert category.kind == "search"
+    assert category.marketplace == "de"
+
+    bestsellers = resolve_url(
+        "https://www.amazon.com/Best-Sellers-Electronics/zgbs/electronics"
+    )
+    assert bestsellers is not None and bestsellers.kind == "bestsellers"
+
+    shortened = resolve_url("https://a.co/d/abcd123")
+    assert shortened is not None and shortened.kind == "shortened"
+
+
+def test_resolve_url_rejects_non_amazon_and_unrecognized():
+    assert resolve_url("https://example.com/dp/B08EXAMPLE1") is None
+    # An Amazon host but an unrecognized path (e.g. the homepage) is unrecognized.
+    assert resolve_url("https://www.amazon.com/") is None
+
+
+async def test_iter_products_unrecognized_yields_error():
+    # A junk / non-Amazon URL yields exactly one invalid_url error item.
+    items = await scrape_products(
+        AmazonScrapeInput(categoryOrProductUrls=[{"url": "https://example.com/foo"}])
+    )
+    assert len(items) == 1
+    assert items[0]["error"] == "invalid_url"
+    assert items[0]["input"] == "https://example.com/foo"
+
+
+def test_valid_product_url_is_ready_for_dispatch():
+    resolved = resolve_url("https://www.amazon.com/dp/B0EXAMPLE1")
+    assert resolved is not None
+    assert resolved.kind == "product"
+
+
+async def test_iter_products_missing_url_key_yields_error():
+    # An entry without a "url" key is treated as an invalid start URL, not a crash.
+    items = await scrape_products(
+        AmazonScrapeInput(categoryOrProductUrls=[{"noturl": "x"}])
+    )
+    assert len(items) == 1
+    assert items[0]["error"] == "invalid_url"

surfsense_backend/tests/unit/platforms/google_search/test_captcha.py

@@ -0,0 +1,37 @@
+"""Offline checks for the /sorry reCAPTCHA solver's pure helpers.
+
+The network solve can't be unit-tested, but the parsing around it can: a missed
+sitekey/data-s silently burns paid solves, so these guard the boundary logic.
+(Proxy reformatting now lives in ``app.utils.captcha.solvers``.)
+"""
+
+from app.proprietary.platforms.google_search import captcha
+
+
+class _Page:
+    def __init__(self, url: str) -> None:
+        self.url = url
+
+
+def test_on_sorry_detects_wall_only():
+    assert captcha.on_sorry(_Page("https://www.google.com/sorry/index?continue=x"))
+    assert not captcha.on_sorry(_Page("https://www.google.com/search?q=notebooklm"))
+    assert not captcha.on_sorry(_Page(""))
+
+
+def test_sitekey_and_data_s_extraction():
+    html = (
+        '<div class="g-recaptcha" data-sitekey="6LdLLIMbAAAAAIl-KLj9p1ePhM"'
+        ' data-s="zBB1ixry9YzY_tok-en"></div>'
+    )
+    assert captcha._SITEKEY_RE.search(html).group(1) == "6LdLLIMbAAAAAIl-KLj9p1ePhM"
+    assert captcha._DATA_S_RE.search(html).group(1) == "zBB1ixry9YzY_tok-en"
+
+
+def test_latch_roundtrip():
+    captcha.reset_solver_latch()
+    assert not captcha.solver_latched()
+    captcha._latch("no balance")
+    assert captcha.solver_latched()
+    captcha.reset_solver_latch()
+    assert not captcha.solver_latched()

surfsense_backend/tests/unit/platforms/google_search/test_fetch_concurrency.py

@@ -16,7 +16,7 @@ class _FakeSession:
         self.release = asyncio.Event()
         self.closed = 0
 
-    async def fetch(self, url, proxy=None):
+    async def fetch(self, url, proxy=None, **kwargs):
         await self.release.wait()
         return "page"
 

surfsense_backend/tests/unit/platforms/google_search/test_fetch_pool.py

@@ -0,0 +1,99 @@
+"""Warm sticky-IP pool bookkeeping (the scale-critical logic in fetch.py).
+
+These pure, lock-guarded helpers decide whether a render reuses a warm IP,
+grows the pool (a paid solve), or waits — and how a finished render admits or
+evicts its IP. Get the pending/inflight accounting wrong and the pool either
+over-solves (cost) or funnels every render onto one IP (re-wall), so the
+transitions are worth pinning down offline.
+"""
+
+import pytest
+
+from app.proprietary.platforms.google_search import fetch
+
+pytestmark = pytest.mark.unit
+
+
+@pytest.fixture(autouse=True)
+def _clean_pool():
+    def reset():
+        fetch._pool.clear()
+        fetch._pool_inflight.clear()
+        fetch._pool_pending = 0
+        fetch._exemption_jar.clear()
+
+    reset()
+    yield
+    reset()
+
+
+def test_take_grows_when_pool_empty():
+    action, proxy = fetch._pool_take()
+    assert action == "grow" and proxy is None
+    assert fetch._pool_pending == 1
+
+
+def test_take_reuses_warm_under_cap():
+    fetch._pool["p1"] = 1.0
+    action, proxy = fetch._pool_take()
+    assert action == "reuse" and proxy == "p1"
+    assert fetch._pool_inflight["p1"] == 1
+
+
+def test_take_spreads_across_least_loaded_ip():
+    fetch._pool.update({"p1": 1.0, "p2": 1.0})
+    fetch._pool_inflight["p1"] = 1  # p2 is idle
+    action, proxy = fetch._pool_take()
+    assert action == "reuse" and proxy == "p2"
+
+
+def test_take_waits_when_full_and_every_ip_capped(monkeypatch):
+    monkeypatch.setattr(fetch, "_WARM_POOL_TARGET", 2)
+    monkeypatch.setattr(fetch, "_WARM_IP_MAX_CONCURRENCY", 1)
+    fetch._pool.update({"p1": 1.0, "p2": 1.0})
+    fetch._pool_inflight.update({"p1": 1, "p2": 1})
+    action, proxy = fetch._pool_take()
+    assert action == "wait" and proxy is None
+    assert fetch._pool_pending == 0  # a wait must NOT reserve a solve
+
+
+def test_take_grows_only_up_to_target_counting_pending(monkeypatch):
+    monkeypatch.setattr(fetch, "_WARM_POOL_TARGET", 2)
+    monkeypatch.setattr(fetch, "_WARM_IP_MAX_CONCURRENCY", 1)
+    fetch._pool["p1"] = 1.0
+    fetch._pool_inflight["p1"] = 1  # warm but capped
+    assert fetch._pool_take()[0] == "grow"  # pool(1)+pending(0) < 2
+    assert fetch._pool_pending == 1
+    assert fetch._pool_take()[0] == "wait"  # pool(1)+pending(1) == 2 → no more solves
+
+
+def test_settle_good_admits_and_releases():
+    fetch._pool_pending = 1
+    fetch._pool_inflight["p1"] = 1
+    fetch._pool_settle("p1", good=True, grew=True)
+    assert "p1" in fetch._pool
+    assert "p1" not in fetch._pool_inflight
+    assert fetch._pool_pending == 0
+
+
+def test_settle_walled_evicts_ip_and_drops_exemption():
+    fetch._pool["p1"] = 1.0
+    fetch._pool_inflight["p1"] = 1
+    fetch._exemption_jar["p1"] = [{"name": "GOOGLE_ABUSE_EXEMPTION"}]
+    fetch._pool_settle("p1", good=False, grew=False)
+    assert "p1" not in fetch._pool
+    assert "p1" not in fetch._exemption_jar
+
+
+def test_adopt_releases_pending_and_pins_ip():
+    fetch._pool_pending = 1  # reserved by a grow that we satisfy via the store
+    fetch._pool_adopt("shared")
+    assert fetch._pool_pending == 0
+    assert "shared" in fetch._pool
+    assert fetch._pool_inflight["shared"] == 1
+
+
+def test_abort_grow_releases_the_reserved_slot():
+    fetch._pool_pending = 2
+    fetch._pool_abort_grow()
+    assert fetch._pool_pending == 1

surfsense_backend/tests/unit/platforms/google_search/test_pool_store.py

@@ -0,0 +1,73 @@
+"""Cross-process warm-IP exemption store (best-effort Redis cache).
+
+Redis itself isn't unit-testable here, but the boundary is: adopt must skip IPs
+this process already holds (or the fleet re-uses nothing new), and every path
+must degrade to a silent no-op when Redis is down (or a hiccup would break the
+fetch instead of just costing a solve).
+"""
+
+import pytest
+
+from app.proprietary.platforms.google_search import pool_store as ps
+
+pytestmark = pytest.mark.unit
+
+
+class _FakeRedis:
+    def __init__(self):
+        self.store: dict[str, str] = {}
+
+    def ping(self):
+        return True
+
+    def set(self, k, v, ex=None):
+        self.store[k] = v
+
+    def get(self, k):
+        return self.store.get(k)
+
+    def delete(self, k):
+        self.store.pop(k, None)
+
+    def scan_iter(self, match=None, count=None):
+        return list(self.store.keys())
+
+
+@pytest.fixture
+def fake(monkeypatch):
+    r = _FakeRedis()
+    monkeypatch.setattr(ps, "_client", r)
+    monkeypatch.setattr(ps, "_disabled", False)
+    return r
+
+
+def test_key_is_stable_and_prefixed():
+    assert ps._key("http://a").startswith(ps._PREFIX)
+    assert ps._key("http://a") == ps._key("http://a")
+    assert ps._key("http://a") != ps._key("http://b")
+
+
+def test_publish_then_adopt_excludes_held_ips(fake):
+    ps._publish_sync("http://a", [{"n": 1}])
+    ps._publish_sync("http://b", [{"n": 2}])
+    proxy, cookies = ps._adopt_sync(exclude={"http://a"})
+    assert proxy == "http://b" and cookies == [{"n": 2}]
+
+
+def test_adopt_none_when_all_held(fake):
+    ps._publish_sync("http://a", [])
+    assert ps._adopt_sync(exclude={"http://a"}) is None
+
+
+def test_evict_removes_entry(fake):
+    ps._publish_sync("http://a", [{"n": 1}])
+    ps._evict_sync("http://a")
+    assert ps._adopt_sync(exclude=set()) is None
+
+
+def test_disabled_is_silent_noop(monkeypatch):
+    monkeypatch.setattr(ps, "_disabled", True)
+    monkeypatch.setattr(ps, "_client", None)
+    assert ps._adopt_sync(set()) is None
+    ps._publish_sync("x", [{"n": 1}])  # must not raise
+    ps._evict_sync("x")  # must not raise

surfsense_backend/tests/unit/platforms/reddit/test_fetch_resilience.py

@@ -1,9 +1,9 @@
 """Offline resilience tests for the Reddit fetch seam and fan-out worker pool.
 
-No network. Fake sessions drive the ``loid`` warm-up + rotate-on-block + backoff
+No network. Fake sessions drive the browser-warm + rotate-on-block + backoff
 paths deterministically (in live runs the first IP warms and returns 200s, so
-these branches rarely fire). Mirrors the youtube sibling's
-``test_fetch_resilience.py`` shape, extended with a fake warm-up.
+these branches rarely fire). The warm is now a browser mint (``holder.warm()``),
+so the fake holder fakes minting a cookie jar instead of HTTP GETs to a warm URL.
 """
 
 from __future__ import annotations
@@ -22,9 +22,8 @@ _LISTING = {"kind": "Listing", "data": {"children": [], "after": None}}
 
 
 class _FakePage:
-    def __init__(self, status: int, *, cookies: dict | None = None, payload=None):
+    def __init__(self, status: int, *, payload=None):
         self.status = status
-        self.cookies = cookies or {}
         self._payload = payload if payload is not None else _LISTING
 
     def json(self):
@@ -36,48 +35,53 @@ class _FakePage:
 
 
 class _FakeSession:
-    """One 'IP': warm-up mints loid per flags; ``.json`` GETs return ``status``."""
+    """One 'IP': ``can_warm`` gates the browser mint; ``.json`` GETs return ``status``."""
 
     def __init__(
         self,
         status: int = 200,
         *,
-        shreddit_loid: bool = True,
-        old_loid: bool = False,
+        can_warm: bool = True,
         payload=None,
     ) -> None:
         self.status = status
-        self.shreddit_loid = shreddit_loid
-        self.old_loid = old_loid
+        self.can_warm = can_warm
         self.payload = payload
         self.json_calls = 0
-        self.warm_calls = 0
 
     async def get(self, url, headers=None, cookies=None):
-        if "svc/shreddit" in url:
-            self.warm_calls += 1
-            ck = {"loid": "x", "session_tracker": "y"} if self.shreddit_loid else {}
-            return _FakePage(200, cookies=ck)
-        if "old.reddit.com" in url:
-            self.warm_calls += 1
-            return _FakePage(200, cookies={"loid": "x"} if self.old_loid else {})
         self.json_calls += 1
         return _FakePage(self.status, payload=self.payload)
 
 
 class _FakeHolder:
-    """Holder whose ``rotate()`` advances to the next fake session (a new IP)."""
+    """Holder whose ``rotate()`` advances to the next fake session (a new IP).
+
+    ``warm()`` fakes the browser mint: it succeeds (stashing a jar) only when the
+    current fake session's ``can_warm`` is set, mirroring a clean vs dirty exit.
+    """
 
     def __init__(self, sessions: list[_FakeSession]) -> None:
         self._sessions = sessions
         self.session = sessions[0]
         self.rotations = 0
         self.warmed = False
+        self.proxy = "http://sticky.example:823"
+        self.cookies: dict[str, str] = {}
+        self.warm_calls = 0
+
+    async def warm(self) -> bool:
+        self.warm_calls += 1
+        if self.session.can_warm:
+            self.cookies = {"loid": "x"}
+            return True
+        return False
 
     async def rotate(self):
         self.rotations += 1
         self.session = self._sessions[min(self.rotations, len(self._sessions) - 1)]
         self.warmed = False  # loid binds to the IP: re-warm on the fresh one
+        self.cookies = {}
         return self.session
 
     async def pace(self) -> None:
@@ -95,21 +99,8 @@ def _no_sleep(monkeypatch) -> None:
 
 
 async def test_warms_then_returns_json():
-    # shreddit is tried first and mints loid -> a single warm call.
-    holder = _FakeHolder([_FakeSession(200, shreddit_loid=True)])
-    token = _current_session.set(holder)
-    try:
-        result = await fetch_json("r/python/hot")
-    finally:
-        _current_session.reset(token)
-    assert result == _LISTING
-    assert holder.rotations == 0
-    assert holder.session.warm_calls == 1  # warmed exactly once
-
-
-async def test_warm_falls_back_to_old_reddit():
-    # shreddit doesn't mint loid, old.reddit does -> still warms on the same IP.
-    holder = _FakeHolder([_FakeSession(200, shreddit_loid=False, old_loid=True)])
+    # The first IP warms (mints a jar) -> a single warm call, no rotation.
+    holder = _FakeHolder([_FakeSession(200, can_warm=True)])
     token = _current_session.set(holder)
     try:
         result = await fetch_json("r/python/hot")
@@ -117,14 +108,16 @@ async def test_warm_falls_back_to_old_reddit():
         _current_session.reset(token)
     assert result == _LISTING
     assert holder.rotations == 0
+    assert holder.warm_calls == 1  # warmed exactly once
+    assert holder.cookies == {"loid": "x"}  # jar stashed for the .json fetch
 
 
 async def test_rotates_when_warm_fails_then_succeeds():
     # IP0 can't mint loid at all -> rotate; IP1 warms fine.
     holder = _FakeHolder(
         [
-            _FakeSession(200, shreddit_loid=False, old_loid=False),
-            _FakeSession(200, shreddit_loid=True),
+            _FakeSession(200, can_warm=False),
+            _FakeSession(200, can_warm=True),
         ]
     )
     token = _current_session.set(holder)
@@ -138,10 +131,7 @@ async def test_rotates_when_warm_fails_then_succeeds():
 
 async def test_raises_when_no_ip_can_warm():
     holder = _FakeHolder(
-        [
-            _FakeSession(200, shreddit_loid=False, old_loid=False)
-            for _ in range(fetch._MAX_ROTATIONS + 1)
-        ]
+        [_FakeSession(200, can_warm=False) for _ in range(fetch._MAX_ROTATIONS + 1)]
     )
     token = _current_session.set(holder)
     try:
@@ -157,7 +147,7 @@ async def test_raises_when_no_ip_can_warm():
 
 
 async def test_rotates_and_rewarms_on_403():
-    holder = _FakeHolder([_FakeSession(403), _FakeSession(200, old_loid=True)])
+    holder = _FakeHolder([_FakeSession(403), _FakeSession(200, can_warm=True)])
     token = _current_session.set(holder)
     try:
         result = await fetch_json("r/python/hot")
@@ -165,7 +155,7 @@ async def test_rotates_and_rewarms_on_403():
         _current_session.reset(token)
     assert result == _LISTING
     assert holder.rotations == 1
-    assert holder.session.warm_calls == 1  # re-warmed on the fresh IP
+    assert holder.warm_calls == 2  # re-warmed on the fresh IP after the 403
 
 
 async def test_404_returns_none_without_rotating():
@@ -185,9 +175,6 @@ async def test_429_backs_off_without_rotating(monkeypatch):
     session = _FakeSession(429)
 
     async def _get(url, headers=None, cookies=None):
-        if "svc/shreddit" in url or "old.reddit.com" in url:
-            session.warm_calls += 1
-            return _FakePage(200, cookies={"loid": "x"})
         session.json_calls += 1
         return _FakePage(429 if session.json_calls == 1 else 200)
 
@@ -266,3 +253,7 @@ async def test_fan_out_closes_all_sessions_on_early_stop(monkeypatch):
 async def test_fan_out_empty_jobs_is_noop():
     out = [x async for x in scraper.fan_out([])]
     assert out == []
+
+
+# Cross-country rotation lives in app.utils.proxy.rotation and is shared with the
+# TikTok sibling; its unit tests live in tests/unit/utils/proxy/test_rotation.py.

surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py

@@ -169,6 +169,37 @@ async def test_listing_dedupes_then_caps_per_target():
     assert [i["id"] for i in items] == ["1", "2"]
 
 
+async def test_search_query_resolves_to_listing_target():
+    # searchQueries must produce a search target (not be silently dropped): a
+    # query with results flows through the listing parse/dedupe/cap path.
+    async def fake_listing(url: str, _count: int) -> list[dict]:
+        assert "search?q=meal%20prep" in url  # query wired into the search URL
+        return [{"id": "1", "author": {"uniqueId": "a"}}]
+
+    items = await scrape_tiktok(
+        TikTokScrapeInput(searchQueries=["meal prep"], resultsPerPage=5),
+        fetch=_no_html,
+        fetch_listing=fake_listing,
+    )
+    assert [i["id"] for i in items] == ["1"]
+
+
+async def test_empty_search_query_degrades_to_error_item():
+    # A withheld anonymous search feed must surface one honest ErrorItem tagged
+    # with the query, never a silent empty.
+    async def fake_listing(_url: str, _count: int) -> list[dict]:
+        return []
+
+    items = await scrape_tiktok(
+        TikTokScrapeInput(searchQueries=["meal prep"], resultsPerPage=5),
+        fetch=_no_html,
+        fetch_listing=fake_listing,
+    )
+    assert len(items) == 1
+    assert items[0]["errorCode"] == "no_items"
+    assert items[0]["input"] == "meal prep"
+
+
 async def test_empty_listing_emits_error_item():
     # A trust-gated/empty feed (0 videos) must surface one honest ErrorItem,
     # tagged with errorCode, rather than vanishing silently.

surfsense_backend/tests/unit/proprietary/web_crawler/test_captcha.py

@@ -2,8 +2,9 @@
 
 The browser/solver boundary is mocked: a fake Playwright ``page`` and a
 monkeypatched ``_harvest_token`` / injection. We assert the glue logic —
-detection, proxy reformatting, attempt counting + state surfacing, the per-URL
-cap, and the no-balance process latch.
+detection, proxy pass-through, attempt counting + state surfacing, the per-URL
+cap, and the no-balance process latch. (Vendor proxy reformatting now lives in
+``app.utils.captcha.solvers`` and is tested there.)
 """
 
 import pytest
@@ -76,24 +77,6 @@ def _clear_latch():
     cap.reset_solver_latch()
 
 
-# --- proxy reformat --------------------------------------------------------
-
-
-class TestProxyReformat:
-    def test_with_auth(self):
-        assert (
-            cap.proxy_url_to_captchatools("http://user:pass@1.2.3.4:8080")
-            == "1.2.3.4:8080:user:pass"
-        )
-
-    def test_without_auth(self):
-        assert cap.proxy_url_to_captchatools("http://1.2.3.4:8080") == "1.2.3.4:8080"
-
-    def test_none_and_garbage(self):
-        assert cap.proxy_url_to_captchatools(None) is None
-        assert cap.proxy_url_to_captchatools("not-a-url") is None
-
-
 # --- detection -------------------------------------------------------------
 
 
@@ -175,8 +158,9 @@ class TestPageAction:
         action(page)
 
         assert state == {"attempts": 1, "solved": True}
-        # Solver egressed from the crawl's proxy, reformatted, with the page UA.
-        assert captured["proxy"] == "1.2.3.4:9000:u:p"
+        # Solver egressed from the crawl's proxy (raw URL; the seam reformats
+        # per vendor), with the page UA.
+        assert captured["proxy"] == "http://u:p@1.2.3.4:9000"
         assert captured["ua"] == "UA/1.0"
         assert captured["ctype"] == "v2"
         assert injected == {"ctype": "v2", "token": "TOKEN"}

surfsense_backend/tests/unit/routes/test_llm_setup_status.py

@@ -0,0 +1,326 @@
+"""Unit tests for the server-authoritative LLM onboarding verdict.
+
+``compute_llm_setup_status`` is the single source of truth for whether a
+workspace can chat. These tests cover the two pieces of genuinely new logic:
+
+1. ``_global_catalog_has_usable_chat`` — a pure check over the operator
+   global catalog (usable model, not mere file presence).
+2. The decision tree in ``compute_llm_setup_status`` — exercised by faking
+   the DB-touching seams (``_clear_invalid_roles`` heals dangling pins,
+   ``_workspace_has_enabled_chat_model`` reports BYOK models) so the routing
+   between ready / needs_setup / global_config / models is asserted directly.
+"""
+
+from __future__ import annotations
+
+from contextlib import ExitStack
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from app.auth.context import AuthContext
+from app.db import Permission
+from app.routes import model_connections_routes as mc
+
+
+@dataclass
+class _FakeUser:
+    id: str = "u1"
+
+
+@dataclass
+class _FakeWorkspace:
+    chat_model_id: int | None = 0
+    vision_model_id: int | None = 0
+    image_gen_model_id: int | None = 0
+    llm_setup_completed_at: datetime | None = None
+
+
+def _global_model(
+    *,
+    model_id: int = -1,
+    connection_id: int = -1,
+    enabled: bool = True,
+    supports_chat: bool = True,
+) -> dict:
+    return {
+        "id": model_id,
+        "connection_id": connection_id,
+        "enabled": enabled,
+        "supports_chat": supports_chat,
+        "capabilities_override": {},
+    }
+
+
+class TestGlobalCatalogHasUsableChat:
+    """Usability, not file existence, is what counts."""
+
+    def test_usable_when_enabled_connection_and_chat_model(self, monkeypatch):
+        monkeypatch.setattr(
+            mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": True}]
+        )
+        monkeypatch.setattr(mc.config, "GLOBAL_MODELS", [_global_model()])
+        assert mc._global_catalog_has_usable_chat() is True
+
+    def test_empty_catalog_is_not_usable(self, monkeypatch):
+        monkeypatch.setattr(mc.config, "GLOBAL_CONNECTIONS", [])
+        monkeypatch.setattr(mc.config, "GLOBAL_MODELS", [])
+        assert mc._global_catalog_has_usable_chat() is False
+
+    def test_disabled_connection_is_not_usable(self, monkeypatch):
+        monkeypatch.setattr(
+            mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": False}]
+        )
+        monkeypatch.setattr(mc.config, "GLOBAL_MODELS", [_global_model()])
+        assert mc._global_catalog_has_usable_chat() is False
+
+    def test_disabled_model_is_not_usable(self, monkeypatch):
+        monkeypatch.setattr(
+            mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": True}]
+        )
+        monkeypatch.setattr(mc.config, "GLOBAL_MODELS", [_global_model(enabled=False)])
+        assert mc._global_catalog_has_usable_chat() is False
+
+    def test_non_chat_model_is_not_usable(self, monkeypatch):
+        monkeypatch.setattr(
+            mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": True}]
+        )
+        monkeypatch.setattr(
+            mc.config, "GLOBAL_MODELS", [_global_model(supports_chat=False)]
+        )
+        assert mc._global_catalog_has_usable_chat() is False
+
+
+async def _run_status(
+    *,
+    file_exists: bool,
+    global_usable: bool,
+    chat_model_id: int,
+    ws_has_chat: bool = False,
+    permissions: list[str] | None = None,
+    workspace: _FakeWorkspace | None = None,
+):
+    """Drive the decision tree with DB-touching seams stubbed out.
+
+    Pass ``workspace`` to preset/inspect ``llm_setup_completed_at`` — it is the
+    object ``_clear_invalid_roles`` returns, and the lazy stamp mutates it in
+    place (``session.commit`` is a no-op AsyncMock).
+    """
+    if permissions is None:
+        permissions = [Permission.FULL_ACCESS.value]
+    if workspace is None:
+        workspace = _FakeWorkspace(chat_model_id=chat_model_id)
+    else:
+        workspace.chat_model_id = chat_model_id
+    with ExitStack() as stack:
+        stack.enter_context(
+            patch.object(mc.config, "GLOBAL_LLM_CONFIG_FILE_EXISTS", file_exists)
+        )
+        stack.enter_context(
+            patch.object(
+                mc, "_global_catalog_has_usable_chat", return_value=global_usable
+            )
+        )
+        stack.enter_context(patch.object(mc, "check_permission", AsyncMock()))
+        stack.enter_context(
+            patch.object(
+                mc, "get_user_permissions", AsyncMock(return_value=permissions)
+            )
+        )
+        stack.enter_context(
+            patch.object(
+                mc,
+                "_clear_invalid_roles",
+                AsyncMock(return_value=workspace),
+            )
+        )
+        stack.enter_context(
+            patch.object(
+                mc,
+                "_workspace_has_enabled_chat_model",
+                AsyncMock(return_value=ws_has_chat),
+            )
+        )
+        return await mc.compute_llm_setup_status(
+            AsyncMock(), AuthContext.session(_FakeUser()), 1
+        )
+
+
+class TestComputeLlmSetupStatus:
+    @pytest.mark.asyncio
+    async def test_no_yaml_no_models_needs_setup(self):
+        result = await _run_status(
+            file_exists=False, global_usable=False, chat_model_id=0, ws_has_chat=False
+        )
+        assert result.status == "needs_setup"
+        assert result.source == "none"
+        assert result.stage == "initial_setup"
+
+    @pytest.mark.asyncio
+    async def test_usable_global_catalog_is_ready(self):
+        ws = _FakeWorkspace()
+        result = await _run_status(
+            file_exists=True, global_usable=True, chat_model_id=0, workspace=ws
+        )
+        assert result.status == "ready"
+        assert result.source == "global_config"
+        assert result.stage == "ready"
+        # Global readiness is never stamped as this workspace's own setup.
+        assert ws.llm_setup_completed_at is None
+
+    @pytest.mark.asyncio
+    async def test_yaml_present_but_empty_catalog_falls_through(self):
+        # File exists but no usable model AND no BYOK => onboarding, not a
+        # dead composer. This is the empty/broken-YAML regression.
+        result = await _run_status(
+            file_exists=True, global_usable=False, chat_model_id=0, ws_has_chat=False
+        )
+        assert result.status == "needs_setup"
+        assert result.source == "none"
+        assert result.stage == "initial_setup"
+
+    @pytest.mark.asyncio
+    async def test_auto_mode_with_workspace_model_is_ready(self):
+        ws = _FakeWorkspace()
+        result = await _run_status(
+            file_exists=False,
+            global_usable=False,
+            chat_model_id=0,
+            ws_has_chat=True,
+            workspace=ws,
+        )
+        assert result.status == "ready"
+        assert result.source == "models"
+        assert result.stage == "ready"
+        assert ws.llm_setup_completed_at is not None
+
+    @pytest.mark.asyncio
+    async def test_auto_mode_counts_global_catalog_without_file(self):
+        result = await _run_status(
+            file_exists=False, global_usable=True, chat_model_id=0, ws_has_chat=False
+        )
+        assert result.status == "ready"
+        assert result.source == "global_config"
+
+    @pytest.mark.asyncio
+    async def test_pinned_workspace_model_is_ready(self):
+        # chat_model_id > 0 survived _clear_invalid_roles => valid + enabled.
+        ws = _FakeWorkspace()
+        result = await _run_status(
+            file_exists=False, global_usable=False, chat_model_id=5, workspace=ws
+        )
+        assert result.status == "ready"
+        assert result.source == "models"
+        assert result.stage == "ready"
+        assert ws.llm_setup_completed_at is not None
+
+    @pytest.mark.asyncio
+    async def test_pinned_global_model_is_ready(self):
+        ws = _FakeWorkspace()
+        result = await _run_status(
+            file_exists=False, global_usable=False, chat_model_id=-3, workspace=ws
+        )
+        assert result.status == "ready"
+        assert result.source == "global_config"
+        assert result.stage == "ready"
+        # A negative pin is global-config readiness, not own setup.
+        assert ws.llm_setup_completed_at is None
+
+    @pytest.mark.asyncio
+    async def test_pinned_dead_model_healed_to_needs_setup(self):
+        # A pin to a deleted/disabled model collapses to 0 in
+        # _clear_invalid_roles; with no fallback model it is needs_setup.
+        result = await _run_status(
+            file_exists=False, global_usable=False, chat_model_id=0, ws_has_chat=False
+        )
+        assert result.status == "needs_setup"
+
+    @pytest.mark.asyncio
+    async def test_can_configure_owner(self):
+        result = await _run_status(
+            file_exists=True,
+            global_usable=True,
+            chat_model_id=0,
+            permissions=[Permission.FULL_ACCESS.value],
+        )
+        assert result.can_configure is True
+
+    @pytest.mark.asyncio
+    async def test_can_configure_editor(self):
+        result = await _run_status(
+            file_exists=True,
+            global_usable=True,
+            chat_model_id=0,
+            permissions=[
+                Permission.LLM_CONFIGS_CREATE.value,
+                Permission.LLM_CONFIGS_READ.value,
+            ],
+        )
+        assert result.can_configure is True
+
+    @pytest.mark.asyncio
+    async def test_can_configure_viewer_is_false(self):
+        result = await _run_status(
+            file_exists=True,
+            global_usable=True,
+            chat_model_id=0,
+            permissions=[Permission.LLM_CONFIGS_READ.value],
+        )
+        assert result.can_configure is False
+
+
+class TestOnboardingStage:
+    """First-run vs. recovery: the durable timestamp splits needs_setup."""
+
+    @pytest.mark.asyncio
+    async def test_fresh_workspace_is_initial_setup(self):
+        result = await _run_status(
+            file_exists=False, global_usable=False, chat_model_id=0, ws_has_chat=False
+        )
+        assert result.stage == "initial_setup"
+
+    @pytest.mark.asyncio
+    async def test_previously_configured_then_lost_is_recovery(self):
+        # Configured before (timestamp set), then deleted: needs_setup but recovery.
+        ws = _FakeWorkspace(llm_setup_completed_at=datetime.now(UTC))
+        result = await _run_status(
+            file_exists=False,
+            global_usable=False,
+            chat_model_id=0,
+            ws_has_chat=False,
+            workspace=ws,
+        )
+        assert result.status == "needs_setup"
+        assert result.stage == "recovery"
+        assert ws.llm_setup_completed_at is not None  # preserved, never cleared
+
+    @pytest.mark.asyncio
+    async def test_stamp_is_not_overwritten_on_subsequent_ready(self):
+        original = datetime(2020, 1, 1, tzinfo=UTC)
+        ws = _FakeWorkspace(llm_setup_completed_at=original)
+        result = await _run_status(
+            file_exists=False,
+            global_usable=False,
+            chat_model_id=0,
+            ws_has_chat=True,
+            workspace=ws,
+        )
+        assert result.stage == "ready"
+        assert ws.llm_setup_completed_at == original
+
+    @pytest.mark.asyncio
+    async def test_global_only_loss_is_initial_setup_not_recovery(self):
+        # Rode global (never stamped), then global lost: a genuine first own-setup.
+        ws = _FakeWorkspace()
+        result = await _run_status(
+            file_exists=False,
+            global_usable=False,
+            chat_model_id=0,
+            ws_has_chat=False,
+            workspace=ws,
+        )
+        assert result.status == "needs_setup"
+        assert result.stage == "initial_setup"
+        assert ws.llm_setup_completed_at is None

surfsense_backend/tests/unit/tasks/celery_tasks/test_schedule_checker_task.py

@@ -59,7 +59,7 @@ def _due_connector(connector_type: SearchSourceConnectorType) -> SimpleNamespace
     return SimpleNamespace(
         id=42,
         connector_type=connector_type,
-        search_space_id=7,
+        workspace_id=7,
         user_id="00000000-0000-0000-0000-000000000001",
         config={},
         periodic_indexing_enabled=True,
@@ -86,13 +86,8 @@ async def _run_checker(monkeypatch: pytest.MonkeyPatch, connector: SimpleNamespa
 
 
 @pytest.mark.asyncio
-async def test_due_bookstack_connector_dispatches_indexing_task(monkeypatch):
-    """A due BookStack connector must dispatch index_bookstack_pages_task.
-
-    Regression test for the connector type missing from the scheduler's
-    task_map, which made periodic BookStack syncs silently no-op with only a
-    "No task found" warning.
-    """
+async def test_due_bookstack_connector_disables_deprecated_indexing(monkeypatch):
+    """A due BookStack connector is retired from periodic indexing, not dispatched."""
     from app.tasks.celery_tasks import connector_tasks
 
     task_mock = MagicMock()
@@ -101,16 +96,9 @@ async def test_due_bookstack_connector_dispatches_indexing_task(monkeypatch):
     connector = _due_connector(SearchSourceConnectorType.BOOKSTACK_CONNECTOR)
     session = await _run_checker(monkeypatch, connector)
 
-    task_mock.delay.assert_called_once_with(
-        connector.id,
-        connector.search_space_id,
-        str(connector.user_id),
-        None,
-        None,
-    )
-    # The next run must be rescheduled, otherwise the connector stays "due"
-    # and is re-examined every minute.
-    assert connector.next_scheduled_at > datetime.now(UTC)
+    task_mock.delay.assert_not_called()
+    assert connector.periodic_indexing_enabled is False
+    assert connector.next_scheduled_at is None
     assert session.commits == 1
 
 

surfsense_backend/tests/unit/utils/captcha/test_solvers.py

@@ -0,0 +1,90 @@
+"""Offline checks for the in-house captcha-solver seam.
+
+The network solve can't be unit-tested, but the boundary logic can: a wrong
+proxy reformat (2captcha's ``ERROR_PROXY_FORMAT``) or dispatching to a provider
+we haven't wired silently burns paid solves / leaks the key to the wrong vendor.
+"""
+
+import pytest
+
+from app.utils.captcha import solvers
+from app.utils.captcha.config import CaptchaConfig
+
+pytestmark = pytest.mark.unit
+
+
+def _cfg(**overrides) -> CaptchaConfig:
+    base = {
+        "enabled": True,
+        "solving_site": "2captcha",
+        "api_key": "key-123",
+        "max_attempts_per_url": 1,
+        "timeout_s": 30,
+        "captcha_type_default": "v2",
+        "v3_min_score": 0.7,
+        "v3_action": "verify",
+    }
+    base.update(overrides)
+    return CaptchaConfig(**base)
+
+
+# --- proxy reformat (2captcha wants login:pass@host:port, NO scheme) --------
+
+
+def test_proxy_login_form_strips_scheme():
+    got = solvers.proxy_login_form("http://user:pass@gw.dataimpulse.com:15673")
+    assert got == "user:pass@gw.dataimpulse.com:15673"
+
+
+def test_proxy_login_form_without_credentials():
+    assert (
+        solvers.proxy_login_form("http://gw.dataimpulse.com:823")
+        == "gw.dataimpulse.com:823"
+    )
+
+
+def test_proxy_login_form_none_on_missing_or_bad():
+    assert solvers.proxy_login_form(None) is None
+    assert solvers.proxy_login_form("not a url") is None
+
+
+# --- capsolver proxy reformat (colon-delimited scheme:host:port:user:pass) --
+
+
+def test_capsolver_proxy_colon_delimited_with_creds():
+    got = solvers.capsolver_proxy("http://user:pass@gw.dataimpulse.com:823")
+    assert got == "http:gw.dataimpulse.com:823:user:pass"
+
+
+def test_capsolver_proxy_without_credentials():
+    assert (
+        solvers.capsolver_proxy("http://gw.dataimpulse.com:823")
+        == "http:gw.dataimpulse.com:823"
+    )
+
+
+def test_capsolver_proxy_none_on_missing_or_bad():
+    assert solvers.capsolver_proxy(None) is None
+    assert solvers.capsolver_proxy("not a url") is None
+
+
+# --- dispatch --------------------------------------------------------------
+
+
+def test_unsupported_provider_raises_solvererror():
+    # An unconfigured provider must fail loudly (latch) rather than POST the key
+    # to a wired vendor's endpoint under the wrong account.
+    with pytest.raises(solvers.SolverUnsupportedError):
+        solvers.solve(
+            _cfg(solving_site="anticaptcha"),
+            challenge_type="v2",
+            sitekey="SK",
+            page_url="https://t.test",
+        )
+    assert issubclass(solvers.SolverUnsupportedError, solvers.SolverError)
+
+
+def test_wired_providers_are_registered():
+    supported = solvers.supported_providers()
+    assert "2captcha" in supported
+    assert "capsolver" in supported

surfsense_backend/tests/unit/utils/proxy/test_dataimpulse_provider.py

@@ -44,6 +44,51 @@ def test_location_stops_at_next_param(monkeypatch: pytest.MonkeyPatch) -> None:
     assert DataImpulseProvider().get_location() == "de"
 
 
+def test_geo_proxy_url_rewrites_country_suffix(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(Config, "PROXY_URL", _URL)
+
+    result = DataImpulseProvider().get_geo_proxy_url("gb")
+
+    assert result == "http://tok123__cr.gb:secret@gw.dataimpulse.com:823"
+
+
+def test_geo_proxy_url_replaces_existing_country_once(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(
+        Config,
+        "PROXY_URL",
+        "http://tok123__cr.us__sid.old:secret@gw.dataimpulse.com:823",
+    )
+
+    result = DataImpulseProvider().get_geo_proxy_url("de")
+
+    assert "__cr.us" not in result
+    assert result.count("__cr.de") == 1
+    assert "__sid.old" not in result
+
+
+def test_geo_proxy_url_without_country_keeps_base_url(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(Config, "PROXY_URL", _URL)
+
+    assert DataImpulseProvider().get_geo_proxy_url(None) == _URL
+    assert DataImpulseProvider().get_geo_proxy_url("") == _URL
+
+
+def test_sticky_proxy_url_composes_country_and_session(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(Config, "PROXY_URL", _URL)
+
+    result = DataImpulseProvider().get_sticky_proxy_url("location-123", "gb")
+
+    assert result == (
+        "http://tok123__cr.gb__sid.location-123:secret@gw.dataimpulse.com:823"
+    )
+
+
 def test_no_country_suffix_yields_empty_location(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:

surfsense_backend/tests/unit/utils/proxy/test_rotation.py

@@ -0,0 +1,53 @@
+"""Unit tests for the shared cross-country exit rotation helper.
+
+Offline: the provider is faked via monkeypatch, so no network/proxy is touched.
+"""
+
+from __future__ import annotations
+
+from app.utils.proxy import rotation
+
+
+class _Prov:
+    def __init__(self, location: str) -> None:
+        self._location = location
+
+    def get_location(self) -> str:
+        return self._location
+
+
+def test_lead_country_leads_and_dedupes(monkeypatch):
+    # The configured/default exit country leads and isn't duplicated in the walk.
+    monkeypatch.setattr(rotation, "get_active_provider", lambda: _Prov("gb"))
+    countries = rotation.rotation_countries()
+
+    assert countries[0] == "gb"
+    assert len(countries) == len(set(countries))  # de-duplicated
+    assert set(rotation.FALLBACK_COUNTRIES) <= set(countries)  # fallbacks kept
+
+
+def test_no_configured_country_uses_fallbacks_only(monkeypatch):
+    # A bare PROXY_URL (no country) leaves just the fallback pools, in order.
+    monkeypatch.setattr(rotation, "get_active_provider", lambda: _Prov(""))
+    assert rotation.rotation_countries() == rotation.FALLBACK_COUNTRIES
+
+
+def test_walk_covers_every_country_and_wraps(monkeypatch):
+    # A whole-pool block can't stall the walk: every country is reached, and the
+    # index cycles (wraps) rather than running off the end.
+    monkeypatch.setattr(rotation, "get_active_provider", lambda: _Prov("us"))
+    countries = rotation.rotation_countries()
+    tried = {rotation.country_for_rotation(n) for n in range(len(countries))}
+
+    assert tried == set(countries)
+    assert rotation.country_for_rotation(len(countries)) == countries[0]  # wraps
+
+
+def test_caller_budgets_cover_every_country():
+    # Each warm-on-block caller must budget enough rotations to try every pool at
+    # least once, else a wholly-blocked lead pool could fail a job prematurely.
+    from app.proprietary.platforms.reddit import fetch as reddit_fetch
+    from app.proprietary.platforms.tiktok.session import client as tiktok_client
+
+    assert len(rotation.FALLBACK_COUNTRIES) <= reddit_fetch._MAX_ROTATIONS
+    assert len(rotation.FALLBACK_COUNTRIES) <= tiktok_client._MAX_ROTATIONS