refactor: enhance Google search scraping capabilities with warm sticky-IP pool and improved captcha handling

surfsense_backend/tests/unit/platforms/google_search/test_captcha.py

@@ -0,0 +1,37 @@
+"""Offline checks for the /sorry reCAPTCHA solver's pure helpers.
+
+The network solve can't be unit-tested, but the parsing around it can: a missed
+sitekey/data-s silently burns paid solves, so these guard the boundary logic.
+(Proxy reformatting now lives in ``app.utils.captcha.solvers``.)
+"""
+
+from app.proprietary.platforms.google_search import captcha
+
+
+class _Page:
+    def __init__(self, url: str) -> None:
+        self.url = url
+
+
+def test_on_sorry_detects_wall_only():
+    assert captcha.on_sorry(_Page("https://www.google.com/sorry/index?continue=x"))
+    assert not captcha.on_sorry(_Page("https://www.google.com/search?q=notebooklm"))
+    assert not captcha.on_sorry(_Page(""))
+
+
+def test_sitekey_and_data_s_extraction():
+    html = (
+        '<div class="g-recaptcha" data-sitekey="6LdLLIMbAAAAAIl-KLj9p1ePhM"'
+        ' data-s="zBB1ixry9YzY_tok-en"></div>'
+    )
+    assert captcha._SITEKEY_RE.search(html).group(1) == "6LdLLIMbAAAAAIl-KLj9p1ePhM"
+    assert captcha._DATA_S_RE.search(html).group(1) == "zBB1ixry9YzY_tok-en"
+
+
+def test_latch_roundtrip():
+    captcha.reset_solver_latch()
+    assert not captcha.solver_latched()
+    captcha._latch("no balance")
+    assert captcha.solver_latched()
+    captcha.reset_solver_latch()
+    assert not captcha.solver_latched()

surfsense_backend/tests/unit/platforms/google_search/test_fetch_concurrency.py

@@ -16,7 +16,7 @@ class _FakeSession:
         self.release = asyncio.Event()
         self.closed = 0
 
-    async def fetch(self, url, proxy=None):
+    async def fetch(self, url, proxy=None, **kwargs):
         await self.release.wait()
         return "page"
 

surfsense_backend/tests/unit/platforms/google_search/test_fetch_pool.py

@@ -0,0 +1,99 @@
+"""Warm sticky-IP pool bookkeeping (the scale-critical logic in fetch.py).
+
+These pure, lock-guarded helpers decide whether a render reuses a warm IP,
+grows the pool (a paid solve), or waits — and how a finished render admits or
+evicts its IP. Get the pending/inflight accounting wrong and the pool either
+over-solves (cost) or funnels every render onto one IP (re-wall), so the
+transitions are worth pinning down offline.
+"""
+
+import pytest
+
+from app.proprietary.platforms.google_search import fetch
+
+pytestmark = pytest.mark.unit
+
+
+@pytest.fixture(autouse=True)
+def _clean_pool():
+    def reset():
+        fetch._pool.clear()
+        fetch._pool_inflight.clear()
+        fetch._pool_pending = 0
+        fetch._exemption_jar.clear()
+
+    reset()
+    yield
+    reset()
+
+
+def test_take_grows_when_pool_empty():
+    action, proxy = fetch._pool_take()
+    assert action == "grow" and proxy is None
+    assert fetch._pool_pending == 1
+
+
+def test_take_reuses_warm_under_cap():
+    fetch._pool["p1"] = 1.0
+    action, proxy = fetch._pool_take()
+    assert action == "reuse" and proxy == "p1"
+    assert fetch._pool_inflight["p1"] == 1
+
+
+def test_take_spreads_across_least_loaded_ip():
+    fetch._pool.update({"p1": 1.0, "p2": 1.0})
+    fetch._pool_inflight["p1"] = 1  # p2 is idle
+    action, proxy = fetch._pool_take()
+    assert action == "reuse" and proxy == "p2"
+
+
+def test_take_waits_when_full_and_every_ip_capped(monkeypatch):
+    monkeypatch.setattr(fetch, "_WARM_POOL_TARGET", 2)
+    monkeypatch.setattr(fetch, "_WARM_IP_MAX_CONCURRENCY", 1)
+    fetch._pool.update({"p1": 1.0, "p2": 1.0})
+    fetch._pool_inflight.update({"p1": 1, "p2": 1})
+    action, proxy = fetch._pool_take()
+    assert action == "wait" and proxy is None
+    assert fetch._pool_pending == 0  # a wait must NOT reserve a solve
+
+
+def test_take_grows_only_up_to_target_counting_pending(monkeypatch):
+    monkeypatch.setattr(fetch, "_WARM_POOL_TARGET", 2)
+    monkeypatch.setattr(fetch, "_WARM_IP_MAX_CONCURRENCY", 1)
+    fetch._pool["p1"] = 1.0
+    fetch._pool_inflight["p1"] = 1  # warm but capped
+    assert fetch._pool_take()[0] == "grow"  # pool(1)+pending(0) < 2
+    assert fetch._pool_pending == 1
+    assert fetch._pool_take()[0] == "wait"  # pool(1)+pending(1) == 2 → no more solves
+
+
+def test_settle_good_admits_and_releases():
+    fetch._pool_pending = 1
+    fetch._pool_inflight["p1"] = 1
+    fetch._pool_settle("p1", good=True, grew=True)
+    assert "p1" in fetch._pool
+    assert "p1" not in fetch._pool_inflight
+    assert fetch._pool_pending == 0
+
+
+def test_settle_walled_evicts_ip_and_drops_exemption():
+    fetch._pool["p1"] = 1.0
+    fetch._pool_inflight["p1"] = 1
+    fetch._exemption_jar["p1"] = [{"name": "GOOGLE_ABUSE_EXEMPTION"}]
+    fetch._pool_settle("p1", good=False, grew=False)
+    assert "p1" not in fetch._pool
+    assert "p1" not in fetch._exemption_jar
+
+
+def test_adopt_releases_pending_and_pins_ip():
+    fetch._pool_pending = 1  # reserved by a grow that we satisfy via the store
+    fetch._pool_adopt("shared")
+    assert fetch._pool_pending == 0
+    assert "shared" in fetch._pool
+    assert fetch._pool_inflight["shared"] == 1
+
+
+def test_abort_grow_releases_the_reserved_slot():
+    fetch._pool_pending = 2
+    fetch._pool_abort_grow()
+    assert fetch._pool_pending == 1

surfsense_backend/tests/unit/platforms/google_search/test_pool_store.py

@@ -0,0 +1,73 @@
+"""Cross-process warm-IP exemption store (best-effort Redis cache).
+
+Redis itself isn't unit-testable here, but the boundary is: adopt must skip IPs
+this process already holds (or the fleet re-uses nothing new), and every path
+must degrade to a silent no-op when Redis is down (or a hiccup would break the
+fetch instead of just costing a solve).
+"""
+
+import pytest
+
+from app.proprietary.platforms.google_search import pool_store as ps
+
+pytestmark = pytest.mark.unit
+
+
+class _FakeRedis:
+    def __init__(self):
+        self.store: dict[str, str] = {}
+
+    def ping(self):
+        return True
+
+    def set(self, k, v, ex=None):
+        self.store[k] = v
+
+    def get(self, k):
+        return self.store.get(k)
+
+    def delete(self, k):
+        self.store.pop(k, None)
+
+    def scan_iter(self, match=None, count=None):
+        return list(self.store.keys())
+
+
+@pytest.fixture
+def fake(monkeypatch):
+    r = _FakeRedis()
+    monkeypatch.setattr(ps, "_client", r)
+    monkeypatch.setattr(ps, "_disabled", False)
+    return r
+
+
+def test_key_is_stable_and_prefixed():
+    assert ps._key("http://a").startswith(ps._PREFIX)
+    assert ps._key("http://a") == ps._key("http://a")
+    assert ps._key("http://a") != ps._key("http://b")
+
+
+def test_publish_then_adopt_excludes_held_ips(fake):
+    ps._publish_sync("http://a", [{"n": 1}])
+    ps._publish_sync("http://b", [{"n": 2}])
+    proxy, cookies = ps._adopt_sync(exclude={"http://a"})
+    assert proxy == "http://b" and cookies == [{"n": 2}]
+
+
+def test_adopt_none_when_all_held(fake):
+    ps._publish_sync("http://a", [])
+    assert ps._adopt_sync(exclude={"http://a"}) is None
+
+
+def test_evict_removes_entry(fake):
+    ps._publish_sync("http://a", [{"n": 1}])
+    ps._evict_sync("http://a")
+    assert ps._adopt_sync(exclude=set()) is None
+
+
+def test_disabled_is_silent_noop(monkeypatch):
+    monkeypatch.setattr(ps, "_disabled", True)
+    monkeypatch.setattr(ps, "_client", None)
+    assert ps._adopt_sync(set()) is None
+    ps._publish_sync("x", [{"n": 1}])  # must not raise
+    ps._evict_sync("x")  # must not raise

surfsense_backend/tests/unit/proprietary/web_crawler/test_captcha.py

@@ -2,8 +2,9 @@
 
 The browser/solver boundary is mocked: a fake Playwright ``page`` and a
 monkeypatched ``_harvest_token`` / injection. We assert the glue logic —
-detection, proxy reformatting, attempt counting + state surfacing, the per-URL
-cap, and the no-balance process latch.
+detection, proxy pass-through, attempt counting + state surfacing, the per-URL
+cap, and the no-balance process latch. (Vendor proxy reformatting now lives in
+``app.utils.captcha.solvers`` and is tested there.)
 """
 
 import pytest
@@ -76,24 +77,6 @@ def _clear_latch():
     cap.reset_solver_latch()
 
 
-# --- proxy reformat --------------------------------------------------------
-
-
-class TestProxyReformat:
-    def test_with_auth(self):
-        assert (
-            cap.proxy_url_to_captchatools("http://user:pass@1.2.3.4:8080")
-            == "1.2.3.4:8080:user:pass"
-        )
-
-    def test_without_auth(self):
-        assert cap.proxy_url_to_captchatools("http://1.2.3.4:8080") == "1.2.3.4:8080"
-
-    def test_none_and_garbage(self):
-        assert cap.proxy_url_to_captchatools(None) is None
-        assert cap.proxy_url_to_captchatools("not-a-url") is None
-
-
 # --- detection -------------------------------------------------------------
 
 
@@ -175,8 +158,9 @@ class TestPageAction:
         action(page)
 
         assert state == {"attempts": 1, "solved": True}
-        # Solver egressed from the crawl's proxy, reformatted, with the page UA.
-        assert captured["proxy"] == "1.2.3.4:9000:u:p"
+        # Solver egressed from the crawl's proxy (raw URL; the seam reformats
+        # per vendor), with the page UA.
+        assert captured["proxy"] == "http://u:p@1.2.3.4:9000"
         assert captured["ua"] == "UA/1.0"
         assert captured["ctype"] == "v2"
         assert injected == {"ctype": "v2", "token": "TOKEN"}

surfsense_backend/tests/unit/utils/captcha/test_solvers.py

@@ -0,0 +1,84 @@
+"""Offline checks for the in-house captcha-solver seam.
+
+The network solve can't be unit-tested, but the boundary logic can: a wrong
+proxy reformat (2captcha's ``ERROR_PROXY_FORMAT``) or dispatching to a provider
+we haven't wired silently burns paid solves / leaks the key to the wrong vendor.
+"""
+
+import pytest
+
+from app.utils.captcha import solvers
+from app.utils.captcha.config import CaptchaConfig
+
+pytestmark = pytest.mark.unit
+
+
+def _cfg(**overrides) -> CaptchaConfig:
+    base = {
+        "enabled": True,
+        "solving_site": "2captcha",
+        "api_key": "key-123",
+        "max_attempts_per_url": 1,
+        "timeout_s": 30,
+        "captcha_type_default": "v2",
+        "v3_min_score": 0.7,
+        "v3_action": "verify",
+    }
+    base.update(overrides)
+    return CaptchaConfig(**base)
+
+
+# --- proxy reformat (2captcha wants login:pass@host:port, NO scheme) --------
+
+
+def test_proxy_login_form_strips_scheme():
+    got = solvers.proxy_login_form("http://user:pass@gw.dataimpulse.com:15673")
+    assert got == "user:pass@gw.dataimpulse.com:15673"
+
+
+def test_proxy_login_form_without_credentials():
+    assert solvers.proxy_login_form("http://gw.dataimpulse.com:823") == "gw.dataimpulse.com:823"
+
+
+def test_proxy_login_form_none_on_missing_or_bad():
+    assert solvers.proxy_login_form(None) is None
+    assert solvers.proxy_login_form("not a url") is None
+
+
+# --- capsolver proxy reformat (colon-delimited scheme:host:port:user:pass) --
+
+
+def test_capsolver_proxy_colon_delimited_with_creds():
+    got = solvers.capsolver_proxy("http://user:pass@gw.dataimpulse.com:823")
+    assert got == "http:gw.dataimpulse.com:823:user:pass"
+
+
+def test_capsolver_proxy_without_credentials():
+    assert solvers.capsolver_proxy("http://gw.dataimpulse.com:823") == "http:gw.dataimpulse.com:823"
+
+
+def test_capsolver_proxy_none_on_missing_or_bad():
+    assert solvers.capsolver_proxy(None) is None
+    assert solvers.capsolver_proxy("not a url") is None
+
+
+# --- dispatch --------------------------------------------------------------
+
+
+def test_unsupported_provider_raises_solvererror():
+    # An unconfigured provider must fail loudly (latch) rather than POST the key
+    # to a wired vendor's endpoint under the wrong account.
+    with pytest.raises(solvers.SolverUnsupported):
+        solvers.solve(
+            _cfg(solving_site="anticaptcha"),
+            challenge_type="v2",
+            sitekey="SK",
+            page_url="https://t.test",
+        )
+    assert issubclass(solvers.SolverUnsupported, solvers.SolverError)
+
+
+def test_wired_providers_are_registered():
+    supported = solvers.supported_providers()
+    assert "2captcha" in supported
+    assert "capsolver" in supported