apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-20 23:21:06 +02:00
Code Issues Projects Releases Packages Wiki Activity

chore: linting

Browse source
This commit is contained in:
DESKTOP-RTLN3BA\$punk 2026-07-17 14:39:39 -07:00
parent be35eb8743
commit c0ebb62fb2
20 changed files with 158 additions and 125 deletions
Download patch file Download diff file Expand all files Collapse all files

3
surfsense_backend/app/proprietary/platforms/google_search/captcha.py
View file

@ -29,8 +29,7 @@ import re
import time
from typing import Any
from app.utils.captcha import CaptchaConfig
from app.utils.captcha import solvers
from app.utils.captcha import CaptchaConfig, solvers
logger = logging.getLogger(__name__)

11
surfsense_backend/app/proprietary/platforms/google_search/fetch.py
View file

@ -42,8 +42,10 @@ from urllib.parse import urlsplit, urlunsplit
from scrapling.fetchers import AsyncFetcher
from app.proprietary.platforms.google_search import captcha as _captcha
from app.proprietary.platforms.google_search import pool_store as _store
from app.proprietary.platforms.google_search import (
captcha as _captcha,
pool_store as _store,
)
from app.utils.captcha import captcha_enabled, get_captcha_config
from app.utils.proxy import get_proxy_url
@ -374,9 +376,8 @@ def _make_page_action(proxy: str | None, cfg):
"""
async def page_action(page):
if _captcha.on_sorry(page):
if await _captcha.solve_sorry(page, proxy, cfg):
_exemption_jar[proxy or ""] = await _captcha.exemption_cookies(page)
if _captcha.on_sorry(page) and await _captcha.solve_sorry(page, proxy, cfg):
_exemption_jar[proxy or ""] = await _captcha.exemption_cookies(page)
return await _expand_blocks(page)
return page_action

2
surfsense_backend/app/proprietary/platforms/google_search/pool_store.py
View file

@ -15,7 +15,7 @@ never block the request loop and don't care which loop the caller is on.
``ponytail:`` the shared store holds only exemptions (the costly artifact), not
per-process render concurrency — global per-IP load is still governed by each
process's local per-IP cap × the number of processes, so size the pool for the
process's local per-IP cap x the number of processes, so size the pool for the
fleet (see ``GOOGLE_SEARCH_WARM_POOL_TARGET``). Full distributed inflight
accounting is the upgrade path if a single shared IP ever gets overloaded.
"""

5
surfsense_backend/app/proprietary/web_crawler/captcha.py
View file

@ -299,11 +299,10 @@ def _is_unrecoverable(exc: Exception) -> bool:
"""True for solver errors that must latch solving off.
Covers the in-house seam's typed errors (``SolverBalanceError`` /
``SolverAuthError`` / ``SolverUnsupported``) plus legacy/no-balance shapes.
``SolverAuthError`` / ``SolverUnsupportedError``) plus legacy/no-balance shapes.
Matched by class name so no solver module must be imported here.
"""
name = type(exc).__name__.lower()
return any(
k in name
for k in ("balance", "apikey", "auth", "unsupported", "wronguser")
k in name for k in ("balance", "apikey", "auth", "unsupported", "wronguser")
)

2
surfsense_backend/app/routes/model_connections_routes.py
View file

@ -863,7 +863,7 @@ async def _workspace_has_enabled_chat_model(
.options(selectinload(Connection.models))
.where(
Connection.workspace_id == workspace_id,
Connection.enabled == True,
Connection.enabled,
)
)
return any(

12
surfsense_backend/app/utils/captcha/solvers.py
View file

@ -10,7 +10,7 @@ the reCAPTCHA-*Enterprise* ``/sorry`` wall). Both express the Enterprise pieces
Google needs — the widget sitekey plus the page's dynamic ``data-s`` token
(something ``captchatools`` could not). More vendors (anticaptcha / capmonster)
are added progressively as new entries in :data:`_PROVIDERS`; until then an
unconfigured provider raises :class:`SolverUnsupported` so callers latch off
unconfigured provider raises :class:`SolverUnsupportedError` so callers latch off
cleanly instead of leaking the API key to the wrong service.
"""
@ -41,7 +41,7 @@ class SolverBalanceError(SolverError):
"""Solver account is out of balance — unrecoverable this process."""
class SolverUnsupported(SolverError):
class SolverUnsupportedError(SolverError):
"""Configured provider has no in-house client yet — unrecoverable."""
@ -108,7 +108,11 @@ def _twocaptcha(
"min_score": cfg.v3_min_score,
}
else: # v2, optionally the Enterprise variant (adds enterprise=1 + data-s)
payload |= {"method": "userrecaptcha", "googlekey": sitekey, "pageurl": page_url}
payload |= {
"method": "userrecaptcha",
"googlekey": sitekey,
"pageurl": page_url,
}
if enterprise:
payload["enterprise"] = 1
if data_s:
@ -295,7 +299,7 @@ def solve(
"""
client = _PROVIDERS.get((cfg.solving_site or "").lower())
if client is None:
raise SolverUnsupported(
raise SolverUnsupportedError(
f"captcha provider {cfg.solving_site!r} has no in-house client "
f"(supported: {supported_providers()})"
)

45
surfsense_backend/scripts/scale_google_search.py
View file

@ -26,7 +26,6 @@ from __future__ import annotations
import argparse
import asyncio
import statistics
import sys
import threading
import time
@ -181,7 +180,9 @@ async def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--count", type=int, default=400)
ap.add_argument("--rate", type=float, default=60.0, help="arrivals/sec (sim clock)")
ap.add_argument("--warm", type=float, default=0.10, help="warm render seconds (sim)")
ap.add_argument(
"--warm", type=float, default=0.10, help="warm render seconds (sim)"
)
ap.add_argument("--solve", type=float, default=0.45, help="solve seconds (sim)")
ap.add_argument("--precheck", type=float, default=0.01)
args = ap.parse_args()
@ -206,23 +207,37 @@ async def main() -> None:
thru_real = thru_sim / scale
print("\n=== Google Search scale simulation ===")
print(f" gate (_MAX_CONCURRENT_PAGES) = {gate}"
+ (f", warm-pool target = {pool}" if pool else " (single-slot sticky IP)"))
print(
f" gate (_MAX_CONCURRENT_PAGES) = {gate}"
+ (f", warm-pool target = {pool}" if pool else " (single-slot sticky IP)")
)
print(f" requests={args.count} arrival_rate={args.rate}/s (sim)")
print(f" --- structure (scale-free) ---")
print(f" paid solves = {metrics.solves} (want ~pool size, not ~requests)")
print(" --- structure (scale-free) ---")
print(
f" paid solves = {metrics.solves} (want ~pool size, not ~requests)"
)
print(f" distinct sticky IPs = {len(metrics.ip_hits)}")
print(f" busiest IP carried = {top_hits}/{metrics.renders} renders "
f"({top_share:.0f}%) (funneling; want ~even spread)")
print(
f" busiest IP carried = {top_hits}/{metrics.renders} renders "
f"({top_share:.0f}%) (funneling; want ~even spread)"
)
print(f" peak renders on 1 IP = {hot_peak} (concurrency; capped by gate)")
print(f" --- throughput / latency (extrapolated to live @ warm={_REAL_WARM_S}s) ---")
print(f" throughput = {thru_real*60:.0f} SERP/min ({thru_real:.2f}/s)")
print(f" latency p50 = {_pct(lat,50)*scale:6.1f}s p95 = {_pct(lat,95)*scale:6.1f}s "
f"p99 = {_pct(lat,99)*scale:6.1f}s")
print(f" ceiling (gate/warm) = {gate/_REAL_WARM_S*60:.0f} SERP/min per process")
print(
f" --- throughput / latency (extrapolated to live @ warm={_REAL_WARM_S}s) ---"
)
print(f" throughput = {thru_real * 60:.0f} SERP/min ({thru_real:.2f}/s)")
print(
f" latency p50 = {_pct(lat, 50) * scale:6.1f}s p95 = {_pct(lat, 95) * scale:6.1f}s "
f"p99 = {_pct(lat, 99) * scale:6.1f}s"
)
print(
f" ceiling (gate/warm) = {gate / _REAL_WARM_S * 60:.0f} SERP/min per process"
)
need = 500 / (gate / _REAL_WARM_S * 60)
print(f" -> to sustain 500 SERP/min you need ~{need:.0f} such processes "
f"(or a larger gate)\n")
print(
f" -> to sustain 500 SERP/min you need ~{need:.0f} such processes "
f"(or a larger gate)\n"
)
if __name__ == "__main__":

38
surfsense_backend/scripts/stress_google_search.py
View file

@ -51,9 +51,7 @@ _INFO = [
class _LogTally(logging.Handler):
"""Counts solves / renders / walls / pool-reuse from the live log stream."""
_RENDER = re.compile(
r"has_results=(\w+).*from_pool=(\w+) pool=(\d+)"
)
_RENDER = re.compile(r"has_results=(\w+).*from_pool=(\w+) pool=(\d+)")
_SOLVE = re.compile(r"\[captcha\] solve (OK|did not)")
def __init__(self) -> None:
@ -147,7 +145,7 @@ async def main() -> None:
lg.setLevel(logging.INFO)
lg.addHandler(tally)
from app.proprietary.platforms.google_search.fetch import ( # noqa: E402
from app.proprietary.platforms.google_search.fetch import (
_WARM_POOL_TARGET,
close_sessions,
)
@ -157,9 +155,11 @@ async def main() -> None:
total_serps = sum(w for _, w in plans)
n_multi = sum(1 for _, w in plans if w > 1)
print(f"\n=== LIVE stress: {args.users} users "
f"({args.users - n_multi} single / {n_multi} multi), "
f"~{total_serps} SERPs, gate={args.gate}, pool_target={_WARM_POOL_TARGET} ===")
print(
f"\n=== LIVE stress: {args.users} users "
f"({args.users - n_multi} single / {n_multi} multi), "
f"~{total_serps} SERPs, gate={args.gate}, pool_target={_WARM_POOL_TARGET} ==="
)
print(" (real solves + proxy spend; warming up...)\n")
results: list[dict] = []
@ -184,17 +184,25 @@ async def main() -> None:
print("=== results ===")
print(f" wall time = {wall:.0f}s")
print(f" requests ok/failed = {len(results) - len(fails)}/{len(fails)}")
print(f" SERPs got/expected = {got_serps}/{total_serps}"
+ (f" ({len(short)} short)" if short else ""))
print(
f" SERPs got/expected = {got_serps}/{total_serps}"
+ (f" ({len(short)} short)" if short else "")
)
print(f" throughput = {got_serps / wall * 60:.0f} SERP/min")
print(f" request latency p50={pct(50):.0f}s p95={pct(95):.0f}s "
f"max={lat[-1] if lat else 0:.0f}s")
print(
f" request latency p50={pct(50):.0f}s p95={pct(95):.0f}s "
f"max={lat[-1] if lat else 0:.0f}s"
)
print(" --- pipeline (from live logs) ---")
print(f" paid solves ok/fail = {tally.solves_ok}/{tally.solves_fail} "
f"(bounded by pool target {_WARM_POOL_TARGET})")
print(
f" paid solves ok/fail = {tally.solves_ok}/{tally.solves_fail} "
f"(bounded by pool target {_WARM_POOL_TARGET})"
)
print(f" renders ok/walled = {tally.renders_ok}/{tally.renders_walled}")
print(f" pool reuse/grow = {tally.reuse}/{tally.grow} "
f"(reuse share {100 * tally.reuse / max(1, tally.reuse + tally.grow):.0f}%)")
print(
f" pool reuse/grow = {tally.reuse}/{tally.grow} "
f"(reuse share {100 * tally.reuse / max(1, tally.reuse + tally.grow):.0f}%)"
)
print(f" peak pool size = {tally.max_pool}")
if fails:
print(" --- failures ---")

16
surfsense_backend/tests/unit/routes/test_llm_setup_status.py
View file

@ -58,7 +58,9 @@ class TestGlobalCatalogHasUsableChat:
"""Usability, not file existence, is what counts."""
def test_usable_when_enabled_connection_and_chat_model(self, monkeypatch):
monkeypatch.setattr(mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": True}])
monkeypatch.setattr(
mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": True}]
)
monkeypatch.setattr(mc.config, "GLOBAL_MODELS", [_global_model()])
assert mc._global_catalog_has_usable_chat() is True
@ -68,17 +70,23 @@ class TestGlobalCatalogHasUsableChat:
assert mc._global_catalog_has_usable_chat() is False
def test_disabled_connection_is_not_usable(self, monkeypatch):
monkeypatch.setattr(mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": False}])
monkeypatch.setattr(
mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": False}]
)
monkeypatch.setattr(mc.config, "GLOBAL_MODELS", [_global_model()])
assert mc._global_catalog_has_usable_chat() is False
def test_disabled_model_is_not_usable(self, monkeypatch):
monkeypatch.setattr(mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": True}])
monkeypatch.setattr(
mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": True}]
)
monkeypatch.setattr(mc.config, "GLOBAL_MODELS", [_global_model(enabled=False)])
assert mc._global_catalog_has_usable_chat() is False
def test_non_chat_model_is_not_usable(self, monkeypatch):
monkeypatch.setattr(mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": True}])
monkeypatch.setattr(
mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": True}]
)
monkeypatch.setattr(
mc.config, "GLOBAL_MODELS", [_global_model(supports_chat=False)]
)

14
surfsense_backend/tests/unit/utils/captcha/test_solvers.py
View file

@ -37,7 +37,10 @@ def test_proxy_login_form_strips_scheme():
def test_proxy_login_form_without_credentials():
assert solvers.proxy_login_form("http://gw.dataimpulse.com:823") == "gw.dataimpulse.com:823"
assert (
solvers.proxy_login_form("http://gw.dataimpulse.com:823")
== "gw.dataimpulse.com:823"
)
def test_proxy_login_form_none_on_missing_or_bad():
@ -54,7 +57,10 @@ def test_capsolver_proxy_colon_delimited_with_creds():
def test_capsolver_proxy_without_credentials():
assert solvers.capsolver_proxy("http://gw.dataimpulse.com:823") == "http:gw.dataimpulse.com:823"
assert (
solvers.capsolver_proxy("http://gw.dataimpulse.com:823")
== "http:gw.dataimpulse.com:823"
)
def test_capsolver_proxy_none_on_missing_or_bad():
@ -68,14 +74,14 @@ def test_capsolver_proxy_none_on_missing_or_bad():
def test_unsupported_provider_raises_solvererror():
# An unconfigured provider must fail loudly (latch) rather than POST the key
# to a wired vendor's endpoint under the wrong account.
with pytest.raises(solvers.SolverUnsupported):
with pytest.raises(solvers.SolverUnsupportedError):
solvers.solve(
_cfg(solving_site="anticaptcha"),
challenge_type="v2",
sitekey="SK",
page_url="https://t.test",
)
assert issubclass(solvers.SolverUnsupported, solvers.SolverError)
assert issubclass(solvers.SolverUnsupportedError, solvers.SolverError)
def test_wired_providers_are_registered():