diff --git a/surfsense_backend/app/proprietary/platforms/google_search/captcha.py b/surfsense_backend/app/proprietary/platforms/google_search/captcha.py
index 2de1e9a0b..f140a975a 100644
--- a/surfsense_backend/app/proprietary/platforms/google_search/captcha.py
+++ b/surfsense_backend/app/proprietary/platforms/google_search/captcha.py
@@ -29,8 +29,7 @@ import re
import time
from typing import Any
-from app.utils.captcha import CaptchaConfig
-from app.utils.captcha import solvers
+from app.utils.captcha import CaptchaConfig, solvers
logger = logging.getLogger(__name__)
diff --git a/surfsense_backend/app/proprietary/platforms/google_search/fetch.py b/surfsense_backend/app/proprietary/platforms/google_search/fetch.py
index f11820047..49c6fcabd 100644
--- a/surfsense_backend/app/proprietary/platforms/google_search/fetch.py
+++ b/surfsense_backend/app/proprietary/platforms/google_search/fetch.py
@@ -42,8 +42,10 @@ from urllib.parse import urlsplit, urlunsplit
from scrapling.fetchers import AsyncFetcher
-from app.proprietary.platforms.google_search import captcha as _captcha
-from app.proprietary.platforms.google_search import pool_store as _store
+from app.proprietary.platforms.google_search import (
+ captcha as _captcha,
+ pool_store as _store,
+)
from app.utils.captcha import captcha_enabled, get_captcha_config
from app.utils.proxy import get_proxy_url
@@ -374,9 +376,8 @@ def _make_page_action(proxy: str | None, cfg):
"""
async def page_action(page):
- if _captcha.on_sorry(page):
- if await _captcha.solve_sorry(page, proxy, cfg):
- _exemption_jar[proxy or ""] = await _captcha.exemption_cookies(page)
+ if _captcha.on_sorry(page) and await _captcha.solve_sorry(page, proxy, cfg):
+ _exemption_jar[proxy or ""] = await _captcha.exemption_cookies(page)
return await _expand_blocks(page)
return page_action
diff --git a/surfsense_backend/app/proprietary/platforms/google_search/pool_store.py b/surfsense_backend/app/proprietary/platforms/google_search/pool_store.py
index f7271a54c..449224252 100644
--- a/surfsense_backend/app/proprietary/platforms/google_search/pool_store.py
+++ b/surfsense_backend/app/proprietary/platforms/google_search/pool_store.py
@@ -15,7 +15,7 @@ never block the request loop and don't care which loop the caller is on.
``ponytail:`` the shared store holds only exemptions (the costly artifact), not
per-process render concurrency — global per-IP load is still governed by each
-process's local per-IP cap × the number of processes, so size the pool for the
+process's local per-IP cap x the number of processes, so size the pool for the
fleet (see ``GOOGLE_SEARCH_WARM_POOL_TARGET``). Full distributed inflight
accounting is the upgrade path if a single shared IP ever gets overloaded.
"""
diff --git a/surfsense_backend/app/proprietary/web_crawler/captcha.py b/surfsense_backend/app/proprietary/web_crawler/captcha.py
index b585a98fc..d092bdfd2 100644
--- a/surfsense_backend/app/proprietary/web_crawler/captcha.py
+++ b/surfsense_backend/app/proprietary/web_crawler/captcha.py
@@ -299,11 +299,10 @@ def _is_unrecoverable(exc: Exception) -> bool:
"""True for solver errors that must latch solving off.
Covers the in-house seam's typed errors (``SolverBalanceError`` /
- ``SolverAuthError`` / ``SolverUnsupported``) plus legacy/no-balance shapes.
+ ``SolverAuthError`` / ``SolverUnsupportedError``) plus legacy/no-balance shapes.
Matched by class name so no solver module must be imported here.
"""
name = type(exc).__name__.lower()
return any(
- k in name
- for k in ("balance", "apikey", "auth", "unsupported", "wronguser")
+ k in name for k in ("balance", "apikey", "auth", "unsupported", "wronguser")
)
diff --git a/surfsense_backend/app/routes/model_connections_routes.py b/surfsense_backend/app/routes/model_connections_routes.py
index 68d9d3da2..a97a70660 100644
--- a/surfsense_backend/app/routes/model_connections_routes.py
+++ b/surfsense_backend/app/routes/model_connections_routes.py
@@ -863,7 +863,7 @@ async def _workspace_has_enabled_chat_model(
.options(selectinload(Connection.models))
.where(
Connection.workspace_id == workspace_id,
- Connection.enabled == True,
+ Connection.enabled,
)
)
return any(
diff --git a/surfsense_backend/app/utils/captcha/solvers.py b/surfsense_backend/app/utils/captcha/solvers.py
index db157f873..510268466 100644
--- a/surfsense_backend/app/utils/captcha/solvers.py
+++ b/surfsense_backend/app/utils/captcha/solvers.py
@@ -10,7 +10,7 @@ the reCAPTCHA-*Enterprise* ``/sorry`` wall). Both express the Enterprise pieces
Google needs — the widget sitekey plus the page's dynamic ``data-s`` token
(something ``captchatools`` could not). More vendors (anticaptcha / capmonster)
are added progressively as new entries in :data:`_PROVIDERS`; until then an
-unconfigured provider raises :class:`SolverUnsupported` so callers latch off
+unconfigured provider raises :class:`SolverUnsupportedError` so callers latch off
cleanly instead of leaking the API key to the wrong service.
"""
@@ -41,7 +41,7 @@ class SolverBalanceError(SolverError):
"""Solver account is out of balance — unrecoverable this process."""
-class SolverUnsupported(SolverError):
+class SolverUnsupportedError(SolverError):
"""Configured provider has no in-house client yet — unrecoverable."""
@@ -108,7 +108,11 @@ def _twocaptcha(
"min_score": cfg.v3_min_score,
}
else: # v2, optionally the Enterprise variant (adds enterprise=1 + data-s)
- payload |= {"method": "userrecaptcha", "googlekey": sitekey, "pageurl": page_url}
+ payload |= {
+ "method": "userrecaptcha",
+ "googlekey": sitekey,
+ "pageurl": page_url,
+ }
if enterprise:
payload["enterprise"] = 1
if data_s:
@@ -295,7 +299,7 @@ def solve(
"""
client = _PROVIDERS.get((cfg.solving_site or "").lower())
if client is None:
- raise SolverUnsupported(
+ raise SolverUnsupportedError(
f"captcha provider {cfg.solving_site!r} has no in-house client "
f"(supported: {supported_providers()})"
)
diff --git a/surfsense_backend/scripts/scale_google_search.py b/surfsense_backend/scripts/scale_google_search.py
index 5e4084b3e..df2b0b2cf 100644
--- a/surfsense_backend/scripts/scale_google_search.py
+++ b/surfsense_backend/scripts/scale_google_search.py
@@ -26,7 +26,6 @@ from __future__ import annotations
import argparse
import asyncio
-import statistics
import sys
import threading
import time
@@ -181,7 +180,9 @@ async def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--count", type=int, default=400)
ap.add_argument("--rate", type=float, default=60.0, help="arrivals/sec (sim clock)")
- ap.add_argument("--warm", type=float, default=0.10, help="warm render seconds (sim)")
+ ap.add_argument(
+ "--warm", type=float, default=0.10, help="warm render seconds (sim)"
+ )
ap.add_argument("--solve", type=float, default=0.45, help="solve seconds (sim)")
ap.add_argument("--precheck", type=float, default=0.01)
args = ap.parse_args()
@@ -206,23 +207,37 @@ async def main() -> None:
thru_real = thru_sim / scale
print("\n=== Google Search scale simulation ===")
- print(f" gate (_MAX_CONCURRENT_PAGES) = {gate}"
- + (f", warm-pool target = {pool}" if pool else " (single-slot sticky IP)"))
+ print(
+ f" gate (_MAX_CONCURRENT_PAGES) = {gate}"
+ + (f", warm-pool target = {pool}" if pool else " (single-slot sticky IP)")
+ )
print(f" requests={args.count} arrival_rate={args.rate}/s (sim)")
- print(f" --- structure (scale-free) ---")
- print(f" paid solves = {metrics.solves} (want ~pool size, not ~requests)")
+ print(" --- structure (scale-free) ---")
+ print(
+ f" paid solves = {metrics.solves} (want ~pool size, not ~requests)"
+ )
print(f" distinct sticky IPs = {len(metrics.ip_hits)}")
- print(f" busiest IP carried = {top_hits}/{metrics.renders} renders "
- f"({top_share:.0f}%) (funneling; want ~even spread)")
+ print(
+ f" busiest IP carried = {top_hits}/{metrics.renders} renders "
+ f"({top_share:.0f}%) (funneling; want ~even spread)"
+ )
print(f" peak renders on 1 IP = {hot_peak} (concurrency; capped by gate)")
- print(f" --- throughput / latency (extrapolated to live @ warm={_REAL_WARM_S}s) ---")
- print(f" throughput = {thru_real*60:.0f} SERP/min ({thru_real:.2f}/s)")
- print(f" latency p50 = {_pct(lat,50)*scale:6.1f}s p95 = {_pct(lat,95)*scale:6.1f}s "
- f"p99 = {_pct(lat,99)*scale:6.1f}s")
- print(f" ceiling (gate/warm) = {gate/_REAL_WARM_S*60:.0f} SERP/min per process")
+ print(
+ f" --- throughput / latency (extrapolated to live @ warm={_REAL_WARM_S}s) ---"
+ )
+ print(f" throughput = {thru_real * 60:.0f} SERP/min ({thru_real:.2f}/s)")
+ print(
+ f" latency p50 = {_pct(lat, 50) * scale:6.1f}s p95 = {_pct(lat, 95) * scale:6.1f}s "
+ f"p99 = {_pct(lat, 99) * scale:6.1f}s"
+ )
+ print(
+ f" ceiling (gate/warm) = {gate / _REAL_WARM_S * 60:.0f} SERP/min per process"
+ )
need = 500 / (gate / _REAL_WARM_S * 60)
- print(f" -> to sustain 500 SERP/min you need ~{need:.0f} such processes "
- f"(or a larger gate)\n")
+ print(
+ f" -> to sustain 500 SERP/min you need ~{need:.0f} such processes "
+ f"(or a larger gate)\n"
+ )
if __name__ == "__main__":
diff --git a/surfsense_backend/scripts/stress_google_search.py b/surfsense_backend/scripts/stress_google_search.py
index e080a16d5..9856a35c2 100644
--- a/surfsense_backend/scripts/stress_google_search.py
+++ b/surfsense_backend/scripts/stress_google_search.py
@@ -51,9 +51,7 @@ _INFO = [
class _LogTally(logging.Handler):
"""Counts solves / renders / walls / pool-reuse from the live log stream."""
- _RENDER = re.compile(
- r"has_results=(\w+).*from_pool=(\w+) pool=(\d+)"
- )
+ _RENDER = re.compile(r"has_results=(\w+).*from_pool=(\w+) pool=(\d+)")
_SOLVE = re.compile(r"\[captcha\] solve (OK|did not)")
def __init__(self) -> None:
@@ -147,7 +145,7 @@ async def main() -> None:
lg.setLevel(logging.INFO)
lg.addHandler(tally)
- from app.proprietary.platforms.google_search.fetch import ( # noqa: E402
+ from app.proprietary.platforms.google_search.fetch import (
_WARM_POOL_TARGET,
close_sessions,
)
@@ -157,9 +155,11 @@ async def main() -> None:
total_serps = sum(w for _, w in plans)
n_multi = sum(1 for _, w in plans if w > 1)
- print(f"\n=== LIVE stress: {args.users} users "
- f"({args.users - n_multi} single / {n_multi} multi), "
- f"~{total_serps} SERPs, gate={args.gate}, pool_target={_WARM_POOL_TARGET} ===")
+ print(
+ f"\n=== LIVE stress: {args.users} users "
+ f"({args.users - n_multi} single / {n_multi} multi), "
+ f"~{total_serps} SERPs, gate={args.gate}, pool_target={_WARM_POOL_TARGET} ==="
+ )
print(" (real solves + proxy spend; warming up...)\n")
results: list[dict] = []
@@ -184,17 +184,25 @@ async def main() -> None:
print("=== results ===")
print(f" wall time = {wall:.0f}s")
print(f" requests ok/failed = {len(results) - len(fails)}/{len(fails)}")
- print(f" SERPs got/expected = {got_serps}/{total_serps}"
- + (f" ({len(short)} short)" if short else ""))
+ print(
+ f" SERPs got/expected = {got_serps}/{total_serps}"
+ + (f" ({len(short)} short)" if short else "")
+ )
print(f" throughput = {got_serps / wall * 60:.0f} SERP/min")
- print(f" request latency p50={pct(50):.0f}s p95={pct(95):.0f}s "
- f"max={lat[-1] if lat else 0:.0f}s")
+ print(
+ f" request latency p50={pct(50):.0f}s p95={pct(95):.0f}s "
+ f"max={lat[-1] if lat else 0:.0f}s"
+ )
print(" --- pipeline (from live logs) ---")
- print(f" paid solves ok/fail = {tally.solves_ok}/{tally.solves_fail} "
- f"(bounded by pool target {_WARM_POOL_TARGET})")
+ print(
+ f" paid solves ok/fail = {tally.solves_ok}/{tally.solves_fail} "
+ f"(bounded by pool target {_WARM_POOL_TARGET})"
+ )
print(f" renders ok/walled = {tally.renders_ok}/{tally.renders_walled}")
- print(f" pool reuse/grow = {tally.reuse}/{tally.grow} "
- f"(reuse share {100 * tally.reuse / max(1, tally.reuse + tally.grow):.0f}%)")
+ print(
+ f" pool reuse/grow = {tally.reuse}/{tally.grow} "
+ f"(reuse share {100 * tally.reuse / max(1, tally.reuse + tally.grow):.0f}%)"
+ )
print(f" peak pool size = {tally.max_pool}")
if fails:
print(" --- failures ---")
diff --git a/surfsense_backend/tests/unit/routes/test_llm_setup_status.py b/surfsense_backend/tests/unit/routes/test_llm_setup_status.py
index 2f07d8bd2..bbfad6689 100644
--- a/surfsense_backend/tests/unit/routes/test_llm_setup_status.py
+++ b/surfsense_backend/tests/unit/routes/test_llm_setup_status.py
@@ -58,7 +58,9 @@ class TestGlobalCatalogHasUsableChat:
"""Usability, not file existence, is what counts."""
def test_usable_when_enabled_connection_and_chat_model(self, monkeypatch):
- monkeypatch.setattr(mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": True}])
+ monkeypatch.setattr(
+ mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": True}]
+ )
monkeypatch.setattr(mc.config, "GLOBAL_MODELS", [_global_model()])
assert mc._global_catalog_has_usable_chat() is True
@@ -68,17 +70,23 @@ class TestGlobalCatalogHasUsableChat:
assert mc._global_catalog_has_usable_chat() is False
def test_disabled_connection_is_not_usable(self, monkeypatch):
- monkeypatch.setattr(mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": False}])
+ monkeypatch.setattr(
+ mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": False}]
+ )
monkeypatch.setattr(mc.config, "GLOBAL_MODELS", [_global_model()])
assert mc._global_catalog_has_usable_chat() is False
def test_disabled_model_is_not_usable(self, monkeypatch):
- monkeypatch.setattr(mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": True}])
+ monkeypatch.setattr(
+ mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": True}]
+ )
monkeypatch.setattr(mc.config, "GLOBAL_MODELS", [_global_model(enabled=False)])
assert mc._global_catalog_has_usable_chat() is False
def test_non_chat_model_is_not_usable(self, monkeypatch):
- monkeypatch.setattr(mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": True}])
+ monkeypatch.setattr(
+ mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": True}]
+ )
monkeypatch.setattr(
mc.config, "GLOBAL_MODELS", [_global_model(supports_chat=False)]
)
diff --git a/surfsense_backend/tests/unit/utils/captcha/test_solvers.py b/surfsense_backend/tests/unit/utils/captcha/test_solvers.py
index 6af4597c2..2bfdcad7e 100644
--- a/surfsense_backend/tests/unit/utils/captcha/test_solvers.py
+++ b/surfsense_backend/tests/unit/utils/captcha/test_solvers.py
@@ -37,7 +37,10 @@ def test_proxy_login_form_strips_scheme():
def test_proxy_login_form_without_credentials():
- assert solvers.proxy_login_form("http://gw.dataimpulse.com:823") == "gw.dataimpulse.com:823"
+ assert (
+ solvers.proxy_login_form("http://gw.dataimpulse.com:823")
+ == "gw.dataimpulse.com:823"
+ )
def test_proxy_login_form_none_on_missing_or_bad():
@@ -54,7 +57,10 @@ def test_capsolver_proxy_colon_delimited_with_creds():
def test_capsolver_proxy_without_credentials():
- assert solvers.capsolver_proxy("http://gw.dataimpulse.com:823") == "http:gw.dataimpulse.com:823"
+ assert (
+ solvers.capsolver_proxy("http://gw.dataimpulse.com:823")
+ == "http:gw.dataimpulse.com:823"
+ )
def test_capsolver_proxy_none_on_missing_or_bad():
@@ -68,14 +74,14 @@ def test_capsolver_proxy_none_on_missing_or_bad():
def test_unsupported_provider_raises_solvererror():
# An unconfigured provider must fail loudly (latch) rather than POST the key
# to a wired vendor's endpoint under the wrong account.
- with pytest.raises(solvers.SolverUnsupported):
+ with pytest.raises(solvers.SolverUnsupportedError):
solvers.solve(
_cfg(solving_site="anticaptcha"),
challenge_type="v2",
sitekey="SK",
page_url="https://t.test",
)
- assert issubclass(solvers.SolverUnsupported, solvers.SolverError)
+ assert issubclass(solvers.SolverUnsupportedError, solvers.SolverError)
def test_wired_providers_are_registered():
diff --git a/surfsense_web/app/dashboard/[workspace_id]/layout.tsx b/surfsense_web/app/dashboard/[workspace_id]/layout.tsx
index 4ef79076e..b372236c4 100644
--- a/surfsense_web/app/dashboard/[workspace_id]/layout.tsx
+++ b/surfsense_web/app/dashboard/[workspace_id]/layout.tsx
@@ -1,6 +1,7 @@
// Server component
-import type React from "react";
+
import { cookies } from "next/headers";
+import type React from "react";
import { DashboardClientLayout } from "./client-layout";
const PLAYGROUND_SIDEBAR_COLLAPSED_COOKIE = "surfsense_playground_sidebar_collapsed";
diff --git a/surfsense_web/components/assistant-ui/connector-popup/components/connector-dialog-header.tsx b/surfsense_web/components/assistant-ui/connector-popup/components/connector-dialog-header.tsx
index 61d1ef844..0105296f2 100644
--- a/surfsense_web/components/assistant-ui/connector-popup/components/connector-dialog-header.tsx
+++ b/surfsense_web/components/assistant-ui/connector-popup/components/connector-dialog-header.tsx
@@ -34,7 +34,7 @@ export const ConnectorDialogHeader: FC