apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-20 23:21:06 +02:00
Code Issues Projects Releases Packages Wiki Activity

chore: linting

Browse source
This commit is contained in:
DESKTOP-RTLN3BA\$punk 2026-07-17 14:39:39 -07:00
parent be35eb8743
commit c0ebb62fb2
20 changed files with 158 additions and 125 deletions
Download patch file Download diff file Expand all files Collapse all files

45
surfsense_backend/scripts/scale_google_search.py
View file

@ -26,7 +26,6 @@ from __future__ import annotations
import argparse
import asyncio
import statistics
import sys
import threading
import time
@ -181,7 +180,9 @@ async def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--count", type=int, default=400)
ap.add_argument("--rate", type=float, default=60.0, help="arrivals/sec (sim clock)")
ap.add_argument("--warm", type=float, default=0.10, help="warm render seconds (sim)")
ap.add_argument(
"--warm", type=float, default=0.10, help="warm render seconds (sim)"
)
ap.add_argument("--solve", type=float, default=0.45, help="solve seconds (sim)")
ap.add_argument("--precheck", type=float, default=0.01)
args = ap.parse_args()
@ -206,23 +207,37 @@ async def main() -> None:
thru_real = thru_sim / scale
print("\n=== Google Search scale simulation ===")
print(f" gate (_MAX_CONCURRENT_PAGES) = {gate}"
+ (f", warm-pool target = {pool}" if pool else " (single-slot sticky IP)"))
print(
f" gate (_MAX_CONCURRENT_PAGES) = {gate}"
+ (f", warm-pool target = {pool}" if pool else " (single-slot sticky IP)")
)
print(f" requests={args.count} arrival_rate={args.rate}/s (sim)")
print(f" --- structure (scale-free) ---")
print(f" paid solves = {metrics.solves} (want ~pool size, not ~requests)")
print(" --- structure (scale-free) ---")
print(
f" paid solves = {metrics.solves} (want ~pool size, not ~requests)"
)
print(f" distinct sticky IPs = {len(metrics.ip_hits)}")
print(f" busiest IP carried = {top_hits}/{metrics.renders} renders "
f"({top_share:.0f}%) (funneling; want ~even spread)")
print(
f" busiest IP carried = {top_hits}/{metrics.renders} renders "
f"({top_share:.0f}%) (funneling; want ~even spread)"
)
print(f" peak renders on 1 IP = {hot_peak} (concurrency; capped by gate)")
print(f" --- throughput / latency (extrapolated to live @ warm={_REAL_WARM_S}s) ---")
print(f" throughput = {thru_real*60:.0f} SERP/min ({thru_real:.2f}/s)")
print(f" latency p50 = {_pct(lat,50)*scale:6.1f}s p95 = {_pct(lat,95)*scale:6.1f}s "
f"p99 = {_pct(lat,99)*scale:6.1f}s")
print(f" ceiling (gate/warm) = {gate/_REAL_WARM_S*60:.0f} SERP/min per process")
print(
f" --- throughput / latency (extrapolated to live @ warm={_REAL_WARM_S}s) ---"
)
print(f" throughput = {thru_real * 60:.0f} SERP/min ({thru_real:.2f}/s)")
print(
f" latency p50 = {_pct(lat, 50) * scale:6.1f}s p95 = {_pct(lat, 95) * scale:6.1f}s "
f"p99 = {_pct(lat, 99) * scale:6.1f}s"
)
print(
f" ceiling (gate/warm) = {gate / _REAL_WARM_S * 60:.0f} SERP/min per process"
)
need = 500 / (gate / _REAL_WARM_S * 60)
print(f" -> to sustain 500 SERP/min you need ~{need:.0f} such processes "
f"(or a larger gate)\n")
print(
f" -> to sustain 500 SERP/min you need ~{need:.0f} such processes "
f"(or a larger gate)\n"
)
if __name__ == "__main__":

38
surfsense_backend/scripts/stress_google_search.py
View file

@ -51,9 +51,7 @@ _INFO = [
class _LogTally(logging.Handler):
"""Counts solves / renders / walls / pool-reuse from the live log stream."""
_RENDER = re.compile(
r"has_results=(\w+).*from_pool=(\w+) pool=(\d+)"
)
_RENDER = re.compile(r"has_results=(\w+).*from_pool=(\w+) pool=(\d+)")
_SOLVE = re.compile(r"\[captcha\] solve (OK|did not)")
def __init__(self) -> None:
@ -147,7 +145,7 @@ async def main() -> None:
lg.setLevel(logging.INFO)
lg.addHandler(tally)
from app.proprietary.platforms.google_search.fetch import ( # noqa: E402
from app.proprietary.platforms.google_search.fetch import (
_WARM_POOL_TARGET,
close_sessions,
)
@ -157,9 +155,11 @@ async def main() -> None:
total_serps = sum(w for _, w in plans)
n_multi = sum(1 for _, w in plans if w > 1)
print(f"\n=== LIVE stress: {args.users} users "
f"({args.users - n_multi} single / {n_multi} multi), "
f"~{total_serps} SERPs, gate={args.gate}, pool_target={_WARM_POOL_TARGET} ===")
print(
f"\n=== LIVE stress: {args.users} users "
f"({args.users - n_multi} single / {n_multi} multi), "
f"~{total_serps} SERPs, gate={args.gate}, pool_target={_WARM_POOL_TARGET} ==="
)
print(" (real solves + proxy spend; warming up...)\n")
results: list[dict] = []
@ -184,17 +184,25 @@ async def main() -> None:
print("=== results ===")
print(f" wall time = {wall:.0f}s")
print(f" requests ok/failed = {len(results) - len(fails)}/{len(fails)}")
print(f" SERPs got/expected = {got_serps}/{total_serps}"
+ (f" ({len(short)} short)" if short else ""))
print(
f" SERPs got/expected = {got_serps}/{total_serps}"
+ (f" ({len(short)} short)" if short else "")
)
print(f" throughput = {got_serps / wall * 60:.0f} SERP/min")
print(f" request latency p50={pct(50):.0f}s p95={pct(95):.0f}s "
f"max={lat[-1] if lat else 0:.0f}s")
print(
f" request latency p50={pct(50):.0f}s p95={pct(95):.0f}s "
f"max={lat[-1] if lat else 0:.0f}s"
)
print(" --- pipeline (from live logs) ---")
print(f" paid solves ok/fail = {tally.solves_ok}/{tally.solves_fail} "
f"(bounded by pool target {_WARM_POOL_TARGET})")
print(
f" paid solves ok/fail = {tally.solves_ok}/{tally.solves_fail} "
f"(bounded by pool target {_WARM_POOL_TARGET})"
)
print(f" renders ok/walled = {tally.renders_ok}/{tally.renders_walled}")
print(f" pool reuse/grow = {tally.reuse}/{tally.grow} "
f"(reuse share {100 * tally.reuse / max(1, tally.reuse + tally.grow):.0f}%)")
print(
f" pool reuse/grow = {tally.reuse}/{tally.grow} "
f"(reuse share {100 * tally.reuse / max(1, tally.reuse + tally.grow):.0f}%)"
)
print(f" peak pool size = {tally.max_pool}")
if fails:
print(" --- failures ---")