chore: linting

This commit is contained in:
DESKTOP-RTLN3BA\$punk 2026-07-17 14:39:39 -07:00
parent be35eb8743
commit c0ebb62fb2
20 changed files with 158 additions and 125 deletions

View file

@ -29,8 +29,7 @@ import re
import time
from typing import Any
from app.utils.captcha import CaptchaConfig
from app.utils.captcha import solvers
from app.utils.captcha import CaptchaConfig, solvers
logger = logging.getLogger(__name__)

View file

@ -42,8 +42,10 @@ from urllib.parse import urlsplit, urlunsplit
from scrapling.fetchers import AsyncFetcher
from app.proprietary.platforms.google_search import captcha as _captcha
from app.proprietary.platforms.google_search import pool_store as _store
from app.proprietary.platforms.google_search import (
captcha as _captcha,
pool_store as _store,
)
from app.utils.captcha import captcha_enabled, get_captcha_config
from app.utils.proxy import get_proxy_url
@ -374,9 +376,8 @@ def _make_page_action(proxy: str | None, cfg):
"""
async def page_action(page):
if _captcha.on_sorry(page):
if await _captcha.solve_sorry(page, proxy, cfg):
_exemption_jar[proxy or ""] = await _captcha.exemption_cookies(page)
if _captcha.on_sorry(page) and await _captcha.solve_sorry(page, proxy, cfg):
_exemption_jar[proxy or ""] = await _captcha.exemption_cookies(page)
return await _expand_blocks(page)
return page_action

View file

@ -15,7 +15,7 @@ never block the request loop and don't care which loop the caller is on.
``ponytail:`` the shared store holds only exemptions (the costly artifact), not
per-process render concurrency global per-IP load is still governed by each
process's local per-IP cap × the number of processes, so size the pool for the
process's local per-IP cap x the number of processes, so size the pool for the
fleet (see ``GOOGLE_SEARCH_WARM_POOL_TARGET``). Full distributed inflight
accounting is the upgrade path if a single shared IP ever gets overloaded.
"""

View file

@ -299,11 +299,10 @@ def _is_unrecoverable(exc: Exception) -> bool:
"""True for solver errors that must latch solving off.
Covers the in-house seam's typed errors (``SolverBalanceError`` /
``SolverAuthError`` / ``SolverUnsupported``) plus legacy/no-balance shapes.
``SolverAuthError`` / ``SolverUnsupportedError``) plus legacy/no-balance shapes.
Matched by class name so no solver module must be imported here.
"""
name = type(exc).__name__.lower()
return any(
k in name
for k in ("balance", "apikey", "auth", "unsupported", "wronguser")
k in name for k in ("balance", "apikey", "auth", "unsupported", "wronguser")
)

View file

@ -863,7 +863,7 @@ async def _workspace_has_enabled_chat_model(
.options(selectinload(Connection.models))
.where(
Connection.workspace_id == workspace_id,
Connection.enabled == True,
Connection.enabled,
)
)
return any(

View file

@ -10,7 +10,7 @@ the reCAPTCHA-*Enterprise* ``/sorry`` wall). Both express the Enterprise pieces
Google needs the widget sitekey plus the page's dynamic ``data-s`` token
(something ``captchatools`` could not). More vendors (anticaptcha / capmonster)
are added progressively as new entries in :data:`_PROVIDERS`; until then an
unconfigured provider raises :class:`SolverUnsupported` so callers latch off
unconfigured provider raises :class:`SolverUnsupportedError` so callers latch off
cleanly instead of leaking the API key to the wrong service.
"""
@ -41,7 +41,7 @@ class SolverBalanceError(SolverError):
"""Solver account is out of balance — unrecoverable this process."""
class SolverUnsupported(SolverError):
class SolverUnsupportedError(SolverError):
"""Configured provider has no in-house client yet — unrecoverable."""
@ -108,7 +108,11 @@ def _twocaptcha(
"min_score": cfg.v3_min_score,
}
else: # v2, optionally the Enterprise variant (adds enterprise=1 + data-s)
payload |= {"method": "userrecaptcha", "googlekey": sitekey, "pageurl": page_url}
payload |= {
"method": "userrecaptcha",
"googlekey": sitekey,
"pageurl": page_url,
}
if enterprise:
payload["enterprise"] = 1
if data_s:
@ -295,7 +299,7 @@ def solve(
"""
client = _PROVIDERS.get((cfg.solving_site or "").lower())
if client is None:
raise SolverUnsupported(
raise SolverUnsupportedError(
f"captcha provider {cfg.solving_site!r} has no in-house client "
f"(supported: {supported_providers()})"
)

View file

@ -26,7 +26,6 @@ from __future__ import annotations
import argparse
import asyncio
import statistics
import sys
import threading
import time
@ -181,7 +180,9 @@ async def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--count", type=int, default=400)
ap.add_argument("--rate", type=float, default=60.0, help="arrivals/sec (sim clock)")
ap.add_argument("--warm", type=float, default=0.10, help="warm render seconds (sim)")
ap.add_argument(
"--warm", type=float, default=0.10, help="warm render seconds (sim)"
)
ap.add_argument("--solve", type=float, default=0.45, help="solve seconds (sim)")
ap.add_argument("--precheck", type=float, default=0.01)
args = ap.parse_args()
@ -206,23 +207,37 @@ async def main() -> None:
thru_real = thru_sim / scale
print("\n=== Google Search scale simulation ===")
print(f" gate (_MAX_CONCURRENT_PAGES) = {gate}"
+ (f", warm-pool target = {pool}" if pool else " (single-slot sticky IP)"))
print(
f" gate (_MAX_CONCURRENT_PAGES) = {gate}"
+ (f", warm-pool target = {pool}" if pool else " (single-slot sticky IP)")
)
print(f" requests={args.count} arrival_rate={args.rate}/s (sim)")
print(f" --- structure (scale-free) ---")
print(f" paid solves = {metrics.solves} (want ~pool size, not ~requests)")
print(" --- structure (scale-free) ---")
print(
f" paid solves = {metrics.solves} (want ~pool size, not ~requests)"
)
print(f" distinct sticky IPs = {len(metrics.ip_hits)}")
print(f" busiest IP carried = {top_hits}/{metrics.renders} renders "
f"({top_share:.0f}%) (funneling; want ~even spread)")
print(
f" busiest IP carried = {top_hits}/{metrics.renders} renders "
f"({top_share:.0f}%) (funneling; want ~even spread)"
)
print(f" peak renders on 1 IP = {hot_peak} (concurrency; capped by gate)")
print(f" --- throughput / latency (extrapolated to live @ warm={_REAL_WARM_S}s) ---")
print(f" throughput = {thru_real*60:.0f} SERP/min ({thru_real:.2f}/s)")
print(f" latency p50 = {_pct(lat,50)*scale:6.1f}s p95 = {_pct(lat,95)*scale:6.1f}s "
f"p99 = {_pct(lat,99)*scale:6.1f}s")
print(f" ceiling (gate/warm) = {gate/_REAL_WARM_S*60:.0f} SERP/min per process")
print(
f" --- throughput / latency (extrapolated to live @ warm={_REAL_WARM_S}s) ---"
)
print(f" throughput = {thru_real * 60:.0f} SERP/min ({thru_real:.2f}/s)")
print(
f" latency p50 = {_pct(lat, 50) * scale:6.1f}s p95 = {_pct(lat, 95) * scale:6.1f}s "
f"p99 = {_pct(lat, 99) * scale:6.1f}s"
)
print(
f" ceiling (gate/warm) = {gate / _REAL_WARM_S * 60:.0f} SERP/min per process"
)
need = 500 / (gate / _REAL_WARM_S * 60)
print(f" -> to sustain 500 SERP/min you need ~{need:.0f} such processes "
f"(or a larger gate)\n")
print(
f" -> to sustain 500 SERP/min you need ~{need:.0f} such processes "
f"(or a larger gate)\n"
)
if __name__ == "__main__":

View file

@ -51,9 +51,7 @@ _INFO = [
class _LogTally(logging.Handler):
"""Counts solves / renders / walls / pool-reuse from the live log stream."""
_RENDER = re.compile(
r"has_results=(\w+).*from_pool=(\w+) pool=(\d+)"
)
_RENDER = re.compile(r"has_results=(\w+).*from_pool=(\w+) pool=(\d+)")
_SOLVE = re.compile(r"\[captcha\] solve (OK|did not)")
def __init__(self) -> None:
@ -147,7 +145,7 @@ async def main() -> None:
lg.setLevel(logging.INFO)
lg.addHandler(tally)
from app.proprietary.platforms.google_search.fetch import ( # noqa: E402
from app.proprietary.platforms.google_search.fetch import (
_WARM_POOL_TARGET,
close_sessions,
)
@ -157,9 +155,11 @@ async def main() -> None:
total_serps = sum(w for _, w in plans)
n_multi = sum(1 for _, w in plans if w > 1)
print(f"\n=== LIVE stress: {args.users} users "
f"({args.users - n_multi} single / {n_multi} multi), "
f"~{total_serps} SERPs, gate={args.gate}, pool_target={_WARM_POOL_TARGET} ===")
print(
f"\n=== LIVE stress: {args.users} users "
f"({args.users - n_multi} single / {n_multi} multi), "
f"~{total_serps} SERPs, gate={args.gate}, pool_target={_WARM_POOL_TARGET} ==="
)
print(" (real solves + proxy spend; warming up...)\n")
results: list[dict] = []
@ -184,17 +184,25 @@ async def main() -> None:
print("=== results ===")
print(f" wall time = {wall:.0f}s")
print(f" requests ok/failed = {len(results) - len(fails)}/{len(fails)}")
print(f" SERPs got/expected = {got_serps}/{total_serps}"
+ (f" ({len(short)} short)" if short else ""))
print(
f" SERPs got/expected = {got_serps}/{total_serps}"
+ (f" ({len(short)} short)" if short else "")
)
print(f" throughput = {got_serps / wall * 60:.0f} SERP/min")
print(f" request latency p50={pct(50):.0f}s p95={pct(95):.0f}s "
f"max={lat[-1] if lat else 0:.0f}s")
print(
f" request latency p50={pct(50):.0f}s p95={pct(95):.0f}s "
f"max={lat[-1] if lat else 0:.0f}s"
)
print(" --- pipeline (from live logs) ---")
print(f" paid solves ok/fail = {tally.solves_ok}/{tally.solves_fail} "
f"(bounded by pool target {_WARM_POOL_TARGET})")
print(
f" paid solves ok/fail = {tally.solves_ok}/{tally.solves_fail} "
f"(bounded by pool target {_WARM_POOL_TARGET})"
)
print(f" renders ok/walled = {tally.renders_ok}/{tally.renders_walled}")
print(f" pool reuse/grow = {tally.reuse}/{tally.grow} "
f"(reuse share {100 * tally.reuse / max(1, tally.reuse + tally.grow):.0f}%)")
print(
f" pool reuse/grow = {tally.reuse}/{tally.grow} "
f"(reuse share {100 * tally.reuse / max(1, tally.reuse + tally.grow):.0f}%)"
)
print(f" peak pool size = {tally.max_pool}")
if fails:
print(" --- failures ---")

View file

@ -58,7 +58,9 @@ class TestGlobalCatalogHasUsableChat:
"""Usability, not file existence, is what counts."""
def test_usable_when_enabled_connection_and_chat_model(self, monkeypatch):
monkeypatch.setattr(mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": True}])
monkeypatch.setattr(
mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": True}]
)
monkeypatch.setattr(mc.config, "GLOBAL_MODELS", [_global_model()])
assert mc._global_catalog_has_usable_chat() is True
@ -68,17 +70,23 @@ class TestGlobalCatalogHasUsableChat:
assert mc._global_catalog_has_usable_chat() is False
def test_disabled_connection_is_not_usable(self, monkeypatch):
monkeypatch.setattr(mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": False}])
monkeypatch.setattr(
mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": False}]
)
monkeypatch.setattr(mc.config, "GLOBAL_MODELS", [_global_model()])
assert mc._global_catalog_has_usable_chat() is False
def test_disabled_model_is_not_usable(self, monkeypatch):
monkeypatch.setattr(mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": True}])
monkeypatch.setattr(
mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": True}]
)
monkeypatch.setattr(mc.config, "GLOBAL_MODELS", [_global_model(enabled=False)])
assert mc._global_catalog_has_usable_chat() is False
def test_non_chat_model_is_not_usable(self, monkeypatch):
monkeypatch.setattr(mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": True}])
monkeypatch.setattr(
mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": True}]
)
monkeypatch.setattr(
mc.config, "GLOBAL_MODELS", [_global_model(supports_chat=False)]
)

View file

@ -37,7 +37,10 @@ def test_proxy_login_form_strips_scheme():
def test_proxy_login_form_without_credentials():
assert solvers.proxy_login_form("http://gw.dataimpulse.com:823") == "gw.dataimpulse.com:823"
assert (
solvers.proxy_login_form("http://gw.dataimpulse.com:823")
== "gw.dataimpulse.com:823"
)
def test_proxy_login_form_none_on_missing_or_bad():
@ -54,7 +57,10 @@ def test_capsolver_proxy_colon_delimited_with_creds():
def test_capsolver_proxy_without_credentials():
assert solvers.capsolver_proxy("http://gw.dataimpulse.com:823") == "http:gw.dataimpulse.com:823"
assert (
solvers.capsolver_proxy("http://gw.dataimpulse.com:823")
== "http:gw.dataimpulse.com:823"
)
def test_capsolver_proxy_none_on_missing_or_bad():
@ -68,14 +74,14 @@ def test_capsolver_proxy_none_on_missing_or_bad():
def test_unsupported_provider_raises_solvererror():
# An unconfigured provider must fail loudly (latch) rather than POST the key
# to a wired vendor's endpoint under the wrong account.
with pytest.raises(solvers.SolverUnsupported):
with pytest.raises(solvers.SolverUnsupportedError):
solvers.solve(
_cfg(solving_site="anticaptcha"),
challenge_type="v2",
sitekey="SK",
page_url="https://t.test",
)
assert issubclass(solvers.SolverUnsupported, solvers.SolverError)
assert issubclass(solvers.SolverUnsupportedError, solvers.SolverError)
def test_wired_providers_are_registered():

View file

@ -1,6 +1,7 @@
// Server component
import type React from "react";
import { cookies } from "next/headers";
import type React from "react";
import { DashboardClientLayout } from "./client-layout";
const PLAYGROUND_SIDEBAR_COLLAPSED_COOKIE = "surfsense_playground_sidebar_collapsed";

View file

@ -34,7 +34,7 @@ export const ConnectorDialogHeader: FC<ConnectorDialogHeaderProps> = ({
MCP Connectors
</DialogTitle>
<DialogDescription className="text-xs sm:text-base text-muted-foreground/80 mt-1 sm:mt-1.5">
Connect external tools and services through MCP.
Connect external tools and services through MCP.
</DialogDescription>
</DialogHeader>

View file

@ -221,9 +221,7 @@ export const DocumentNode = React.memo(function DocumentNode({
className={cn(
"absolute inset-0 flex items-center justify-center transition-opacity",
canMention &&
(isMentioned
? "opacity-0"
: "max-sm:opacity-0 group-hover/item:opacity-0")
(isMentioned ? "opacity-0" : "max-sm:opacity-0 group-hover/item:opacity-0")
)}
>
{getDocumentTypeIcon(

View file

@ -390,75 +390,75 @@ export const FolderNode = React.memo(function FolderNode({
</Button>
</DropdownMenuTrigger>
<DropdownMenuContent align="end" className="w-40">
{isWatched && onRescan && (
{isWatched && onRescan && (
<DropdownMenuItem
onClick={(e) => {
e.stopPropagation();
handleRescan();
}}
>
<RefreshCw className={cn("mr-2 h-4 w-4", isRescanning && "animate-spin")} />
Re-scan
</DropdownMenuItem>
)}
{isWatched && onStopWatching && (
<DropdownMenuItem
onClick={(e) => {
e.stopPropagation();
onStopWatching(folder);
}}
>
<EyeOff className="mr-2 h-4 w-4" />
Stop watching
</DropdownMenuItem>
)}
<DropdownMenuItem
onClick={(e) => {
e.stopPropagation();
handleRescan();
onCreateSubfolder(folder.id);
}}
>
<RefreshCw className={cn("mr-2 h-4 w-4", isRescanning && "animate-spin")} />
Re-scan
<FolderPlus className="mr-2 h-4 w-4" />
New subfolder
</DropdownMenuItem>
)}
{isWatched && onStopWatching && (
<DropdownMenuItem
onClick={(e) => {
e.stopPropagation();
onStopWatching(folder);
startRename();
}}
>
<EyeOff className="mr-2 h-4 w-4" />
Stop watching
<Pencil className="mr-2 h-4 w-4" />
Rename
</DropdownMenuItem>
)}
<DropdownMenuItem
onClick={(e) => {
e.stopPropagation();
onCreateSubfolder(folder.id);
}}
>
<FolderPlus className="mr-2 h-4 w-4" />
New subfolder
</DropdownMenuItem>
<DropdownMenuItem
onClick={(e) => {
e.stopPropagation();
startRename();
}}
>
<Pencil className="mr-2 h-4 w-4" />
Rename
</DropdownMenuItem>
<DropdownMenuItem
onClick={(e) => {
e.stopPropagation();
onMove(folder);
}}
>
<Move className="mr-2 h-4 w-4" />
Move to...
</DropdownMenuItem>
{onExportFolder && (
<DropdownMenuItem
onClick={(e) => {
e.stopPropagation();
onExportFolder(folder);
onMove(folder);
}}
>
<Download className="mr-2 h-4 w-4" />
Export folder
<Move className="mr-2 h-4 w-4" />
Move to...
</DropdownMenuItem>
{onExportFolder && (
<DropdownMenuItem
onClick={(e) => {
e.stopPropagation();
onExportFolder(folder);
}}
>
<Download className="mr-2 h-4 w-4" />
Export folder
</DropdownMenuItem>
)}
<DropdownMenuItem
onClick={(e) => {
e.stopPropagation();
onDelete(folder);
}}
>
<Trash2 className="mr-2 h-4 w-4" />
Delete
</DropdownMenuItem>
)}
<DropdownMenuItem
onClick={(e) => {
e.stopPropagation();
onDelete(folder);
}}
>
<Trash2 className="mr-2 h-4 w-4" />
Delete
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
</div>

View file

@ -466,8 +466,7 @@ const CATEGORIES: HeroCategory[] = [
{
id: "report",
title: "AI Report Generator",
description:
"Turn your research into cited reports, then export to PDF or Markdown.",
description: "Turn your research into cited reports, then export to PDF or Markdown.",
src: `${HERO_TUTORIAL}/ReportGenGif_compressed.mp4`,
},
{
@ -658,8 +657,8 @@ export function HeroSection() {
"relative mb-8 max-w-2xl text-left text-sm text-neutral-600 antialiased sm:text-base md:text-lg dark:text-neutral-400"
)}
>
SurfSense is an open-source open web research platform, like NotebookLM but with
live data connectors. Your AI agents research the live web with structured data from
SurfSense is an open-source open web research platform, like NotebookLM but with live
data connectors. Your AI agents research the live web with structured data from
Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, and any page on the
open web.
</p>

View file

@ -81,9 +81,7 @@ export function CreateWorkspaceDialog({ open, onOpenChange }: CreateWorkspaceDia
// which is the authoritative net regardless.
const isInitialSetup = result.llm_setup?.stage === "initial_setup";
router.push(
isInitialSetup
? `/dashboard/${result.id}/onboard`
: `/dashboard/${result.id}/new-chat`
isInitialSetup ? `/dashboard/${result.id}/onboard` : `/dashboard/${result.id}/new-chat`
);
} catch (error) {
console.error("Failed to create workspace:", error);

View file

@ -96,7 +96,7 @@ export function AgentSetupTabs({ options }: { options?: Partial<McpSnippetOption
const snippet = client[transport];
const config = snippet.build(resolved);
return (
<TabsContent key={client.id} value={client.id} className="min-w-0 space-y-3">
<TabsContent key={client.id} value={client.id} className="min-w-0 space-y-3">
<ol className="list-decimal space-y-1 pl-5 text-sm leading-relaxed text-muted-foreground">
{snippet.steps.map((step) => (
<li key={step}>{step}</li>

View file

@ -45,7 +45,8 @@ export function ConnectAgentDialog({ className }: { className?: string }) {
<DialogHeader>
<DialogTitle>Connect to Claude Code, Codex, OpenCode</DialogTitle>
<DialogDescription>
Give your coding agent access to SurfSense scrapers and your knowledge base. Create an API key under API Keys, choose your agent, then paste the config.
Give your coding agent access to SurfSense scrapers and your knowledge base. Create an
API key under API Keys, choose your agent, then paste the config.
</DialogDescription>
</DialogHeader>
<AgentSetupTabs options={{ baseUrl: BACKEND_URL || undefined }} />

View file

@ -43,10 +43,7 @@ class ModelConnectionsApiService {
};
getLlmSetupStatus = async (workspaceId: number): Promise<LlmSetupStatus> => {
return baseApiService.get(
`/api/v1/workspaces/${workspaceId}/llm-setup-status`,
llmSetupStatus
);
return baseApiService.get(`/api/v1/workspaces/${workspaceId}/llm-setup-status`, llmSetupStatus);
};
getModelProviders = async (): Promise<ModelProviderRead[]> => {

View file

@ -77,8 +77,7 @@ export const googleSearch: ConnectorPageContent = {
},
{
label: "Related searches",
description:
"Suggested and related queries, to map how people actually phrase the search.",
description: "Suggested and related queries, to map how people actually phrase the search.",
},
{
label: "SERP metadata",