mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-20 23:21:06 +02:00
chore: linting
This commit is contained in:
parent
be35eb8743
commit
c0ebb62fb2
20 changed files with 158 additions and 125 deletions
|
|
@ -29,8 +29,7 @@ import re
|
||||||
import time
|
import time
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from app.utils.captcha import CaptchaConfig
|
from app.utils.captcha import CaptchaConfig, solvers
|
||||||
from app.utils.captcha import solvers
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -42,8 +42,10 @@ from urllib.parse import urlsplit, urlunsplit
|
||||||
|
|
||||||
from scrapling.fetchers import AsyncFetcher
|
from scrapling.fetchers import AsyncFetcher
|
||||||
|
|
||||||
from app.proprietary.platforms.google_search import captcha as _captcha
|
from app.proprietary.platforms.google_search import (
|
||||||
from app.proprietary.platforms.google_search import pool_store as _store
|
captcha as _captcha,
|
||||||
|
pool_store as _store,
|
||||||
|
)
|
||||||
from app.utils.captcha import captcha_enabled, get_captcha_config
|
from app.utils.captcha import captcha_enabled, get_captcha_config
|
||||||
from app.utils.proxy import get_proxy_url
|
from app.utils.proxy import get_proxy_url
|
||||||
|
|
||||||
|
|
@ -374,9 +376,8 @@ def _make_page_action(proxy: str | None, cfg):
|
||||||
"""
|
"""
|
||||||
|
|
||||||
async def page_action(page):
|
async def page_action(page):
|
||||||
if _captcha.on_sorry(page):
|
if _captcha.on_sorry(page) and await _captcha.solve_sorry(page, proxy, cfg):
|
||||||
if await _captcha.solve_sorry(page, proxy, cfg):
|
_exemption_jar[proxy or ""] = await _captcha.exemption_cookies(page)
|
||||||
_exemption_jar[proxy or ""] = await _captcha.exemption_cookies(page)
|
|
||||||
return await _expand_blocks(page)
|
return await _expand_blocks(page)
|
||||||
|
|
||||||
return page_action
|
return page_action
|
||||||
|
|
|
||||||
|
|
@ -15,7 +15,7 @@ never block the request loop and don't care which loop the caller is on.
|
||||||
|
|
||||||
``ponytail:`` the shared store holds only exemptions (the costly artifact), not
|
``ponytail:`` the shared store holds only exemptions (the costly artifact), not
|
||||||
per-process render concurrency — global per-IP load is still governed by each
|
per-process render concurrency — global per-IP load is still governed by each
|
||||||
process's local per-IP cap × the number of processes, so size the pool for the
|
process's local per-IP cap x the number of processes, so size the pool for the
|
||||||
fleet (see ``GOOGLE_SEARCH_WARM_POOL_TARGET``). Full distributed inflight
|
fleet (see ``GOOGLE_SEARCH_WARM_POOL_TARGET``). Full distributed inflight
|
||||||
accounting is the upgrade path if a single shared IP ever gets overloaded.
|
accounting is the upgrade path if a single shared IP ever gets overloaded.
|
||||||
"""
|
"""
|
||||||
|
|
|
||||||
|
|
@ -299,11 +299,10 @@ def _is_unrecoverable(exc: Exception) -> bool:
|
||||||
"""True for solver errors that must latch solving off.
|
"""True for solver errors that must latch solving off.
|
||||||
|
|
||||||
Covers the in-house seam's typed errors (``SolverBalanceError`` /
|
Covers the in-house seam's typed errors (``SolverBalanceError`` /
|
||||||
``SolverAuthError`` / ``SolverUnsupported``) plus legacy/no-balance shapes.
|
``SolverAuthError`` / ``SolverUnsupportedError``) plus legacy/no-balance shapes.
|
||||||
Matched by class name so no solver module must be imported here.
|
Matched by class name so no solver module must be imported here.
|
||||||
"""
|
"""
|
||||||
name = type(exc).__name__.lower()
|
name = type(exc).__name__.lower()
|
||||||
return any(
|
return any(
|
||||||
k in name
|
k in name for k in ("balance", "apikey", "auth", "unsupported", "wronguser")
|
||||||
for k in ("balance", "apikey", "auth", "unsupported", "wronguser")
|
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -863,7 +863,7 @@ async def _workspace_has_enabled_chat_model(
|
||||||
.options(selectinload(Connection.models))
|
.options(selectinload(Connection.models))
|
||||||
.where(
|
.where(
|
||||||
Connection.workspace_id == workspace_id,
|
Connection.workspace_id == workspace_id,
|
||||||
Connection.enabled == True,
|
Connection.enabled,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
return any(
|
return any(
|
||||||
|
|
|
||||||
|
|
@ -10,7 +10,7 @@ the reCAPTCHA-*Enterprise* ``/sorry`` wall). Both express the Enterprise pieces
|
||||||
Google needs — the widget sitekey plus the page's dynamic ``data-s`` token
|
Google needs — the widget sitekey plus the page's dynamic ``data-s`` token
|
||||||
(something ``captchatools`` could not). More vendors (anticaptcha / capmonster)
|
(something ``captchatools`` could not). More vendors (anticaptcha / capmonster)
|
||||||
are added progressively as new entries in :data:`_PROVIDERS`; until then an
|
are added progressively as new entries in :data:`_PROVIDERS`; until then an
|
||||||
unconfigured provider raises :class:`SolverUnsupported` so callers latch off
|
unconfigured provider raises :class:`SolverUnsupportedError` so callers latch off
|
||||||
cleanly instead of leaking the API key to the wrong service.
|
cleanly instead of leaking the API key to the wrong service.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
@ -41,7 +41,7 @@ class SolverBalanceError(SolverError):
|
||||||
"""Solver account is out of balance — unrecoverable this process."""
|
"""Solver account is out of balance — unrecoverable this process."""
|
||||||
|
|
||||||
|
|
||||||
class SolverUnsupported(SolverError):
|
class SolverUnsupportedError(SolverError):
|
||||||
"""Configured provider has no in-house client yet — unrecoverable."""
|
"""Configured provider has no in-house client yet — unrecoverable."""
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -108,7 +108,11 @@ def _twocaptcha(
|
||||||
"min_score": cfg.v3_min_score,
|
"min_score": cfg.v3_min_score,
|
||||||
}
|
}
|
||||||
else: # v2, optionally the Enterprise variant (adds enterprise=1 + data-s)
|
else: # v2, optionally the Enterprise variant (adds enterprise=1 + data-s)
|
||||||
payload |= {"method": "userrecaptcha", "googlekey": sitekey, "pageurl": page_url}
|
payload |= {
|
||||||
|
"method": "userrecaptcha",
|
||||||
|
"googlekey": sitekey,
|
||||||
|
"pageurl": page_url,
|
||||||
|
}
|
||||||
if enterprise:
|
if enterprise:
|
||||||
payload["enterprise"] = 1
|
payload["enterprise"] = 1
|
||||||
if data_s:
|
if data_s:
|
||||||
|
|
@ -295,7 +299,7 @@ def solve(
|
||||||
"""
|
"""
|
||||||
client = _PROVIDERS.get((cfg.solving_site or "").lower())
|
client = _PROVIDERS.get((cfg.solving_site or "").lower())
|
||||||
if client is None:
|
if client is None:
|
||||||
raise SolverUnsupported(
|
raise SolverUnsupportedError(
|
||||||
f"captcha provider {cfg.solving_site!r} has no in-house client "
|
f"captcha provider {cfg.solving_site!r} has no in-house client "
|
||||||
f"(supported: {supported_providers()})"
|
f"(supported: {supported_providers()})"
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -26,7 +26,6 @@ from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import asyncio
|
import asyncio
|
||||||
import statistics
|
|
||||||
import sys
|
import sys
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
|
|
@ -181,7 +180,9 @@ async def main() -> None:
|
||||||
ap = argparse.ArgumentParser()
|
ap = argparse.ArgumentParser()
|
||||||
ap.add_argument("--count", type=int, default=400)
|
ap.add_argument("--count", type=int, default=400)
|
||||||
ap.add_argument("--rate", type=float, default=60.0, help="arrivals/sec (sim clock)")
|
ap.add_argument("--rate", type=float, default=60.0, help="arrivals/sec (sim clock)")
|
||||||
ap.add_argument("--warm", type=float, default=0.10, help="warm render seconds (sim)")
|
ap.add_argument(
|
||||||
|
"--warm", type=float, default=0.10, help="warm render seconds (sim)"
|
||||||
|
)
|
||||||
ap.add_argument("--solve", type=float, default=0.45, help="solve seconds (sim)")
|
ap.add_argument("--solve", type=float, default=0.45, help="solve seconds (sim)")
|
||||||
ap.add_argument("--precheck", type=float, default=0.01)
|
ap.add_argument("--precheck", type=float, default=0.01)
|
||||||
args = ap.parse_args()
|
args = ap.parse_args()
|
||||||
|
|
@ -206,23 +207,37 @@ async def main() -> None:
|
||||||
thru_real = thru_sim / scale
|
thru_real = thru_sim / scale
|
||||||
|
|
||||||
print("\n=== Google Search scale simulation ===")
|
print("\n=== Google Search scale simulation ===")
|
||||||
print(f" gate (_MAX_CONCURRENT_PAGES) = {gate}"
|
print(
|
||||||
+ (f", warm-pool target = {pool}" if pool else " (single-slot sticky IP)"))
|
f" gate (_MAX_CONCURRENT_PAGES) = {gate}"
|
||||||
|
+ (f", warm-pool target = {pool}" if pool else " (single-slot sticky IP)")
|
||||||
|
)
|
||||||
print(f" requests={args.count} arrival_rate={args.rate}/s (sim)")
|
print(f" requests={args.count} arrival_rate={args.rate}/s (sim)")
|
||||||
print(f" --- structure (scale-free) ---")
|
print(" --- structure (scale-free) ---")
|
||||||
print(f" paid solves = {metrics.solves} (want ~pool size, not ~requests)")
|
print(
|
||||||
|
f" paid solves = {metrics.solves} (want ~pool size, not ~requests)"
|
||||||
|
)
|
||||||
print(f" distinct sticky IPs = {len(metrics.ip_hits)}")
|
print(f" distinct sticky IPs = {len(metrics.ip_hits)}")
|
||||||
print(f" busiest IP carried = {top_hits}/{metrics.renders} renders "
|
print(
|
||||||
f"({top_share:.0f}%) (funneling; want ~even spread)")
|
f" busiest IP carried = {top_hits}/{metrics.renders} renders "
|
||||||
|
f"({top_share:.0f}%) (funneling; want ~even spread)"
|
||||||
|
)
|
||||||
print(f" peak renders on 1 IP = {hot_peak} (concurrency; capped by gate)")
|
print(f" peak renders on 1 IP = {hot_peak} (concurrency; capped by gate)")
|
||||||
print(f" --- throughput / latency (extrapolated to live @ warm={_REAL_WARM_S}s) ---")
|
print(
|
||||||
print(f" throughput = {thru_real*60:.0f} SERP/min ({thru_real:.2f}/s)")
|
f" --- throughput / latency (extrapolated to live @ warm={_REAL_WARM_S}s) ---"
|
||||||
print(f" latency p50 = {_pct(lat,50)*scale:6.1f}s p95 = {_pct(lat,95)*scale:6.1f}s "
|
)
|
||||||
f"p99 = {_pct(lat,99)*scale:6.1f}s")
|
print(f" throughput = {thru_real * 60:.0f} SERP/min ({thru_real:.2f}/s)")
|
||||||
print(f" ceiling (gate/warm) = {gate/_REAL_WARM_S*60:.0f} SERP/min per process")
|
print(
|
||||||
|
f" latency p50 = {_pct(lat, 50) * scale:6.1f}s p95 = {_pct(lat, 95) * scale:6.1f}s "
|
||||||
|
f"p99 = {_pct(lat, 99) * scale:6.1f}s"
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
f" ceiling (gate/warm) = {gate / _REAL_WARM_S * 60:.0f} SERP/min per process"
|
||||||
|
)
|
||||||
need = 500 / (gate / _REAL_WARM_S * 60)
|
need = 500 / (gate / _REAL_WARM_S * 60)
|
||||||
print(f" -> to sustain 500 SERP/min you need ~{need:.0f} such processes "
|
print(
|
||||||
f"(or a larger gate)\n")
|
f" -> to sustain 500 SERP/min you need ~{need:.0f} such processes "
|
||||||
|
f"(or a larger gate)\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|
|
||||||
|
|
@ -51,9 +51,7 @@ _INFO = [
|
||||||
class _LogTally(logging.Handler):
|
class _LogTally(logging.Handler):
|
||||||
"""Counts solves / renders / walls / pool-reuse from the live log stream."""
|
"""Counts solves / renders / walls / pool-reuse from the live log stream."""
|
||||||
|
|
||||||
_RENDER = re.compile(
|
_RENDER = re.compile(r"has_results=(\w+).*from_pool=(\w+) pool=(\d+)")
|
||||||
r"has_results=(\w+).*from_pool=(\w+) pool=(\d+)"
|
|
||||||
)
|
|
||||||
_SOLVE = re.compile(r"\[captcha\] solve (OK|did not)")
|
_SOLVE = re.compile(r"\[captcha\] solve (OK|did not)")
|
||||||
|
|
||||||
def __init__(self) -> None:
|
def __init__(self) -> None:
|
||||||
|
|
@ -147,7 +145,7 @@ async def main() -> None:
|
||||||
lg.setLevel(logging.INFO)
|
lg.setLevel(logging.INFO)
|
||||||
lg.addHandler(tally)
|
lg.addHandler(tally)
|
||||||
|
|
||||||
from app.proprietary.platforms.google_search.fetch import ( # noqa: E402
|
from app.proprietary.platforms.google_search.fetch import (
|
||||||
_WARM_POOL_TARGET,
|
_WARM_POOL_TARGET,
|
||||||
close_sessions,
|
close_sessions,
|
||||||
)
|
)
|
||||||
|
|
@ -157,9 +155,11 @@ async def main() -> None:
|
||||||
total_serps = sum(w for _, w in plans)
|
total_serps = sum(w for _, w in plans)
|
||||||
n_multi = sum(1 for _, w in plans if w > 1)
|
n_multi = sum(1 for _, w in plans if w > 1)
|
||||||
|
|
||||||
print(f"\n=== LIVE stress: {args.users} users "
|
print(
|
||||||
f"({args.users - n_multi} single / {n_multi} multi), "
|
f"\n=== LIVE stress: {args.users} users "
|
||||||
f"~{total_serps} SERPs, gate={args.gate}, pool_target={_WARM_POOL_TARGET} ===")
|
f"({args.users - n_multi} single / {n_multi} multi), "
|
||||||
|
f"~{total_serps} SERPs, gate={args.gate}, pool_target={_WARM_POOL_TARGET} ==="
|
||||||
|
)
|
||||||
print(" (real solves + proxy spend; warming up...)\n")
|
print(" (real solves + proxy spend; warming up...)\n")
|
||||||
|
|
||||||
results: list[dict] = []
|
results: list[dict] = []
|
||||||
|
|
@ -184,17 +184,25 @@ async def main() -> None:
|
||||||
print("=== results ===")
|
print("=== results ===")
|
||||||
print(f" wall time = {wall:.0f}s")
|
print(f" wall time = {wall:.0f}s")
|
||||||
print(f" requests ok/failed = {len(results) - len(fails)}/{len(fails)}")
|
print(f" requests ok/failed = {len(results) - len(fails)}/{len(fails)}")
|
||||||
print(f" SERPs got/expected = {got_serps}/{total_serps}"
|
print(
|
||||||
+ (f" ({len(short)} short)" if short else ""))
|
f" SERPs got/expected = {got_serps}/{total_serps}"
|
||||||
|
+ (f" ({len(short)} short)" if short else "")
|
||||||
|
)
|
||||||
print(f" throughput = {got_serps / wall * 60:.0f} SERP/min")
|
print(f" throughput = {got_serps / wall * 60:.0f} SERP/min")
|
||||||
print(f" request latency p50={pct(50):.0f}s p95={pct(95):.0f}s "
|
print(
|
||||||
f"max={lat[-1] if lat else 0:.0f}s")
|
f" request latency p50={pct(50):.0f}s p95={pct(95):.0f}s "
|
||||||
|
f"max={lat[-1] if lat else 0:.0f}s"
|
||||||
|
)
|
||||||
print(" --- pipeline (from live logs) ---")
|
print(" --- pipeline (from live logs) ---")
|
||||||
print(f" paid solves ok/fail = {tally.solves_ok}/{tally.solves_fail} "
|
print(
|
||||||
f"(bounded by pool target {_WARM_POOL_TARGET})")
|
f" paid solves ok/fail = {tally.solves_ok}/{tally.solves_fail} "
|
||||||
|
f"(bounded by pool target {_WARM_POOL_TARGET})"
|
||||||
|
)
|
||||||
print(f" renders ok/walled = {tally.renders_ok}/{tally.renders_walled}")
|
print(f" renders ok/walled = {tally.renders_ok}/{tally.renders_walled}")
|
||||||
print(f" pool reuse/grow = {tally.reuse}/{tally.grow} "
|
print(
|
||||||
f"(reuse share {100 * tally.reuse / max(1, tally.reuse + tally.grow):.0f}%)")
|
f" pool reuse/grow = {tally.reuse}/{tally.grow} "
|
||||||
|
f"(reuse share {100 * tally.reuse / max(1, tally.reuse + tally.grow):.0f}%)"
|
||||||
|
)
|
||||||
print(f" peak pool size = {tally.max_pool}")
|
print(f" peak pool size = {tally.max_pool}")
|
||||||
if fails:
|
if fails:
|
||||||
print(" --- failures ---")
|
print(" --- failures ---")
|
||||||
|
|
|
||||||
|
|
@ -58,7 +58,9 @@ class TestGlobalCatalogHasUsableChat:
|
||||||
"""Usability, not file existence, is what counts."""
|
"""Usability, not file existence, is what counts."""
|
||||||
|
|
||||||
def test_usable_when_enabled_connection_and_chat_model(self, monkeypatch):
|
def test_usable_when_enabled_connection_and_chat_model(self, monkeypatch):
|
||||||
monkeypatch.setattr(mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": True}])
|
monkeypatch.setattr(
|
||||||
|
mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": True}]
|
||||||
|
)
|
||||||
monkeypatch.setattr(mc.config, "GLOBAL_MODELS", [_global_model()])
|
monkeypatch.setattr(mc.config, "GLOBAL_MODELS", [_global_model()])
|
||||||
assert mc._global_catalog_has_usable_chat() is True
|
assert mc._global_catalog_has_usable_chat() is True
|
||||||
|
|
||||||
|
|
@ -68,17 +70,23 @@ class TestGlobalCatalogHasUsableChat:
|
||||||
assert mc._global_catalog_has_usable_chat() is False
|
assert mc._global_catalog_has_usable_chat() is False
|
||||||
|
|
||||||
def test_disabled_connection_is_not_usable(self, monkeypatch):
|
def test_disabled_connection_is_not_usable(self, monkeypatch):
|
||||||
monkeypatch.setattr(mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": False}])
|
monkeypatch.setattr(
|
||||||
|
mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": False}]
|
||||||
|
)
|
||||||
monkeypatch.setattr(mc.config, "GLOBAL_MODELS", [_global_model()])
|
monkeypatch.setattr(mc.config, "GLOBAL_MODELS", [_global_model()])
|
||||||
assert mc._global_catalog_has_usable_chat() is False
|
assert mc._global_catalog_has_usable_chat() is False
|
||||||
|
|
||||||
def test_disabled_model_is_not_usable(self, monkeypatch):
|
def test_disabled_model_is_not_usable(self, monkeypatch):
|
||||||
monkeypatch.setattr(mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": True}])
|
monkeypatch.setattr(
|
||||||
|
mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": True}]
|
||||||
|
)
|
||||||
monkeypatch.setattr(mc.config, "GLOBAL_MODELS", [_global_model(enabled=False)])
|
monkeypatch.setattr(mc.config, "GLOBAL_MODELS", [_global_model(enabled=False)])
|
||||||
assert mc._global_catalog_has_usable_chat() is False
|
assert mc._global_catalog_has_usable_chat() is False
|
||||||
|
|
||||||
def test_non_chat_model_is_not_usable(self, monkeypatch):
|
def test_non_chat_model_is_not_usable(self, monkeypatch):
|
||||||
monkeypatch.setattr(mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": True}])
|
monkeypatch.setattr(
|
||||||
|
mc.config, "GLOBAL_CONNECTIONS", [{"id": -1, "enabled": True}]
|
||||||
|
)
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
mc.config, "GLOBAL_MODELS", [_global_model(supports_chat=False)]
|
mc.config, "GLOBAL_MODELS", [_global_model(supports_chat=False)]
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -37,7 +37,10 @@ def test_proxy_login_form_strips_scheme():
|
||||||
|
|
||||||
|
|
||||||
def test_proxy_login_form_without_credentials():
|
def test_proxy_login_form_without_credentials():
|
||||||
assert solvers.proxy_login_form("http://gw.dataimpulse.com:823") == "gw.dataimpulse.com:823"
|
assert (
|
||||||
|
solvers.proxy_login_form("http://gw.dataimpulse.com:823")
|
||||||
|
== "gw.dataimpulse.com:823"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_proxy_login_form_none_on_missing_or_bad():
|
def test_proxy_login_form_none_on_missing_or_bad():
|
||||||
|
|
@ -54,7 +57,10 @@ def test_capsolver_proxy_colon_delimited_with_creds():
|
||||||
|
|
||||||
|
|
||||||
def test_capsolver_proxy_without_credentials():
|
def test_capsolver_proxy_without_credentials():
|
||||||
assert solvers.capsolver_proxy("http://gw.dataimpulse.com:823") == "http:gw.dataimpulse.com:823"
|
assert (
|
||||||
|
solvers.capsolver_proxy("http://gw.dataimpulse.com:823")
|
||||||
|
== "http:gw.dataimpulse.com:823"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_capsolver_proxy_none_on_missing_or_bad():
|
def test_capsolver_proxy_none_on_missing_or_bad():
|
||||||
|
|
@ -68,14 +74,14 @@ def test_capsolver_proxy_none_on_missing_or_bad():
|
||||||
def test_unsupported_provider_raises_solvererror():
|
def test_unsupported_provider_raises_solvererror():
|
||||||
# An unconfigured provider must fail loudly (latch) rather than POST the key
|
# An unconfigured provider must fail loudly (latch) rather than POST the key
|
||||||
# to a wired vendor's endpoint under the wrong account.
|
# to a wired vendor's endpoint under the wrong account.
|
||||||
with pytest.raises(solvers.SolverUnsupported):
|
with pytest.raises(solvers.SolverUnsupportedError):
|
||||||
solvers.solve(
|
solvers.solve(
|
||||||
_cfg(solving_site="anticaptcha"),
|
_cfg(solving_site="anticaptcha"),
|
||||||
challenge_type="v2",
|
challenge_type="v2",
|
||||||
sitekey="SK",
|
sitekey="SK",
|
||||||
page_url="https://t.test",
|
page_url="https://t.test",
|
||||||
)
|
)
|
||||||
assert issubclass(solvers.SolverUnsupported, solvers.SolverError)
|
assert issubclass(solvers.SolverUnsupportedError, solvers.SolverError)
|
||||||
|
|
||||||
|
|
||||||
def test_wired_providers_are_registered():
|
def test_wired_providers_are_registered():
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
// Server component
|
// Server component
|
||||||
import type React from "react";
|
|
||||||
import { cookies } from "next/headers";
|
import { cookies } from "next/headers";
|
||||||
|
import type React from "react";
|
||||||
import { DashboardClientLayout } from "./client-layout";
|
import { DashboardClientLayout } from "./client-layout";
|
||||||
|
|
||||||
const PLAYGROUND_SIDEBAR_COLLAPSED_COOKIE = "surfsense_playground_sidebar_collapsed";
|
const PLAYGROUND_SIDEBAR_COLLAPSED_COOKIE = "surfsense_playground_sidebar_collapsed";
|
||||||
|
|
|
||||||
|
|
@ -34,7 +34,7 @@ export const ConnectorDialogHeader: FC<ConnectorDialogHeaderProps> = ({
|
||||||
MCP Connectors
|
MCP Connectors
|
||||||
</DialogTitle>
|
</DialogTitle>
|
||||||
<DialogDescription className="text-xs sm:text-base text-muted-foreground/80 mt-1 sm:mt-1.5">
|
<DialogDescription className="text-xs sm:text-base text-muted-foreground/80 mt-1 sm:mt-1.5">
|
||||||
Connect external tools and services through MCP.
|
Connect external tools and services through MCP.
|
||||||
</DialogDescription>
|
</DialogDescription>
|
||||||
</DialogHeader>
|
</DialogHeader>
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -221,9 +221,7 @@ export const DocumentNode = React.memo(function DocumentNode({
|
||||||
className={cn(
|
className={cn(
|
||||||
"absolute inset-0 flex items-center justify-center transition-opacity",
|
"absolute inset-0 flex items-center justify-center transition-opacity",
|
||||||
canMention &&
|
canMention &&
|
||||||
(isMentioned
|
(isMentioned ? "opacity-0" : "max-sm:opacity-0 group-hover/item:opacity-0")
|
||||||
? "opacity-0"
|
|
||||||
: "max-sm:opacity-0 group-hover/item:opacity-0")
|
|
||||||
)}
|
)}
|
||||||
>
|
>
|
||||||
{getDocumentTypeIcon(
|
{getDocumentTypeIcon(
|
||||||
|
|
|
||||||
|
|
@ -390,75 +390,75 @@ export const FolderNode = React.memo(function FolderNode({
|
||||||
</Button>
|
</Button>
|
||||||
</DropdownMenuTrigger>
|
</DropdownMenuTrigger>
|
||||||
<DropdownMenuContent align="end" className="w-40">
|
<DropdownMenuContent align="end" className="w-40">
|
||||||
{isWatched && onRescan && (
|
{isWatched && onRescan && (
|
||||||
|
<DropdownMenuItem
|
||||||
|
onClick={(e) => {
|
||||||
|
e.stopPropagation();
|
||||||
|
handleRescan();
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<RefreshCw className={cn("mr-2 h-4 w-4", isRescanning && "animate-spin")} />
|
||||||
|
Re-scan
|
||||||
|
</DropdownMenuItem>
|
||||||
|
)}
|
||||||
|
{isWatched && onStopWatching && (
|
||||||
|
<DropdownMenuItem
|
||||||
|
onClick={(e) => {
|
||||||
|
e.stopPropagation();
|
||||||
|
onStopWatching(folder);
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<EyeOff className="mr-2 h-4 w-4" />
|
||||||
|
Stop watching
|
||||||
|
</DropdownMenuItem>
|
||||||
|
)}
|
||||||
<DropdownMenuItem
|
<DropdownMenuItem
|
||||||
onClick={(e) => {
|
onClick={(e) => {
|
||||||
e.stopPropagation();
|
e.stopPropagation();
|
||||||
handleRescan();
|
onCreateSubfolder(folder.id);
|
||||||
}}
|
}}
|
||||||
>
|
>
|
||||||
<RefreshCw className={cn("mr-2 h-4 w-4", isRescanning && "animate-spin")} />
|
<FolderPlus className="mr-2 h-4 w-4" />
|
||||||
Re-scan
|
New subfolder
|
||||||
</DropdownMenuItem>
|
</DropdownMenuItem>
|
||||||
)}
|
|
||||||
{isWatched && onStopWatching && (
|
|
||||||
<DropdownMenuItem
|
<DropdownMenuItem
|
||||||
onClick={(e) => {
|
onClick={(e) => {
|
||||||
e.stopPropagation();
|
e.stopPropagation();
|
||||||
onStopWatching(folder);
|
startRename();
|
||||||
}}
|
}}
|
||||||
>
|
>
|
||||||
<EyeOff className="mr-2 h-4 w-4" />
|
<Pencil className="mr-2 h-4 w-4" />
|
||||||
Stop watching
|
Rename
|
||||||
</DropdownMenuItem>
|
</DropdownMenuItem>
|
||||||
)}
|
|
||||||
<DropdownMenuItem
|
|
||||||
onClick={(e) => {
|
|
||||||
e.stopPropagation();
|
|
||||||
onCreateSubfolder(folder.id);
|
|
||||||
}}
|
|
||||||
>
|
|
||||||
<FolderPlus className="mr-2 h-4 w-4" />
|
|
||||||
New subfolder
|
|
||||||
</DropdownMenuItem>
|
|
||||||
<DropdownMenuItem
|
|
||||||
onClick={(e) => {
|
|
||||||
e.stopPropagation();
|
|
||||||
startRename();
|
|
||||||
}}
|
|
||||||
>
|
|
||||||
<Pencil className="mr-2 h-4 w-4" />
|
|
||||||
Rename
|
|
||||||
</DropdownMenuItem>
|
|
||||||
<DropdownMenuItem
|
|
||||||
onClick={(e) => {
|
|
||||||
e.stopPropagation();
|
|
||||||
onMove(folder);
|
|
||||||
}}
|
|
||||||
>
|
|
||||||
<Move className="mr-2 h-4 w-4" />
|
|
||||||
Move to...
|
|
||||||
</DropdownMenuItem>
|
|
||||||
{onExportFolder && (
|
|
||||||
<DropdownMenuItem
|
<DropdownMenuItem
|
||||||
onClick={(e) => {
|
onClick={(e) => {
|
||||||
e.stopPropagation();
|
e.stopPropagation();
|
||||||
onExportFolder(folder);
|
onMove(folder);
|
||||||
}}
|
}}
|
||||||
>
|
>
|
||||||
<Download className="mr-2 h-4 w-4" />
|
<Move className="mr-2 h-4 w-4" />
|
||||||
Export folder
|
Move to...
|
||||||
|
</DropdownMenuItem>
|
||||||
|
{onExportFolder && (
|
||||||
|
<DropdownMenuItem
|
||||||
|
onClick={(e) => {
|
||||||
|
e.stopPropagation();
|
||||||
|
onExportFolder(folder);
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<Download className="mr-2 h-4 w-4" />
|
||||||
|
Export folder
|
||||||
|
</DropdownMenuItem>
|
||||||
|
)}
|
||||||
|
<DropdownMenuItem
|
||||||
|
onClick={(e) => {
|
||||||
|
e.stopPropagation();
|
||||||
|
onDelete(folder);
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<Trash2 className="mr-2 h-4 w-4" />
|
||||||
|
Delete
|
||||||
</DropdownMenuItem>
|
</DropdownMenuItem>
|
||||||
)}
|
|
||||||
<DropdownMenuItem
|
|
||||||
onClick={(e) => {
|
|
||||||
e.stopPropagation();
|
|
||||||
onDelete(folder);
|
|
||||||
}}
|
|
||||||
>
|
|
||||||
<Trash2 className="mr-2 h-4 w-4" />
|
|
||||||
Delete
|
|
||||||
</DropdownMenuItem>
|
|
||||||
</DropdownMenuContent>
|
</DropdownMenuContent>
|
||||||
</DropdownMenu>
|
</DropdownMenu>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
||||||
|
|
@ -466,8 +466,7 @@ const CATEGORIES: HeroCategory[] = [
|
||||||
{
|
{
|
||||||
id: "report",
|
id: "report",
|
||||||
title: "AI Report Generator",
|
title: "AI Report Generator",
|
||||||
description:
|
description: "Turn your research into cited reports, then export to PDF or Markdown.",
|
||||||
"Turn your research into cited reports, then export to PDF or Markdown.",
|
|
||||||
src: `${HERO_TUTORIAL}/ReportGenGif_compressed.mp4`,
|
src: `${HERO_TUTORIAL}/ReportGenGif_compressed.mp4`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|
@ -658,8 +657,8 @@ export function HeroSection() {
|
||||||
"relative mb-8 max-w-2xl text-left text-sm text-neutral-600 antialiased sm:text-base md:text-lg dark:text-neutral-400"
|
"relative mb-8 max-w-2xl text-left text-sm text-neutral-600 antialiased sm:text-base md:text-lg dark:text-neutral-400"
|
||||||
)}
|
)}
|
||||||
>
|
>
|
||||||
SurfSense is an open-source open web research platform, like NotebookLM but with
|
SurfSense is an open-source open web research platform, like NotebookLM but with live
|
||||||
live data connectors. Your AI agents research the live web with structured data from
|
data connectors. Your AI agents research the live web with structured data from
|
||||||
Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, and any page on the
|
Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, and any page on the
|
||||||
open web.
|
open web.
|
||||||
</p>
|
</p>
|
||||||
|
|
|
||||||
|
|
@ -81,9 +81,7 @@ export function CreateWorkspaceDialog({ open, onOpenChange }: CreateWorkspaceDia
|
||||||
// which is the authoritative net regardless.
|
// which is the authoritative net regardless.
|
||||||
const isInitialSetup = result.llm_setup?.stage === "initial_setup";
|
const isInitialSetup = result.llm_setup?.stage === "initial_setup";
|
||||||
router.push(
|
router.push(
|
||||||
isInitialSetup
|
isInitialSetup ? `/dashboard/${result.id}/onboard` : `/dashboard/${result.id}/new-chat`
|
||||||
? `/dashboard/${result.id}/onboard`
|
|
||||||
: `/dashboard/${result.id}/new-chat`
|
|
||||||
);
|
);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error("Failed to create workspace:", error);
|
console.error("Failed to create workspace:", error);
|
||||||
|
|
|
||||||
|
|
@ -96,7 +96,7 @@ export function AgentSetupTabs({ options }: { options?: Partial<McpSnippetOption
|
||||||
const snippet = client[transport];
|
const snippet = client[transport];
|
||||||
const config = snippet.build(resolved);
|
const config = snippet.build(resolved);
|
||||||
return (
|
return (
|
||||||
<TabsContent key={client.id} value={client.id} className="min-w-0 space-y-3">
|
<TabsContent key={client.id} value={client.id} className="min-w-0 space-y-3">
|
||||||
<ol className="list-decimal space-y-1 pl-5 text-sm leading-relaxed text-muted-foreground">
|
<ol className="list-decimal space-y-1 pl-5 text-sm leading-relaxed text-muted-foreground">
|
||||||
{snippet.steps.map((step) => (
|
{snippet.steps.map((step) => (
|
||||||
<li key={step}>{step}</li>
|
<li key={step}>{step}</li>
|
||||||
|
|
|
||||||
|
|
@ -45,7 +45,8 @@ export function ConnectAgentDialog({ className }: { className?: string }) {
|
||||||
<DialogHeader>
|
<DialogHeader>
|
||||||
<DialogTitle>Connect to Claude Code, Codex, OpenCode…</DialogTitle>
|
<DialogTitle>Connect to Claude Code, Codex, OpenCode…</DialogTitle>
|
||||||
<DialogDescription>
|
<DialogDescription>
|
||||||
Give your coding agent access to SurfSense scrapers and your knowledge base. Create an API key under API Keys, choose your agent, then paste the config.
|
Give your coding agent access to SurfSense scrapers and your knowledge base. Create an
|
||||||
|
API key under API Keys, choose your agent, then paste the config.
|
||||||
</DialogDescription>
|
</DialogDescription>
|
||||||
</DialogHeader>
|
</DialogHeader>
|
||||||
<AgentSetupTabs options={{ baseUrl: BACKEND_URL || undefined }} />
|
<AgentSetupTabs options={{ baseUrl: BACKEND_URL || undefined }} />
|
||||||
|
|
|
||||||
|
|
@ -43,10 +43,7 @@ class ModelConnectionsApiService {
|
||||||
};
|
};
|
||||||
|
|
||||||
getLlmSetupStatus = async (workspaceId: number): Promise<LlmSetupStatus> => {
|
getLlmSetupStatus = async (workspaceId: number): Promise<LlmSetupStatus> => {
|
||||||
return baseApiService.get(
|
return baseApiService.get(`/api/v1/workspaces/${workspaceId}/llm-setup-status`, llmSetupStatus);
|
||||||
`/api/v1/workspaces/${workspaceId}/llm-setup-status`,
|
|
||||||
llmSetupStatus
|
|
||||||
);
|
|
||||||
};
|
};
|
||||||
|
|
||||||
getModelProviders = async (): Promise<ModelProviderRead[]> => {
|
getModelProviders = async (): Promise<ModelProviderRead[]> => {
|
||||||
|
|
|
||||||
|
|
@ -77,8 +77,7 @@ export const googleSearch: ConnectorPageContent = {
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
label: "Related searches",
|
label: "Related searches",
|
||||||
description:
|
description: "Suggested and related queries, to map how people actually phrase the search.",
|
||||||
"Suggested and related queries, to map how people actually phrase the search.",
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
label: "SERP metadata",
|
label: "SERP metadata",
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue