diff --git a/VERSION b/VERSION index cd9d21e61..bb951c884 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.33 +0.0.34 diff --git a/surfsense_backend/app/proprietary/platforms/reddit/fetch.py b/surfsense_backend/app/proprietary/platforms/reddit/fetch.py index 6922ce167..50fad389d 100644 --- a/surfsense_backend/app/proprietary/platforms/reddit/fetch.py +++ b/surfsense_backend/app/proprietary/platforms/reddit/fetch.py @@ -1,25 +1,30 @@ -"""Proxy-aware fetch seam for the Reddit scraper (no browser). +"""Proxy-aware fetch seam for the Reddit scraper (browser-warm, HTTP-fetch). All network I/O flows through :func:`fetch_json` and always egresses through the residential proxy (a direct hit would expose and risk-block the server IP). -Reddit deprecated *cold* unauthenticated ``.json`` (a bare anonymous GET now -403s). The maintained anonymous recipe (proven live 2026-07-04, see -``scripts/e2e_reddit_scraper.py`` step 0) is: +Reddit shut anonymous ``.json`` (~May 2026) and, crucially, put a JavaScript +challenge in front of minting the anonymous ``loid`` session cookie: a plain +HTTP GET (even Chrome-impersonated) can no longer mint ``loid`` — it just 403s +on the bot wall (confirmed live 2026-07-18, and yt-dlp #16877). So the recipe is +now two-phase on ONE sticky exit IP: - warm one anonymous session cookie (``loid``) with a plain GET to - ``www.reddit.com/svc/shreddit/`` (``old.reddit.com`` fallback), then - GET ``www.reddit.com//.json?raw_json=1`` through that same - Chrome-impersonated, sticky-IP session. Which warm URL mints ``loid`` is - exit-IP dependent, so the order is a tiebreak — see :func:`warm_session`. + 1. WARM (once per session): open a real patchright-Chromium stealth browser + (:func:`warm_session`) on a sticky proxy IP, load a public HTML page so + the JS challenge runs and mints the anonymous cookie jar (incl. ``loid``). + 2. FETCH (many, fast): replay that minted jar through a plain-HTTP, + Chrome-impersonated ``FetcherSession`` pinned to the SAME sticky IP, + GETting ``www.reddit.com//.json?raw_json=1``. The ``.json`` body + comes back as raw JSON (no browser HTML wrapper) because this phase is + plain HTTP, keeping the fast/cheap fan-out — browser cost is paid ONCE + per warm, not per fetch. -``loid`` is Reddit's equivalent of Google Maps' ``NID`` session cookie: an -anonymous, logged-out id that unlocks the public API — no account, no browser. -This module is a direct port of ``../youtube/innertube.py``'s rotate-on-block -sticky-session pattern (``_RotatingSession`` + ``_current_session`` ContextVar + -``open_proxy_holder``/``bind_proxy_holder``/``proxy_session``), with a -Reddit-specific :func:`warm_session` bolted on and a ``.json``-shaped -:func:`fetch_json` instead of an InnerTube POST. +``loid`` binds to the exit IP, so both phases share one sticky proxy session id +(see :meth:`_RotatingSession._open`); a 403/blocked IP rotates to a fresh sticky +id + country and re-warms. This module is a port of ``../youtube/innertube.py``'s +rotate-on-block sticky-session pattern (``_RotatingSession`` + ``_current_session`` +ContextVar + ``open_proxy_holder``/``bind_proxy_holder``/``proxy_session``), with +the browser warm bolted on and a ``.json``-shaped :func:`fetch_json`. """ from __future__ import annotations @@ -29,15 +34,16 @@ import json import logging import random import time +import uuid from contextlib import asynccontextmanager, suppress from contextvars import ContextVar from datetime import UTC, datetime from typing import Any from urllib.parse import urlencode -from scrapling.fetchers import AsyncFetcher, FetcherSession +from scrapling.fetchers import AsyncFetcher, AsyncStealthySession, FetcherSession -from app.utils.proxy import get_geo_proxy_url, get_proxy_url +from app.utils.proxy import get_proxy_url, get_sticky_proxy_url # Shared cross-country rotation walk (also used by the TikTok sibling). Kept under # the historical private names this module and its tests reference. @@ -111,29 +117,49 @@ _HEADERS = {"Accept-Language": "en-US,en;q=0.9"} # (the caller filters on ``includeNSFW`` downstream). Mirrors the probe. _OVER18_COOKIES = {"over18": "1"} -# ``svc/shreddit`` needs *a* path to render; any always-public subreddit mints -# ``loid`` just the same. ``r/popular`` always exists, so it's a safe default -# regardless of which target this session ends up serving. -_WARM_SLUG = "r/popular" -_SHREDDIT_URL = ( - "https://www.reddit.com/svc/shreddit/{slug}" - "?render-mode=partial&seeker-session=false" -) -_OLD_REDDIT_URL = "https://old.reddit.com/" +# The browser warm loads *a* public HTML page so Reddit's JS challenge runs and +# mints the anonymous jar; any always-public subreddit works. ``r/popular`` +# always exists, so it's a safe default regardless of this session's target. +_WARM_HTML_URL = "https://www.reddit.com/r/popular/" _LOID_COOKIE = "loid" +# The stealth browser warm (cold start + the page's own ``load``) lands in +# ~6-15s once the solve_cloudflare/network_idle dead-waits are dropped (see +# warm_session), so 30s is a generous ceiling that still bounds a dead exit; the +# fast HTTP ``.json`` fetches keep the tight _REQUEST_TIMEOUT_S above. +_WARM_TIMEOUT_MS = 30_000 + +# Bound concurrent browser warms so a wide fan-out (up to _FANOUT_CONCURRENCY=16 +# workers, each warming once) can't spawn 16 Chromiums at once and OOM the box. +# Warms are staggered, not serialized: a freed slot starts the next worker's warm +# while already-warmed workers fetch over plain HTTP. +# ponytail: 4 is tuned for a ~2GB container; raise on a bigger box, lower if warm +# spikes push memory. Ceiling: a burst of >4 cold workers queues behind this. +_WARM_CONCURRENCY = 4 +_warm_slots = asyncio.Semaphore(_WARM_CONCURRENCY) + def now_iso() -> str: """UTC timestamp in the millisecond ISO shape used by scraper output.""" return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" -def _response_cookie_names(page: Any) -> set[str]: - """Cookie names set by a response (best-effort across scrapling shapes).""" +def _browser_cookie_jar(page: Any) -> dict[str, str]: + """Extract a ``{name: value}`` jar from a browser Response (best-effort). + + Scrapling's browser Response exposes cookies as a list of ``{name, value, + ...}`` dicts (patchright/playwright shape); tolerate a plain ``{name: value}`` + dict too so a shape change or a fake in tests still works. + """ cookies = getattr(page, "cookies", None) if isinstance(cookies, dict): - return set(cookies.keys()) - return set() + return {str(k): str(v) for k, v in cookies.items()} + jar: dict[str, str] = {} + if isinstance(cookies, (list, tuple)): + for item in cookies: + if isinstance(item, dict) and "name" in item and "value" in item: + jar[str(item["name"])] = str(item["value"]) + return jar def _parse_json(page: Any) -> Any | None: @@ -185,23 +211,44 @@ class _RotatingSession: self.rotations = 0 self.warmed = False self.country = "" + # Sticky proxy URL shared by the browser warm AND the HTTP fetch session + # so both egress the SAME exit IP (``loid`` binds to that IP). The minted + # cookie jar is replayed on every ``.json`` fetch. + self.proxy: str | None = None + self.cookies: dict[str, str] = {} self._last_at = 0.0 async def _open(self) -> None: self.warmed = False + self.cookies = {} self.country = _country_for_rotation(self.rotations) - proxy = get_geo_proxy_url(self.country) - if proxy is None: + # A fresh vendor session id per (re)open pins a distinct sticky exit IP, + # so rotating drops the dirty IP AND its dead cookie jar in one step. + session_id = uuid.uuid4().hex[:12] + self.proxy = get_sticky_proxy_url(session_id, self.country) + if self.proxy is None: self._cm = self.session = None return self._cm = FetcherSession( - proxy=proxy, + proxy=self.proxy, stealthy_headers=True, impersonate="chrome", timeout=_REQUEST_TIMEOUT_S, ) self.session = await self._cm.__aenter__() + async def warm(self) -> bool: + """Browser-mint the anonymous cookie jar on this session's sticky IP. + + Returns ``True`` and stores the jar (:attr:`cookies`) when ``loid`` was + minted, else ``False`` (caller rotates the IP and retries). + """ + jar = await warm_session(self.proxy) + if jar: + self.cookies = jar + return True + return False + async def close(self) -> None: if self._cm is not None: with suppress(Exception): # best-effort teardown @@ -260,49 +307,58 @@ async def proxy_session(): await holder.close() -async def warm_session(session: Any, *, slug: str = _WARM_SLUG) -> bool: - """Mint an anonymous ``loid`` cookie on a freshly opened session. +async def warm_session(proxy: str | None) -> dict[str, str] | None: + """Browser-mint the anonymous cookie jar (incl. ``loid``) on a sticky IP. - Returns ``True`` when a ``loid`` was issued (the session can now reach - ``.json``), else ``False`` (caller rotates the IP and retries). + Reddit gates ``loid`` minting behind a JS challenge that a plain-HTTP client + can't clear, so this spins up patchright-Chromium ONCE per session on the + given sticky proxy IP, loads a public HTML page so the challenge runs, and + returns the minted ``{name: value}`` cookie jar. Returns ``None`` when + ``loid`` wasn't minted (a dirty exit IP), so the caller rotates to a fresh + sticky IP and retries. Proven live 2026-07-18 (see e2e step 0). - Tries ``svc/shreddit`` first, then ``old.reddit`` (yt-dlp's primary) as - the fallback. Live probes 2026-07-04 saw both directions across exit IPs, - but a live run 2026-07-06 had ``old.reddit`` 403 on 12/12 fresh IPs while - ``shreddit`` minted every time — Reddit appears to have shut old.reddit to - anonymous traffic, so shreddit-first saves one guaranteed-403 round trip - per warm-up. The fallback is what actually matters — it preserves - correctness if a given IP (or Reddit) flips back the other way. That cost - is amortized: ``fan_out`` reuses one warmed session per worker across many - jobs, so warm-up runs once per worker, not per fetch. + Takes the sticky ``proxy`` URL (never the fetch session) so the browser + egresses the SAME exit IP the HTTP ``.json`` fetches will replay the jar on. + A module-level semaphore bounds concurrent browser launches under fan-out. - ponytail: sequential two-source warm burns 1 wasted request on ~half of new - sessions. A parallel warm (gather both, take whichever mints) removes the - latency but always spends 2 requests; not worth it while warm-up is - once-per-worker. Revisit only if session churn (not reuse) dominates. - - Takes an already-open ``session`` (never constructs one) so tests can drive - warm/rotate deterministically with a fake session, exactly like the youtube - sibling's fetch-resilience tests. + What actually clears Reddit's wall is the stealth browser + ``google_search`` + referer, NOT captcha solving. ``solve_cloudflare`` and ``network_idle`` were + each ~60s of dead wait per warm (Reddit's wall isn't Cloudflare, so the solver + hunts a challenge that never appears; and its long-lived connections never go + network-idle) — dropping both took the warm from ~133s to ~10s with ``loid`` + still minting 3/3 across us/gb/de exits (measured 2026-07-18). A rare miss + still self-heals: :func:`fetch_json` rotates to a fresh sticky IP and re-warms. """ - seen: set[str] = set() - with suppress(Exception): - page = await session.get(_SHREDDIT_URL.format(slug=slug), headers=_HEADERS) - seen |= _response_cookie_names(page) - if _LOID_COOKIE in seen: - return True - - # Fallback: mints loid on exit IPs where shreddit 403s instead. - with suppress(Exception): - page = await session.get(_OLD_REDDIT_URL, headers=_HEADERS) - seen |= _response_cookie_names(page) - return _LOID_COOKIE in seen + if proxy is None: + return None + async with _warm_slots: + try: + async with AsyncStealthySession( + headless=True, + google_search=True, + network_idle=False, + proxy=proxy, + timeout=_WARM_TIMEOUT_MS, + ) as sess: + page = await sess.fetch(_WARM_HTML_URL) + jar = _browser_cookie_jar(page) + return jar if _LOID_COOKIE in jar else None + except Exception as e: # a browser crash must not abort the flow + logger.warning("[reddit] browser warm failed: %s", e) + return None -async def _get_page(session: Any, url: str) -> Any: - """GET through the warmed sticky session, or a one-shot proxied fetch.""" +async def _get_page(session: Any, url: str, cookies: dict[str, str]) -> Any: + """GET through the warmed sticky session, or a one-shot proxied fetch. + + ``cookies`` is the browser-minted jar; ``over18`` is merged so NSFW listings + aren't blanked. The one-shot path (no bound session) has no minted jar and is + a best-effort fallback only — it can't clear the JS challenge. + """ if session is not None: - return await session.get(url, headers=_HEADERS, cookies=_OVER18_COOKIES) + return await session.get( + url, headers=_HEADERS, cookies={**cookies, **_OVER18_COOKIES} + ) return await AsyncFetcher.get( url, headers=_HEADERS, @@ -336,7 +392,7 @@ async def fetch_json(path: str, params: dict[str, Any] | None = None) -> Any | N session = holder.session try: if session is not None and not holder.warmed: - warmed_ok = await warm_session(session) + warmed_ok = await holder.warm() holder.warmed = True # attempted; don't re-warm this IP if not warmed_ok: if attempt < _MAX_ROTATIONS: @@ -348,7 +404,7 @@ async def fetch_json(path: str, params: dict[str, Any] | None = None) -> Any | N ) await holder.pace() - page = await _get_page(session, url) + page = await _get_page(session, url, holder.cookies) status = page.status if status == 200: diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py index bc8af56ff..9fb0ced37 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py @@ -10,10 +10,11 @@ signs correctly for whatever version TikTok ships. The pure response-shape parsing lives in :func:`items_from_response`; this module is the untested browser-I/O glue (covered by the e2e smoke, not unit tests). -Needs a residential proxy; datacenter IPs get empty bodies and 429s. The profile -feed returns an empty 200 to headless sessions, so :func:`fetch_item_list` goes -headful only when ``CRAWL_HEADED_XVFB_ENABLED`` promises an Xvfb display — else it -stays headless and degrades to an ``ErrorItem`` instead of crashing on launch. +Needs a residential proxy; datacenter IPs get empty bodies and 429s. TikTok also +withholds the feed (empty 200, or a redirect to /login) from the provider's +unpinned worldwide pool, so :func:`_fetch_with_rotation` walks country-pinned +exits until a draw is non-empty — the same escape the Reddit/ttwid warm path uses +(see :mod:`app.utils.proxy.rotation`). """ from __future__ import annotations @@ -22,6 +23,7 @@ import asyncio import logging from collections.abc import Callable from typing import Any +from urllib.parse import urlsplit from scrapling.fetchers import StealthyFetcher @@ -30,7 +32,8 @@ from app.proprietary.web_crawler.stealth import ( build_stealthy_kwargs, get_stealth_config, ) -from app.utils.proxy import get_proxy_url +from app.utils.proxy import get_geo_proxy_url +from app.utils.proxy.rotation import country_for_rotation from ..extraction import ( comments_from_response, @@ -224,6 +227,20 @@ def _build_page_action( return page_action +def _primer_url(url: str) -> str: + """A tiny same-origin URL (``/robots.txt``) for the initial navigation. + + Scrapling's outer ``page.goto`` waits for the ``load`` event, which TikTok's + SPA feed pages never fire — so navigating straight to the target burns the + full 30s timeout every fetch. Priming on ``robots.txt`` (a plain-text 200 that + fires ``load`` at once, same origin so referer/cookies stay coherent) lets the + ``page_action`` then reach the real target with ``domcontentloaded`` — ~4x + faster on profile feeds with no loss of capture. + """ + parts = urlsplit(url) + return f"{parts.scheme}://{parts.netloc}/robots.txt" + + def _fetch_sync( url: str, target_count: int, @@ -231,15 +248,16 @@ def _fetch_sync( extract: ExtractFn, interact: InteractFn, *, + proxy: str | None, headless: bool = True, ) -> list[dict[str, Any]]: collected: list[dict[str, Any]] = [] kwargs = build_stealthy_kwargs(get_stealth_config()) StealthyFetcher.fetch( - url, + _primer_url(url), headless=headless, network_idle=False, - proxy=get_proxy_url(), + proxy=proxy, page_action=_build_page_action( collected, url, target_count, markers, extract, interact ), @@ -248,40 +266,68 @@ def _fetch_sync( return collected[:target_count] -async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, Any]]: - """Return up to ``target_count`` itemStructs from a listing page's XHRs. +async def _fetch_with_rotation( + page_url: str, + target_count: int, + markers: tuple[str, ...], + extract: ExtractFn, + interact: InteractFn, + *, + headless: bool = True, +) -> list[dict[str, Any]]: + """Capture matching XHRs, walking exit *countries* until a draw is non-empty. - Headful when ``CRAWL_HEADED_XVFB_ENABLED`` promises a display (the profile feed - is empty to headless sessions); headless otherwise so launch never fails. - Retries an empty draw up to ``TIKTOK_LISTING_MAX_ATTEMPTS`` for a fresh exit IP. + TikTok withholds ``item_list``/``comment`` bodies from DataImpulse's unpinned + worldwide pool (the pool ``PROXY_URL`` uses for SERP health) but serves them on + country-pinned exits, so each empty draw retries on a fresh country rather than + the same flagged pool (see :mod:`app.utils.proxy.rotation`). Non-geo providers + ignore the country and re-draw their one URL, so this stays a safe no-op there. """ - headless = not config.CRAWL_HEADED_XVFB_ENABLED attempts = max(1, config.TIKTOK_LISTING_MAX_ATTEMPTS) - for attempt in range(1, attempts + 1): + for attempt in range(attempts): + proxy = get_geo_proxy_url(country_for_rotation(attempt)) items = await asyncio.to_thread( _fetch_sync, page_url, target_count, - _ITEM_LIST_MARKERS, - items_from_response, - _scroll_page, + markers, + extract, + interact, + proxy=proxy, headless=headless, ) - if items or attempt == attempts: + if items or attempt == attempts - 1: return items logger.info( - "[tiktok] empty item_list for %s (attempt %d/%d); retrying on a fresh exit IP", + "[tiktok] empty %s for %s (attempt %d/%d); retrying on a fresh exit country", + markers[0], page_url, - attempt, + attempt + 1, attempts, ) return [] +async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, Any]]: + """Return up to ``target_count`` itemStructs from a listing page's XHRs. + + Headful when ``CRAWL_HEADED_XVFB_ENABLED`` promises a display, headless + otherwise so launch never fails. Retries empty draws across a spread of exit + countries (``TIKTOK_LISTING_MAX_ATTEMPTS``) to escape a TikTok-flagged pool. + """ + return await _fetch_with_rotation( + page_url, + target_count, + _ITEM_LIST_MARKERS, + items_from_response, + _scroll_page, + headless=not config.CRAWL_HEADED_XVFB_ENABLED, + ) + + async def fetch_user_search(page_url: str, target_count: int) -> list[dict[str, Any]]: """Return up to ``target_count`` ``user_info`` records from a user-search page.""" - return await asyncio.to_thread( - _fetch_sync, + return await _fetch_with_rotation( page_url, target_count, _USER_SEARCH_MARKERS, @@ -292,8 +338,7 @@ async def fetch_user_search(page_url: str, target_count: int) -> list[dict[str, async def fetch_comments(page_url: str, target_count: int) -> list[dict[str, Any]]: """Return up to ``target_count`` raw comment records from a video page's XHRs.""" - return await asyncio.to_thread( - _fetch_sync, + return await _fetch_with_rotation( page_url, target_count, _COMMENT_MARKERS, @@ -304,8 +349,7 @@ async def fetch_comments(page_url: str, target_count: int) -> list[dict[str, Any async def fetch_trending(page_url: str, target_count: int) -> list[dict[str, Any]]: """Return up to ``target_count`` trending itemStructs from the Explore feed.""" - return await asyncio.to_thread( - _fetch_sync, + return await _fetch_with_rotation( page_url, target_count, _EXPLORE_MARKERS, diff --git a/surfsense_backend/pyproject.toml b/surfsense_backend/pyproject.toml index 8fc222fda..bd346b306 100644 --- a/surfsense_backend/pyproject.toml +++ b/surfsense_backend/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "surf-new-backend" -version = "0.0.33" +version = "0.0.34" description = "SurfSense Backend" requires-python = ">=3.12" dependencies = [ diff --git a/surfsense_backend/scripts/e2e_reddit_scraper.py b/surfsense_backend/scripts/e2e_reddit_scraper.py index 7bedc0812..c38365079 100644 --- a/surfsense_backend/scripts/e2e_reddit_scraper.py +++ b/surfsense_backend/scripts/e2e_reddit_scraper.py @@ -9,10 +9,10 @@ This is NOT a pytest test (it needs live network + a residential/custom proxy). It: Step 0 — go/no-go probe (folds in the old scripts/reddit_probe.py): open a - proxy session, warm a ``loid`` (svc/shreddit first, old.reddit fallback), - then do sequential ``.json`` fetches on the SAME sticky IP and assert each - returns a Reddit Listing. If this fails the whole approach is invalid — - later steps are skipped. + proxy session, browser-warm a ``loid`` cookie jar on the sticky exit IP, + then do sequential plain-HTTP ``.json`` fetches on the SAME sticky IP + (replaying the jar) and assert each returns a Reddit Listing. If this fails + the whole approach is invalid — later steps are skipped. Step 1 — scrape a discovered post URL (post + a few comments). Step 2 — scrape a subreddit listing. Step 3 — run a search query. @@ -96,8 +96,11 @@ async def step0_probe() -> bool: return _check( "proxy configured", False, "no proxy -> set PROXY_PROVIDER + creds" ) - minted = await warm_session(holder.session) + jar = await warm_session(holder.proxy) + if jar: + holder.cookies = jar holder.warmed = True # don't let fetch_json re-warm; we just warmed it + minted = bool(jar) _check("loid warm-up minted a session", minted) oks: list[bool] = [] for path in (f"r/{_SUBREDDIT}/hot", "r/programming/new", f"r/{_SUBREDDIT}/hot"): diff --git a/surfsense_backend/scripts/e2e_tiktok_scrape.py b/surfsense_backend/scripts/e2e_tiktok_scrape.py index 4a0063303..81ff1afe4 100644 --- a/surfsense_backend/scripts/e2e_tiktok_scrape.py +++ b/surfsense_backend/scripts/e2e_tiktok_scrape.py @@ -39,6 +39,12 @@ from urllib.parse import urlsplit from dotenv import load_dotenv # --- bootstrap: load .env and put the backend root on sys.path before app.* --- +# Scraped captions carry arbitrary Unicode (emoji, CJK, decorative glyphs); the +# Windows cp1252 console can't encode it and would abort the whole run on a single +# character. Emit UTF-8 and replace anything un-encodable instead of crashing. +if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(encoding="utf-8", errors="replace") + _BACKEND_ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(_BACKEND_ROOT)) for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"): @@ -50,7 +56,7 @@ _FIXTURES = _BACKEND_ROOT / "tests" / "fixtures" / "tiktok" # Evergreen public targets: a regular high-volume creator, a broad hashtag, and # a common search term. -_PROFILE = "nasa" +_PROFILE = "tiktok" _HASHTAG = "food" _SEARCH = "meal prep" _COUNT = 5 diff --git a/surfsense_backend/tests/unit/platforms/reddit/test_fetch_resilience.py b/surfsense_backend/tests/unit/platforms/reddit/test_fetch_resilience.py index 6f5362fd5..aeddcbc5c 100644 --- a/surfsense_backend/tests/unit/platforms/reddit/test_fetch_resilience.py +++ b/surfsense_backend/tests/unit/platforms/reddit/test_fetch_resilience.py @@ -1,9 +1,9 @@ """Offline resilience tests for the Reddit fetch seam and fan-out worker pool. -No network. Fake sessions drive the ``loid`` warm-up + rotate-on-block + backoff +No network. Fake sessions drive the browser-warm + rotate-on-block + backoff paths deterministically (in live runs the first IP warms and returns 200s, so -these branches rarely fire). Mirrors the youtube sibling's -``test_fetch_resilience.py`` shape, extended with a fake warm-up. +these branches rarely fire). The warm is now a browser mint (``holder.warm()``), +so the fake holder fakes minting a cookie jar instead of HTTP GETs to a warm URL. """ from __future__ import annotations @@ -22,9 +22,8 @@ _LISTING = {"kind": "Listing", "data": {"children": [], "after": None}} class _FakePage: - def __init__(self, status: int, *, cookies: dict | None = None, payload=None): + def __init__(self, status: int, *, payload=None): self.status = status - self.cookies = cookies or {} self._payload = payload if payload is not None else _LISTING def json(self): @@ -36,48 +35,53 @@ class _FakePage: class _FakeSession: - """One 'IP': warm-up mints loid per flags; ``.json`` GETs return ``status``.""" + """One 'IP': ``can_warm`` gates the browser mint; ``.json`` GETs return ``status``.""" def __init__( self, status: int = 200, *, - shreddit_loid: bool = True, - old_loid: bool = False, + can_warm: bool = True, payload=None, ) -> None: self.status = status - self.shreddit_loid = shreddit_loid - self.old_loid = old_loid + self.can_warm = can_warm self.payload = payload self.json_calls = 0 - self.warm_calls = 0 async def get(self, url, headers=None, cookies=None): - if "svc/shreddit" in url: - self.warm_calls += 1 - ck = {"loid": "x", "session_tracker": "y"} if self.shreddit_loid else {} - return _FakePage(200, cookies=ck) - if "old.reddit.com" in url: - self.warm_calls += 1 - return _FakePage(200, cookies={"loid": "x"} if self.old_loid else {}) self.json_calls += 1 return _FakePage(self.status, payload=self.payload) class _FakeHolder: - """Holder whose ``rotate()`` advances to the next fake session (a new IP).""" + """Holder whose ``rotate()`` advances to the next fake session (a new IP). + + ``warm()`` fakes the browser mint: it succeeds (stashing a jar) only when the + current fake session's ``can_warm`` is set, mirroring a clean vs dirty exit. + """ def __init__(self, sessions: list[_FakeSession]) -> None: self._sessions = sessions self.session = sessions[0] self.rotations = 0 self.warmed = False + self.proxy = "http://sticky.example:823" + self.cookies: dict[str, str] = {} + self.warm_calls = 0 + + async def warm(self) -> bool: + self.warm_calls += 1 + if self.session.can_warm: + self.cookies = {"loid": "x"} + return True + return False async def rotate(self): self.rotations += 1 self.session = self._sessions[min(self.rotations, len(self._sessions) - 1)] self.warmed = False # loid binds to the IP: re-warm on the fresh one + self.cookies = {} return self.session async def pace(self) -> None: @@ -95,21 +99,8 @@ def _no_sleep(monkeypatch) -> None: async def test_warms_then_returns_json(): - # shreddit is tried first and mints loid -> a single warm call. - holder = _FakeHolder([_FakeSession(200, shreddit_loid=True)]) - token = _current_session.set(holder) - try: - result = await fetch_json("r/python/hot") - finally: - _current_session.reset(token) - assert result == _LISTING - assert holder.rotations == 0 - assert holder.session.warm_calls == 1 # warmed exactly once - - -async def test_warm_falls_back_to_old_reddit(): - # shreddit doesn't mint loid, old.reddit does -> still warms on the same IP. - holder = _FakeHolder([_FakeSession(200, shreddit_loid=False, old_loid=True)]) + # The first IP warms (mints a jar) -> a single warm call, no rotation. + holder = _FakeHolder([_FakeSession(200, can_warm=True)]) token = _current_session.set(holder) try: result = await fetch_json("r/python/hot") @@ -117,14 +108,16 @@ async def test_warm_falls_back_to_old_reddit(): _current_session.reset(token) assert result == _LISTING assert holder.rotations == 0 + assert holder.warm_calls == 1 # warmed exactly once + assert holder.cookies == {"loid": "x"} # jar stashed for the .json fetch async def test_rotates_when_warm_fails_then_succeeds(): # IP0 can't mint loid at all -> rotate; IP1 warms fine. holder = _FakeHolder( [ - _FakeSession(200, shreddit_loid=False, old_loid=False), - _FakeSession(200, shreddit_loid=True), + _FakeSession(200, can_warm=False), + _FakeSession(200, can_warm=True), ] ) token = _current_session.set(holder) @@ -138,10 +131,7 @@ async def test_rotates_when_warm_fails_then_succeeds(): async def test_raises_when_no_ip_can_warm(): holder = _FakeHolder( - [ - _FakeSession(200, shreddit_loid=False, old_loid=False) - for _ in range(fetch._MAX_ROTATIONS + 1) - ] + [_FakeSession(200, can_warm=False) for _ in range(fetch._MAX_ROTATIONS + 1)] ) token = _current_session.set(holder) try: @@ -157,7 +147,7 @@ async def test_raises_when_no_ip_can_warm(): async def test_rotates_and_rewarms_on_403(): - holder = _FakeHolder([_FakeSession(403), _FakeSession(200, old_loid=True)]) + holder = _FakeHolder([_FakeSession(403), _FakeSession(200, can_warm=True)]) token = _current_session.set(holder) try: result = await fetch_json("r/python/hot") @@ -165,7 +155,7 @@ async def test_rotates_and_rewarms_on_403(): _current_session.reset(token) assert result == _LISTING assert holder.rotations == 1 - assert holder.session.warm_calls == 1 # re-warmed on the fresh IP + assert holder.warm_calls == 2 # re-warmed on the fresh IP after the 403 async def test_404_returns_none_without_rotating(): @@ -185,9 +175,6 @@ async def test_429_backs_off_without_rotating(monkeypatch): session = _FakeSession(429) async def _get(url, headers=None, cookies=None): - if "svc/shreddit" in url or "old.reddit.com" in url: - session.warm_calls += 1 - return _FakePage(200, cookies={"loid": "x"}) session.json_calls += 1 return _FakePage(429 if session.json_calls == 1 else 200) diff --git a/surfsense_backend/uv.lock b/surfsense_backend/uv.lock index 72e1a8553..3a27770b6 100644 --- a/surfsense_backend/uv.lock +++ b/surfsense_backend/uv.lock @@ -45,6 +45,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -88,6 +91,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -131,6 +137,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -174,6 +183,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -217,6 +229,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -260,6 +275,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -303,6 +321,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -346,6 +367,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -389,6 +413,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -432,6 +459,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -489,6 +519,10 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -532,6 +566,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -575,6 +612,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -618,6 +658,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -661,6 +704,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -704,6 +750,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -747,6 +796,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -790,6 +842,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -833,6 +888,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -876,6 +934,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -919,6 +980,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -976,6 +1040,10 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1019,6 +1087,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1062,6 +1133,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1105,6 +1179,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1148,6 +1225,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1191,6 +1271,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1234,6 +1317,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1277,6 +1363,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1320,6 +1409,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1363,6 +1455,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1406,6 +1501,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1463,6 +1561,10 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1506,6 +1608,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1591,6 +1696,12 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1634,6 +1745,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1677,6 +1791,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1762,6 +1879,12 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1847,6 +1970,12 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1890,6 +2019,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", ] conflicts = [[ @@ -10737,7 +10869,7 @@ wheels = [ [[package]] name = "surf-new-backend" -version = "0.0.33" +version = "0.0.34" source = { editable = "." } dependencies = [ { name = "alembic" }, diff --git a/surfsense_browser_extension/package.json b/surfsense_browser_extension/package.json index 43754e271..02a5b7b95 100644 --- a/surfsense_browser_extension/package.json +++ b/surfsense_browser_extension/package.json @@ -1,7 +1,7 @@ { "name": "surfsense_browser_extension", "displayName": "Surfsense Browser Extension", - "version": "0.0.33", + "version": "0.0.34", "description": "Extension to collect Browsing History for SurfSense.", "author": "https://github.com/MODSetter", "engines": { diff --git a/surfsense_desktop/package.json b/surfsense_desktop/package.json index 8d07f30fa..3eb3484f9 100644 --- a/surfsense_desktop/package.json +++ b/surfsense_desktop/package.json @@ -1,7 +1,7 @@ { "name": "surfsense-desktop", "productName": "SurfSense", - "version": "0.0.33", + "version": "0.0.34", "description": "SurfSense Desktop App", "main": "dist/main.js", "scripts": { diff --git a/surfsense_web/package.json b/surfsense_web/package.json index 59bf872a8..82e4144f0 100644 --- a/surfsense_web/package.json +++ b/surfsense_web/package.json @@ -1,6 +1,6 @@ { "name": "surfsense_web", - "version": "0.0.33", + "version": "0.0.34", "private": true, "packageManager": "pnpm@10.26.0", "description": "SurfSense Frontend",