mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-20 23:21:06 +02:00
feat: bumped version to 0.0.34
This commit is contained in:
DESKTOP-RTLN3BA\$punk
parent
726d3f784e
commit
87b0ba86e4
11 changed files with 383 additions and 155 deletions
2
VERSION
2
VERSION
|
|
@ -1 +1 @@
|
|||
0.0.33
|
||||
0.0.34
|
||||
|
|
|
|||
|
|
@ -1,25 +1,30 @@
|
|||
"""Proxy-aware fetch seam for the Reddit scraper (no browser).
|
||||
"""Proxy-aware fetch seam for the Reddit scraper (browser-warm, HTTP-fetch).
|
||||
|
||||
All network I/O flows through :func:`fetch_json` and always egresses through the
|
||||
residential proxy (a direct hit would expose and risk-block the server IP).
|
||||
|
||||
Reddit deprecated *cold* unauthenticated ``.json`` (a bare anonymous GET now
|
||||
403s). The maintained anonymous recipe (proven live 2026-07-04, see
|
||||
``scripts/e2e_reddit_scraper.py`` step 0) is:
|
||||
Reddit shut anonymous ``.json`` (~May 2026) and, crucially, put a JavaScript
|
||||
challenge in front of minting the anonymous ``loid`` session cookie: a plain
|
||||
HTTP GET (even Chrome-impersonated) can no longer mint ``loid`` — it just 403s
|
||||
on the bot wall (confirmed live 2026-07-18, and yt-dlp #16877). So the recipe is
|
||||
now two-phase on ONE sticky exit IP:
|
||||
|
||||
warm one anonymous session cookie (``loid``) with a plain GET to
|
||||
``www.reddit.com/svc/shreddit/<slug>`` (``old.reddit.com`` fallback), then
|
||||
GET ``www.reddit.com/<path>/.json?raw_json=1`` through that same
|
||||
Chrome-impersonated, sticky-IP session. Which warm URL mints ``loid`` is
|
||||
exit-IP dependent, so the order is a tiebreak — see :func:`warm_session`.
|
||||
1. WARM (once per session): open a real patchright-Chromium stealth browser
|
||||
(:func:`warm_session`) on a sticky proxy IP, load a public HTML page so
|
||||
the JS challenge runs and mints the anonymous cookie jar (incl. ``loid``).
|
||||
2. FETCH (many, fast): replay that minted jar through a plain-HTTP,
|
||||
Chrome-impersonated ``FetcherSession`` pinned to the SAME sticky IP,
|
||||
GETting ``www.reddit.com/<path>/.json?raw_json=1``. The ``.json`` body
|
||||
comes back as raw JSON (no browser HTML wrapper) because this phase is
|
||||
plain HTTP, keeping the fast/cheap fan-out — browser cost is paid ONCE
|
||||
per warm, not per fetch.
|
||||
|
||||
``loid`` is Reddit's equivalent of Google Maps' ``NID`` session cookie: an
|
||||
anonymous, logged-out id that unlocks the public API — no account, no browser.
|
||||
This module is a direct port of ``../youtube/innertube.py``'s rotate-on-block
|
||||
sticky-session pattern (``_RotatingSession`` + ``_current_session`` ContextVar +
|
||||
``open_proxy_holder``/``bind_proxy_holder``/``proxy_session``), with a
|
||||
Reddit-specific :func:`warm_session` bolted on and a ``.json``-shaped
|
||||
:func:`fetch_json` instead of an InnerTube POST.
|
||||
``loid`` binds to the exit IP, so both phases share one sticky proxy session id
|
||||
(see :meth:`_RotatingSession._open`); a 403/blocked IP rotates to a fresh sticky
|
||||
id + country and re-warms. This module is a port of ``../youtube/innertube.py``'s
|
||||
rotate-on-block sticky-session pattern (``_RotatingSession`` + ``_current_session``
|
||||
ContextVar + ``open_proxy_holder``/``bind_proxy_holder``/``proxy_session``), with
|
||||
the browser warm bolted on and a ``.json``-shaped :func:`fetch_json`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -29,15 +34,16 @@ import json
|
|||
import logging
|
||||
import random
|
||||
import time
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager, suppress
|
||||
from contextvars import ContextVar
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from scrapling.fetchers import AsyncFetcher, FetcherSession
|
||||
from scrapling.fetchers import AsyncFetcher, AsyncStealthySession, FetcherSession
|
||||
|
||||
from app.utils.proxy import get_geo_proxy_url, get_proxy_url
|
||||
from app.utils.proxy import get_proxy_url, get_sticky_proxy_url
|
||||
|
||||
# Shared cross-country rotation walk (also used by the TikTok sibling). Kept under
|
||||
# the historical private names this module and its tests reference.
|
||||
|
|
@ -111,29 +117,49 @@ _HEADERS = {"Accept-Language": "en-US,en;q=0.9"}
|
|||
# (the caller filters on ``includeNSFW`` downstream). Mirrors the probe.
|
||||
_OVER18_COOKIES = {"over18": "1"}
|
||||
|
||||
# ``svc/shreddit`` needs *a* path to render; any always-public subreddit mints
|
||||
# ``loid`` just the same. ``r/popular`` always exists, so it's a safe default
|
||||
# regardless of which target this session ends up serving.
|
||||
_WARM_SLUG = "r/popular"
|
||||
_SHREDDIT_URL = (
|
||||
"https://www.reddit.com/svc/shreddit/{slug}"
|
||||
"?render-mode=partial&seeker-session=false"
|
||||
)
|
||||
_OLD_REDDIT_URL = "https://old.reddit.com/"
|
||||
# The browser warm loads *a* public HTML page so Reddit's JS challenge runs and
|
||||
# mints the anonymous jar; any always-public subreddit works. ``r/popular``
|
||||
# always exists, so it's a safe default regardless of this session's target.
|
||||
_WARM_HTML_URL = "https://www.reddit.com/r/popular/"
|
||||
_LOID_COOKIE = "loid"
|
||||
|
||||
# The stealth browser warm (cold start + the page's own ``load``) lands in
|
||||
# ~6-15s once the solve_cloudflare/network_idle dead-waits are dropped (see
|
||||
# warm_session), so 30s is a generous ceiling that still bounds a dead exit; the
|
||||
# fast HTTP ``.json`` fetches keep the tight _REQUEST_TIMEOUT_S above.
|
||||
_WARM_TIMEOUT_MS = 30_000
|
||||
|
||||
# Bound concurrent browser warms so a wide fan-out (up to _FANOUT_CONCURRENCY=16
|
||||
# workers, each warming once) can't spawn 16 Chromiums at once and OOM the box.
|
||||
# Warms are staggered, not serialized: a freed slot starts the next worker's warm
|
||||
# while already-warmed workers fetch over plain HTTP.
|
||||
# ponytail: 4 is tuned for a ~2GB container; raise on a bigger box, lower if warm
|
||||
# spikes push memory. Ceiling: a burst of >4 cold workers queues behind this.
|
||||
_WARM_CONCURRENCY = 4
|
||||
_warm_slots = asyncio.Semaphore(_WARM_CONCURRENCY)
|
||||
|
||||
|
||||
def now_iso() -> str:
|
||||
"""UTC timestamp in the millisecond ISO shape used by scraper output."""
|
||||
return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
|
||||
|
||||
|
||||
def _response_cookie_names(page: Any) -> set[str]:
|
||||
"""Cookie names set by a response (best-effort across scrapling shapes)."""
|
||||
def _browser_cookie_jar(page: Any) -> dict[str, str]:
|
||||
"""Extract a ``{name: value}`` jar from a browser Response (best-effort).
|
||||
|
||||
Scrapling's browser Response exposes cookies as a list of ``{name, value,
|
||||
...}`` dicts (patchright/playwright shape); tolerate a plain ``{name: value}``
|
||||
dict too so a shape change or a fake in tests still works.
|
||||
"""
|
||||
cookies = getattr(page, "cookies", None)
|
||||
if isinstance(cookies, dict):
|
||||
return set(cookies.keys())
|
||||
return set()
|
||||
return {str(k): str(v) for k, v in cookies.items()}
|
||||
jar: dict[str, str] = {}
|
||||
if isinstance(cookies, (list, tuple)):
|
||||
for item in cookies:
|
||||
if isinstance(item, dict) and "name" in item and "value" in item:
|
||||
jar[str(item["name"])] = str(item["value"])
|
||||
return jar
|
||||
|
||||
|
||||
def _parse_json(page: Any) -> Any | None:
|
||||
|
|
@ -185,23 +211,44 @@ class _RotatingSession:
|
|||
self.rotations = 0
|
||||
self.warmed = False
|
||||
self.country = ""
|
||||
# Sticky proxy URL shared by the browser warm AND the HTTP fetch session
|
||||
# so both egress the SAME exit IP (``loid`` binds to that IP). The minted
|
||||
# cookie jar is replayed on every ``.json`` fetch.
|
||||
self.proxy: str | None = None
|
||||
self.cookies: dict[str, str] = {}
|
||||
self._last_at = 0.0
|
||||
|
||||
async def _open(self) -> None:
|
||||
self.warmed = False
|
||||
self.cookies = {}
|
||||
self.country = _country_for_rotation(self.rotations)
|
||||
proxy = get_geo_proxy_url(self.country)
|
||||
if proxy is None:
|
||||
# A fresh vendor session id per (re)open pins a distinct sticky exit IP,
|
||||
# so rotating drops the dirty IP AND its dead cookie jar in one step.
|
||||
session_id = uuid.uuid4().hex[:12]
|
||||
self.proxy = get_sticky_proxy_url(session_id, self.country)
|
||||
if self.proxy is None:
|
||||
self._cm = self.session = None
|
||||
return
|
||||
self._cm = FetcherSession(
|
||||
proxy=proxy,
|
||||
proxy=self.proxy,
|
||||
stealthy_headers=True,
|
||||
impersonate="chrome",
|
||||
timeout=_REQUEST_TIMEOUT_S,
|
||||
)
|
||||
self.session = await self._cm.__aenter__()
|
||||
|
||||
async def warm(self) -> bool:
|
||||
"""Browser-mint the anonymous cookie jar on this session's sticky IP.
|
||||
|
||||
Returns ``True`` and stores the jar (:attr:`cookies`) when ``loid`` was
|
||||
minted, else ``False`` (caller rotates the IP and retries).
|
||||
"""
|
||||
jar = await warm_session(self.proxy)
|
||||
if jar:
|
||||
self.cookies = jar
|
||||
return True
|
||||
return False
|
||||
|
||||
async def close(self) -> None:
|
||||
if self._cm is not None:
|
||||
with suppress(Exception): # best-effort teardown
|
||||
|
|
@ -260,49 +307,58 @@ async def proxy_session():
|
|||
await holder.close()
|
||||
|
||||
|
||||
async def warm_session(session: Any, *, slug: str = _WARM_SLUG) -> bool:
|
||||
"""Mint an anonymous ``loid`` cookie on a freshly opened session.
|
||||
async def warm_session(proxy: str | None) -> dict[str, str] | None:
|
||||
"""Browser-mint the anonymous cookie jar (incl. ``loid``) on a sticky IP.
|
||||
|
||||
Returns ``True`` when a ``loid`` was issued (the session can now reach
|
||||
``.json``), else ``False`` (caller rotates the IP and retries).
|
||||
Reddit gates ``loid`` minting behind a JS challenge that a plain-HTTP client
|
||||
can't clear, so this spins up patchright-Chromium ONCE per session on the
|
||||
given sticky proxy IP, loads a public HTML page so the challenge runs, and
|
||||
returns the minted ``{name: value}`` cookie jar. Returns ``None`` when
|
||||
``loid`` wasn't minted (a dirty exit IP), so the caller rotates to a fresh
|
||||
sticky IP and retries. Proven live 2026-07-18 (see e2e step 0).
|
||||
|
||||
Tries ``svc/shreddit`` first, then ``old.reddit`` (yt-dlp's primary) as
|
||||
the fallback. Live probes 2026-07-04 saw both directions across exit IPs,
|
||||
but a live run 2026-07-06 had ``old.reddit`` 403 on 12/12 fresh IPs while
|
||||
``shreddit`` minted every time — Reddit appears to have shut old.reddit to
|
||||
anonymous traffic, so shreddit-first saves one guaranteed-403 round trip
|
||||
per warm-up. The fallback is what actually matters — it preserves
|
||||
correctness if a given IP (or Reddit) flips back the other way. That cost
|
||||
is amortized: ``fan_out`` reuses one warmed session per worker across many
|
||||
jobs, so warm-up runs once per worker, not per fetch.
|
||||
Takes the sticky ``proxy`` URL (never the fetch session) so the browser
|
||||
egresses the SAME exit IP the HTTP ``.json`` fetches will replay the jar on.
|
||||
A module-level semaphore bounds concurrent browser launches under fan-out.
|
||||
|
||||
ponytail: sequential two-source warm burns 1 wasted request on ~half of new
|
||||
sessions. A parallel warm (gather both, take whichever mints) removes the
|
||||
latency but always spends 2 requests; not worth it while warm-up is
|
||||
once-per-worker. Revisit only if session churn (not reuse) dominates.
|
||||
|
||||
Takes an already-open ``session`` (never constructs one) so tests can drive
|
||||
warm/rotate deterministically with a fake session, exactly like the youtube
|
||||
sibling's fetch-resilience tests.
|
||||
What actually clears Reddit's wall is the stealth browser + ``google_search``
|
||||
referer, NOT captcha solving. ``solve_cloudflare`` and ``network_idle`` were
|
||||
each ~60s of dead wait per warm (Reddit's wall isn't Cloudflare, so the solver
|
||||
hunts a challenge that never appears; and its long-lived connections never go
|
||||
network-idle) — dropping both took the warm from ~133s to ~10s with ``loid``
|
||||
still minting 3/3 across us/gb/de exits (measured 2026-07-18). A rare miss
|
||||
still self-heals: :func:`fetch_json` rotates to a fresh sticky IP and re-warms.
|
||||
"""
|
||||
seen: set[str] = set()
|
||||
with suppress(Exception):
|
||||
page = await session.get(_SHREDDIT_URL.format(slug=slug), headers=_HEADERS)
|
||||
seen |= _response_cookie_names(page)
|
||||
if _LOID_COOKIE in seen:
|
||||
return True
|
||||
|
||||
# Fallback: mints loid on exit IPs where shreddit 403s instead.
|
||||
with suppress(Exception):
|
||||
page = await session.get(_OLD_REDDIT_URL, headers=_HEADERS)
|
||||
seen |= _response_cookie_names(page)
|
||||
return _LOID_COOKIE in seen
|
||||
if proxy is None:
|
||||
return None
|
||||
async with _warm_slots:
|
||||
try:
|
||||
async with AsyncStealthySession(
|
||||
headless=True,
|
||||
google_search=True,
|
||||
network_idle=False,
|
||||
proxy=proxy,
|
||||
timeout=_WARM_TIMEOUT_MS,
|
||||
) as sess:
|
||||
page = await sess.fetch(_WARM_HTML_URL)
|
||||
jar = _browser_cookie_jar(page)
|
||||
return jar if _LOID_COOKIE in jar else None
|
||||
except Exception as e: # a browser crash must not abort the flow
|
||||
logger.warning("[reddit] browser warm failed: %s", e)
|
||||
return None
|
||||
|
||||
|
||||
async def _get_page(session: Any, url: str) -> Any:
|
||||
"""GET through the warmed sticky session, or a one-shot proxied fetch."""
|
||||
async def _get_page(session: Any, url: str, cookies: dict[str, str]) -> Any:
|
||||
"""GET through the warmed sticky session, or a one-shot proxied fetch.
|
||||
|
||||
``cookies`` is the browser-minted jar; ``over18`` is merged so NSFW listings
|
||||
aren't blanked. The one-shot path (no bound session) has no minted jar and is
|
||||
a best-effort fallback only — it can't clear the JS challenge.
|
||||
"""
|
||||
if session is not None:
|
||||
return await session.get(url, headers=_HEADERS, cookies=_OVER18_COOKIES)
|
||||
return await session.get(
|
||||
url, headers=_HEADERS, cookies={**cookies, **_OVER18_COOKIES}
|
||||
)
|
||||
return await AsyncFetcher.get(
|
||||
url,
|
||||
headers=_HEADERS,
|
||||
|
|
@ -336,7 +392,7 @@ async def fetch_json(path: str, params: dict[str, Any] | None = None) -> Any | N
|
|||
session = holder.session
|
||||
try:
|
||||
if session is not None and not holder.warmed:
|
||||
warmed_ok = await warm_session(session)
|
||||
warmed_ok = await holder.warm()
|
||||
holder.warmed = True # attempted; don't re-warm this IP
|
||||
if not warmed_ok:
|
||||
if attempt < _MAX_ROTATIONS:
|
||||
|
|
@ -348,7 +404,7 @@ async def fetch_json(path: str, params: dict[str, Any] | None = None) -> Any | N
|
|||
)
|
||||
|
||||
await holder.pace()
|
||||
page = await _get_page(session, url)
|
||||
page = await _get_page(session, url, holder.cookies)
|
||||
status = page.status
|
||||
|
||||
if status == 200:
|
||||
|
|
|
|||
|
|
@ -10,10 +10,11 @@ signs correctly for whatever version TikTok ships.
|
|||
The pure response-shape parsing lives in :func:`items_from_response`; this module
|
||||
is the untested browser-I/O glue (covered by the e2e smoke, not unit tests).
|
||||
|
||||
Needs a residential proxy; datacenter IPs get empty bodies and 429s. The profile
|
||||
feed returns an empty 200 to headless sessions, so :func:`fetch_item_list` goes
|
||||
headful only when ``CRAWL_HEADED_XVFB_ENABLED`` promises an Xvfb display — else it
|
||||
stays headless and degrades to an ``ErrorItem`` instead of crashing on launch.
|
||||
Needs a residential proxy; datacenter IPs get empty bodies and 429s. TikTok also
|
||||
withholds the feed (empty 200, or a redirect to /login) from the provider's
|
||||
unpinned worldwide pool, so :func:`_fetch_with_rotation` walks country-pinned
|
||||
exits until a draw is non-empty — the same escape the Reddit/ttwid warm path uses
|
||||
(see :mod:`app.utils.proxy.rotation`).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -22,6 +23,7 @@ import asyncio
|
|||
import logging
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from scrapling.fetchers import StealthyFetcher
|
||||
|
||||
|
|
@ -30,7 +32,8 @@ from app.proprietary.web_crawler.stealth import (
|
|||
build_stealthy_kwargs,
|
||||
get_stealth_config,
|
||||
)
|
||||
from app.utils.proxy import get_proxy_url
|
||||
from app.utils.proxy import get_geo_proxy_url
|
||||
from app.utils.proxy.rotation import country_for_rotation
|
||||
|
||||
from ..extraction import (
|
||||
comments_from_response,
|
||||
|
|
@ -224,6 +227,20 @@ def _build_page_action(
|
|||
return page_action
|
||||
|
||||
|
||||
def _primer_url(url: str) -> str:
|
||||
"""A tiny same-origin URL (``/robots.txt``) for the initial navigation.
|
||||
|
||||
Scrapling's outer ``page.goto`` waits for the ``load`` event, which TikTok's
|
||||
SPA feed pages never fire — so navigating straight to the target burns the
|
||||
full 30s timeout every fetch. Priming on ``robots.txt`` (a plain-text 200 that
|
||||
fires ``load`` at once, same origin so referer/cookies stay coherent) lets the
|
||||
``page_action`` then reach the real target with ``domcontentloaded`` — ~4x
|
||||
faster on profile feeds with no loss of capture.
|
||||
"""
|
||||
parts = urlsplit(url)
|
||||
return f"{parts.scheme}://{parts.netloc}/robots.txt"
|
||||
|
||||
|
||||
def _fetch_sync(
|
||||
url: str,
|
||||
target_count: int,
|
||||
|
|
@ -231,15 +248,16 @@ def _fetch_sync(
|
|||
extract: ExtractFn,
|
||||
interact: InteractFn,
|
||||
*,
|
||||
proxy: str | None,
|
||||
headless: bool = True,
|
||||
) -> list[dict[str, Any]]:
|
||||
collected: list[dict[str, Any]] = []
|
||||
kwargs = build_stealthy_kwargs(get_stealth_config())
|
||||
StealthyFetcher.fetch(
|
||||
url,
|
||||
_primer_url(url),
|
||||
headless=headless,
|
||||
network_idle=False,
|
||||
proxy=get_proxy_url(),
|
||||
proxy=proxy,
|
||||
page_action=_build_page_action(
|
||||
collected, url, target_count, markers, extract, interact
|
||||
),
|
||||
|
|
@ -248,40 +266,68 @@ def _fetch_sync(
|
|||
return collected[:target_count]
|
||||
|
||||
|
||||
async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, Any]]:
|
||||
"""Return up to ``target_count`` itemStructs from a listing page's XHRs.
|
||||
async def _fetch_with_rotation(
|
||||
page_url: str,
|
||||
target_count: int,
|
||||
markers: tuple[str, ...],
|
||||
extract: ExtractFn,
|
||||
interact: InteractFn,
|
||||
*,
|
||||
headless: bool = True,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Capture matching XHRs, walking exit *countries* until a draw is non-empty.
|
||||
|
||||
Headful when ``CRAWL_HEADED_XVFB_ENABLED`` promises a display (the profile feed
|
||||
is empty to headless sessions); headless otherwise so launch never fails.
|
||||
Retries an empty draw up to ``TIKTOK_LISTING_MAX_ATTEMPTS`` for a fresh exit IP.
|
||||
TikTok withholds ``item_list``/``comment`` bodies from DataImpulse's unpinned
|
||||
worldwide pool (the pool ``PROXY_URL`` uses for SERP health) but serves them on
|
||||
country-pinned exits, so each empty draw retries on a fresh country rather than
|
||||
the same flagged pool (see :mod:`app.utils.proxy.rotation`). Non-geo providers
|
||||
ignore the country and re-draw their one URL, so this stays a safe no-op there.
|
||||
"""
|
||||
headless = not config.CRAWL_HEADED_XVFB_ENABLED
|
||||
attempts = max(1, config.TIKTOK_LISTING_MAX_ATTEMPTS)
|
||||
for attempt in range(1, attempts + 1):
|
||||
for attempt in range(attempts):
|
||||
proxy = get_geo_proxy_url(country_for_rotation(attempt))
|
||||
items = await asyncio.to_thread(
|
||||
_fetch_sync,
|
||||
page_url,
|
||||
target_count,
|
||||
_ITEM_LIST_MARKERS,
|
||||
items_from_response,
|
||||
_scroll_page,
|
||||
markers,
|
||||
extract,
|
||||
interact,
|
||||
proxy=proxy,
|
||||
headless=headless,
|
||||
)
|
||||
if items or attempt == attempts:
|
||||
if items or attempt == attempts - 1:
|
||||
return items
|
||||
logger.info(
|
||||
"[tiktok] empty item_list for %s (attempt %d/%d); retrying on a fresh exit IP",
|
||||
"[tiktok] empty %s for %s (attempt %d/%d); retrying on a fresh exit country",
|
||||
markers[0],
|
||||
page_url,
|
||||
attempt,
|
||||
attempt + 1,
|
||||
attempts,
|
||||
)
|
||||
return []
|
||||
|
||||
|
||||
async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, Any]]:
|
||||
"""Return up to ``target_count`` itemStructs from a listing page's XHRs.
|
||||
|
||||
Headful when ``CRAWL_HEADED_XVFB_ENABLED`` promises a display, headless
|
||||
otherwise so launch never fails. Retries empty draws across a spread of exit
|
||||
countries (``TIKTOK_LISTING_MAX_ATTEMPTS``) to escape a TikTok-flagged pool.
|
||||
"""
|
||||
return await _fetch_with_rotation(
|
||||
page_url,
|
||||
target_count,
|
||||
_ITEM_LIST_MARKERS,
|
||||
items_from_response,
|
||||
_scroll_page,
|
||||
headless=not config.CRAWL_HEADED_XVFB_ENABLED,
|
||||
)
|
||||
|
||||
|
||||
async def fetch_user_search(page_url: str, target_count: int) -> list[dict[str, Any]]:
|
||||
"""Return up to ``target_count`` ``user_info`` records from a user-search page."""
|
||||
return await asyncio.to_thread(
|
||||
_fetch_sync,
|
||||
return await _fetch_with_rotation(
|
||||
page_url,
|
||||
target_count,
|
||||
_USER_SEARCH_MARKERS,
|
||||
|
|
@ -292,8 +338,7 @@ async def fetch_user_search(page_url: str, target_count: int) -> list[dict[str,
|
|||
|
||||
async def fetch_comments(page_url: str, target_count: int) -> list[dict[str, Any]]:
|
||||
"""Return up to ``target_count`` raw comment records from a video page's XHRs."""
|
||||
return await asyncio.to_thread(
|
||||
_fetch_sync,
|
||||
return await _fetch_with_rotation(
|
||||
page_url,
|
||||
target_count,
|
||||
_COMMENT_MARKERS,
|
||||
|
|
@ -304,8 +349,7 @@ async def fetch_comments(page_url: str, target_count: int) -> list[dict[str, Any
|
|||
|
||||
async def fetch_trending(page_url: str, target_count: int) -> list[dict[str, Any]]:
|
||||
"""Return up to ``target_count`` trending itemStructs from the Explore feed."""
|
||||
return await asyncio.to_thread(
|
||||
_fetch_sync,
|
||||
return await _fetch_with_rotation(
|
||||
page_url,
|
||||
target_count,
|
||||
_EXPLORE_MARKERS,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "surf-new-backend"
|
||||
version = "0.0.33"
|
||||
version = "0.0.34"
|
||||
description = "SurfSense Backend"
|
||||
requires-python = ">=3.12"
|
||||
dependencies = [
|
||||
|
|
|
|||
|
|
@ -9,10 +9,10 @@ This is NOT a pytest test (it needs live network + a residential/custom proxy).
|
|||
It:
|
||||
|
||||
Step 0 — go/no-go probe (folds in the old scripts/reddit_probe.py): open a
|
||||
proxy session, warm a ``loid`` (svc/shreddit first, old.reddit fallback),
|
||||
then do sequential ``.json`` fetches on the SAME sticky IP and assert each
|
||||
returns a Reddit Listing. If this fails the whole approach is invalid —
|
||||
later steps are skipped.
|
||||
proxy session, browser-warm a ``loid`` cookie jar on the sticky exit IP,
|
||||
then do sequential plain-HTTP ``.json`` fetches on the SAME sticky IP
|
||||
(replaying the jar) and assert each returns a Reddit Listing. If this fails
|
||||
the whole approach is invalid — later steps are skipped.
|
||||
Step 1 — scrape a discovered post URL (post + a few comments).
|
||||
Step 2 — scrape a subreddit listing.
|
||||
Step 3 — run a search query.
|
||||
|
|
@ -96,8 +96,11 @@ async def step0_probe() -> bool:
|
|||
return _check(
|
||||
"proxy configured", False, "no proxy -> set PROXY_PROVIDER + creds"
|
||||
)
|
||||
minted = await warm_session(holder.session)
|
||||
jar = await warm_session(holder.proxy)
|
||||
if jar:
|
||||
holder.cookies = jar
|
||||
holder.warmed = True # don't let fetch_json re-warm; we just warmed it
|
||||
minted = bool(jar)
|
||||
_check("loid warm-up minted a session", minted)
|
||||
oks: list[bool] = []
|
||||
for path in (f"r/{_SUBREDDIT}/hot", "r/programming/new", f"r/{_SUBREDDIT}/hot"):
|
||||
|
|
|
|||
|
|
@ -39,6 +39,12 @@ from urllib.parse import urlsplit
|
|||
from dotenv import load_dotenv
|
||||
|
||||
# --- bootstrap: load .env and put the backend root on sys.path before app.* ---
|
||||
# Scraped captions carry arbitrary Unicode (emoji, CJK, decorative glyphs); the
|
||||
# Windows cp1252 console can't encode it and would abort the whole run on a single
|
||||
# character. Emit UTF-8 and replace anything un-encodable instead of crashing.
|
||||
if hasattr(sys.stdout, "reconfigure"):
|
||||
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
||||
|
||||
_BACKEND_ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(_BACKEND_ROOT))
|
||||
for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"):
|
||||
|
|
@ -50,7 +56,7 @@ _FIXTURES = _BACKEND_ROOT / "tests" / "fixtures" / "tiktok"
|
|||
|
||||
# Evergreen public targets: a regular high-volume creator, a broad hashtag, and
|
||||
# a common search term.
|
||||
_PROFILE = "nasa"
|
||||
_PROFILE = "tiktok"
|
||||
_HASHTAG = "food"
|
||||
_SEARCH = "meal prep"
|
||||
_COUNT = 5
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
"""Offline resilience tests for the Reddit fetch seam and fan-out worker pool.
|
||||
|
||||
No network. Fake sessions drive the ``loid`` warm-up + rotate-on-block + backoff
|
||||
No network. Fake sessions drive the browser-warm + rotate-on-block + backoff
|
||||
paths deterministically (in live runs the first IP warms and returns 200s, so
|
||||
these branches rarely fire). Mirrors the youtube sibling's
|
||||
``test_fetch_resilience.py`` shape, extended with a fake warm-up.
|
||||
these branches rarely fire). The warm is now a browser mint (``holder.warm()``),
|
||||
so the fake holder fakes minting a cookie jar instead of HTTP GETs to a warm URL.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -22,9 +22,8 @@ _LISTING = {"kind": "Listing", "data": {"children": [], "after": None}}
|
|||
|
||||
|
||||
class _FakePage:
|
||||
def __init__(self, status: int, *, cookies: dict | None = None, payload=None):
|
||||
def __init__(self, status: int, *, payload=None):
|
||||
self.status = status
|
||||
self.cookies = cookies or {}
|
||||
self._payload = payload if payload is not None else _LISTING
|
||||
|
||||
def json(self):
|
||||
|
|
@ -36,48 +35,53 @@ class _FakePage:
|
|||
|
||||
|
||||
class _FakeSession:
|
||||
"""One 'IP': warm-up mints loid per flags; ``.json`` GETs return ``status``."""
|
||||
"""One 'IP': ``can_warm`` gates the browser mint; ``.json`` GETs return ``status``."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
status: int = 200,
|
||||
*,
|
||||
shreddit_loid: bool = True,
|
||||
old_loid: bool = False,
|
||||
can_warm: bool = True,
|
||||
payload=None,
|
||||
) -> None:
|
||||
self.status = status
|
||||
self.shreddit_loid = shreddit_loid
|
||||
self.old_loid = old_loid
|
||||
self.can_warm = can_warm
|
||||
self.payload = payload
|
||||
self.json_calls = 0
|
||||
self.warm_calls = 0
|
||||
|
||||
async def get(self, url, headers=None, cookies=None):
|
||||
if "svc/shreddit" in url:
|
||||
self.warm_calls += 1
|
||||
ck = {"loid": "x", "session_tracker": "y"} if self.shreddit_loid else {}
|
||||
return _FakePage(200, cookies=ck)
|
||||
if "old.reddit.com" in url:
|
||||
self.warm_calls += 1
|
||||
return _FakePage(200, cookies={"loid": "x"} if self.old_loid else {})
|
||||
self.json_calls += 1
|
||||
return _FakePage(self.status, payload=self.payload)
|
||||
|
||||
|
||||
class _FakeHolder:
|
||||
"""Holder whose ``rotate()`` advances to the next fake session (a new IP)."""
|
||||
"""Holder whose ``rotate()`` advances to the next fake session (a new IP).
|
||||
|
||||
``warm()`` fakes the browser mint: it succeeds (stashing a jar) only when the
|
||||
current fake session's ``can_warm`` is set, mirroring a clean vs dirty exit.
|
||||
"""
|
||||
|
||||
def __init__(self, sessions: list[_FakeSession]) -> None:
|
||||
self._sessions = sessions
|
||||
self.session = sessions[0]
|
||||
self.rotations = 0
|
||||
self.warmed = False
|
||||
self.proxy = "http://sticky.example:823"
|
||||
self.cookies: dict[str, str] = {}
|
||||
self.warm_calls = 0
|
||||
|
||||
async def warm(self) -> bool:
|
||||
self.warm_calls += 1
|
||||
if self.session.can_warm:
|
||||
self.cookies = {"loid": "x"}
|
||||
return True
|
||||
return False
|
||||
|
||||
async def rotate(self):
|
||||
self.rotations += 1
|
||||
self.session = self._sessions[min(self.rotations, len(self._sessions) - 1)]
|
||||
self.warmed = False # loid binds to the IP: re-warm on the fresh one
|
||||
self.cookies = {}
|
||||
return self.session
|
||||
|
||||
async def pace(self) -> None:
|
||||
|
|
@ -95,21 +99,8 @@ def _no_sleep(monkeypatch) -> None:
|
|||
|
||||
|
||||
async def test_warms_then_returns_json():
|
||||
# shreddit is tried first and mints loid -> a single warm call.
|
||||
holder = _FakeHolder([_FakeSession(200, shreddit_loid=True)])
|
||||
token = _current_session.set(holder)
|
||||
try:
|
||||
result = await fetch_json("r/python/hot")
|
||||
finally:
|
||||
_current_session.reset(token)
|
||||
assert result == _LISTING
|
||||
assert holder.rotations == 0
|
||||
assert holder.session.warm_calls == 1 # warmed exactly once
|
||||
|
||||
|
||||
async def test_warm_falls_back_to_old_reddit():
|
||||
# shreddit doesn't mint loid, old.reddit does -> still warms on the same IP.
|
||||
holder = _FakeHolder([_FakeSession(200, shreddit_loid=False, old_loid=True)])
|
||||
# The first IP warms (mints a jar) -> a single warm call, no rotation.
|
||||
holder = _FakeHolder([_FakeSession(200, can_warm=True)])
|
||||
token = _current_session.set(holder)
|
||||
try:
|
||||
result = await fetch_json("r/python/hot")
|
||||
|
|
@ -117,14 +108,16 @@ async def test_warm_falls_back_to_old_reddit():
|
|||
_current_session.reset(token)
|
||||
assert result == _LISTING
|
||||
assert holder.rotations == 0
|
||||
assert holder.warm_calls == 1 # warmed exactly once
|
||||
assert holder.cookies == {"loid": "x"} # jar stashed for the .json fetch
|
||||
|
||||
|
||||
async def test_rotates_when_warm_fails_then_succeeds():
|
||||
# IP0 can't mint loid at all -> rotate; IP1 warms fine.
|
||||
holder = _FakeHolder(
|
||||
[
|
||||
_FakeSession(200, shreddit_loid=False, old_loid=False),
|
||||
_FakeSession(200, shreddit_loid=True),
|
||||
_FakeSession(200, can_warm=False),
|
||||
_FakeSession(200, can_warm=True),
|
||||
]
|
||||
)
|
||||
token = _current_session.set(holder)
|
||||
|
|
@ -138,10 +131,7 @@ async def test_rotates_when_warm_fails_then_succeeds():
|
|||
|
||||
async def test_raises_when_no_ip_can_warm():
|
||||
holder = _FakeHolder(
|
||||
[
|
||||
_FakeSession(200, shreddit_loid=False, old_loid=False)
|
||||
for _ in range(fetch._MAX_ROTATIONS + 1)
|
||||
]
|
||||
[_FakeSession(200, can_warm=False) for _ in range(fetch._MAX_ROTATIONS + 1)]
|
||||
)
|
||||
token = _current_session.set(holder)
|
||||
try:
|
||||
|
|
@ -157,7 +147,7 @@ async def test_raises_when_no_ip_can_warm():
|
|||
|
||||
|
||||
async def test_rotates_and_rewarms_on_403():
|
||||
holder = _FakeHolder([_FakeSession(403), _FakeSession(200, old_loid=True)])
|
||||
holder = _FakeHolder([_FakeSession(403), _FakeSession(200, can_warm=True)])
|
||||
token = _current_session.set(holder)
|
||||
try:
|
||||
result = await fetch_json("r/python/hot")
|
||||
|
|
@ -165,7 +155,7 @@ async def test_rotates_and_rewarms_on_403():
|
|||
_current_session.reset(token)
|
||||
assert result == _LISTING
|
||||
assert holder.rotations == 1
|
||||
assert holder.session.warm_calls == 1 # re-warmed on the fresh IP
|
||||
assert holder.warm_calls == 2 # re-warmed on the fresh IP after the 403
|
||||
|
||||
|
||||
async def test_404_returns_none_without_rotating():
|
||||
|
|
@ -185,9 +175,6 @@ async def test_429_backs_off_without_rotating(monkeypatch):
|
|||
session = _FakeSession(429)
|
||||
|
||||
async def _get(url, headers=None, cookies=None):
|
||||
if "svc/shreddit" in url or "old.reddit.com" in url:
|
||||
session.warm_calls += 1
|
||||
return _FakePage(200, cookies={"loid": "x"})
|
||||
session.json_calls += 1
|
||||
return _FakePage(429 if session.json_calls == 1 else 200)
|
||||
|
||||
|
|
|
|||
134
surfsense_backend/uv.lock
generated
134
surfsense_backend/uv.lock
generated
|
|
@ -45,6 +45,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -88,6 +91,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -131,6 +137,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -174,6 +183,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -217,6 +229,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -260,6 +275,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -303,6 +321,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -346,6 +367,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -389,6 +413,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -432,6 +459,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -489,6 +519,10 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -532,6 +566,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -575,6 +612,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -618,6 +658,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -661,6 +704,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -704,6 +750,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -747,6 +796,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -790,6 +842,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -833,6 +888,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -876,6 +934,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -919,6 +980,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -976,6 +1040,10 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -1019,6 +1087,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -1062,6 +1133,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -1105,6 +1179,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -1148,6 +1225,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -1191,6 +1271,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -1234,6 +1317,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -1277,6 +1363,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -1320,6 +1409,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -1363,6 +1455,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -1406,6 +1501,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -1463,6 +1561,10 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -1506,6 +1608,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -1591,6 +1696,12 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -1634,6 +1745,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -1677,6 +1791,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -1762,6 +1879,12 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -1847,6 +1970,12 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -1890,6 +2019,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
]
|
||||
conflicts = [[
|
||||
|
|
@ -10737,7 +10869,7 @@ wheels = [
|
|||
|
||||
[[package]]
|
||||
name = "surf-new-backend"
|
||||
version = "0.0.33"
|
||||
version = "0.0.34"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "alembic" },
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"name": "surfsense_browser_extension",
|
||||
"displayName": "Surfsense Browser Extension",
|
||||
"version": "0.0.33",
|
||||
"version": "0.0.34",
|
||||
"description": "Extension to collect Browsing History for SurfSense.",
|
||||
"author": "https://github.com/MODSetter",
|
||||
"engines": {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"name": "surfsense-desktop",
|
||||
"productName": "SurfSense",
|
||||
"version": "0.0.33",
|
||||
"version": "0.0.34",
|
||||
"description": "SurfSense Desktop App",
|
||||
"main": "dist/main.js",
|
||||
"scripts": {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "surfsense_web",
|
||||
"version": "0.0.33",
|
||||
"version": "0.0.34",
|
||||
"private": true,
|
||||
"packageManager": "pnpm@10.26.0",
|
||||
"description": "SurfSense Frontend",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue