Merge pull request #1571 from AnishSarkar22/feat/reddit-scraping

feat(scrapers): add anonymous Reddit scraper

surfsense_backend/tests/unit/scrapers/reddit/test_fetch_resilience.py

@@ -0,0 +1,268 @@
+"""Offline resilience tests for the Reddit fetch seam and fan-out worker pool.
+
+No network. Fake sessions drive the ``loid`` warm-up + rotate-on-block + backoff
+paths deterministically (in live runs the first IP warms and returns 200s, so
+these branches rarely fire). Mirrors the youtube sibling's
+``test_fetch_resilience.py`` shape, extended with a fake warm-up.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import AsyncIterator
+
+from app.proprietary.scrapers.reddit import fetch, scraper
+from app.proprietary.scrapers.reddit.fetch import (
+    RedditAccessBlockedError,
+    _current_session,
+    fetch_json,
+)
+
+_LISTING = {"kind": "Listing", "data": {"children": [], "after": None}}
+
+
+class _FakePage:
+    def __init__(self, status: int, *, cookies: dict | None = None, payload=None):
+        self.status = status
+        self.cookies = cookies or {}
+        self._payload = payload if payload is not None else _LISTING
+
+    def json(self):
+        return self._payload
+
+    @property
+    def body(self) -> str:
+        return json.dumps(self._payload)
+
+
+class _FakeSession:
+    """One 'IP': warm-up mints loid per flags; ``.json`` GETs return ``status``."""
+
+    def __init__(
+        self,
+        status: int = 200,
+        *,
+        shreddit_loid: bool = True,
+        old_loid: bool = False,
+        payload=None,
+    ) -> None:
+        self.status = status
+        self.shreddit_loid = shreddit_loid
+        self.old_loid = old_loid
+        self.payload = payload
+        self.json_calls = 0
+        self.warm_calls = 0
+
+    async def get(self, url, headers=None, cookies=None):
+        if "svc/shreddit" in url:
+            self.warm_calls += 1
+            ck = {"loid": "x", "session_tracker": "y"} if self.shreddit_loid else {}
+            return _FakePage(200, cookies=ck)
+        if "old.reddit.com" in url:
+            self.warm_calls += 1
+            return _FakePage(200, cookies={"loid": "x"} if self.old_loid else {})
+        self.json_calls += 1
+        return _FakePage(self.status, payload=self.payload)
+
+
+class _FakeHolder:
+    """Holder whose ``rotate()`` advances to the next fake session (a new IP)."""
+
+    def __init__(self, sessions: list[_FakeSession]) -> None:
+        self._sessions = sessions
+        self.session = sessions[0]
+        self.rotations = 0
+        self.warmed = False
+
+    async def rotate(self):
+        self.rotations += 1
+        self.session = self._sessions[min(self.rotations, len(self._sessions) - 1)]
+        self.warmed = False  # loid binds to the IP: re-warm on the fresh one
+        return self.session
+
+    async def pace(self) -> None:
+        return None
+
+    async def close(self) -> None:
+        return None
+
+
+def _no_sleep(monkeypatch) -> None:
+    async def _noop(_seconds):
+        return None
+
+    monkeypatch.setattr(fetch.asyncio, "sleep", _noop)
+
+
+async def test_warms_then_returns_json():
+    # old.reddit is tried first and mints loid -> a single warm call.
+    holder = _FakeHolder([_FakeSession(200, old_loid=True)])
+    token = _current_session.set(holder)
+    try:
+        result = await fetch_json("r/python/hot")
+    finally:
+        _current_session.reset(token)
+    assert result == _LISTING
+    assert holder.rotations == 0
+    assert holder.session.warm_calls == 1  # warmed exactly once
+
+
+async def test_warm_falls_back_to_shreddit():
+    # old.reddit doesn't mint loid, shreddit does -> still warms on the same IP.
+    holder = _FakeHolder([_FakeSession(200, shreddit_loid=True, old_loid=False)])
+    token = _current_session.set(holder)
+    try:
+        result = await fetch_json("r/python/hot")
+    finally:
+        _current_session.reset(token)
+    assert result == _LISTING
+    assert holder.rotations == 0
+
+
+async def test_rotates_when_warm_fails_then_succeeds():
+    # IP0 can't mint loid at all -> rotate; IP1 warms fine.
+    holder = _FakeHolder(
+        [
+            _FakeSession(200, shreddit_loid=False, old_loid=False),
+            _FakeSession(200, shreddit_loid=True),
+        ]
+    )
+    token = _current_session.set(holder)
+    try:
+        result = await fetch_json("r/python/hot")
+    finally:
+        _current_session.reset(token)
+    assert result == _LISTING
+    assert holder.rotations == 1
+
+
+async def test_raises_when_no_ip_can_warm():
+    holder = _FakeHolder(
+        [
+            _FakeSession(200, shreddit_loid=False, old_loid=False)
+            for _ in range(fetch._MAX_ROTATIONS + 1)
+        ]
+    )
+    token = _current_session.set(holder)
+    try:
+        raised = False
+        try:
+            await fetch_json("r/python/hot")
+        except RedditAccessBlockedError:
+            raised = True
+    finally:
+        _current_session.reset(token)
+    assert raised
+    assert holder.rotations == fetch._MAX_ROTATIONS
+
+
+async def test_rotates_and_rewarms_on_403():
+    holder = _FakeHolder([_FakeSession(403), _FakeSession(200, old_loid=True)])
+    token = _current_session.set(holder)
+    try:
+        result = await fetch_json("r/python/hot")
+    finally:
+        _current_session.reset(token)
+    assert result == _LISTING
+    assert holder.rotations == 1
+    assert holder.session.warm_calls == 1  # re-warmed on the fresh IP
+
+
+async def test_404_returns_none_without_rotating():
+    holder = _FakeHolder([_FakeSession(404), _FakeSession(200)])
+    token = _current_session.set(holder)
+    try:
+        result = await fetch_json("r/python/comments/missing")
+    finally:
+        _current_session.reset(token)
+    assert result is None
+    assert holder.rotations == 0
+
+
+async def test_429_backs_off_without_rotating(monkeypatch):
+    _no_sleep(monkeypatch)
+    # Same IP: 429 first, then a healthy 200 on retry (no rotation).
+    session = _FakeSession(429)
+
+    async def _get(url, headers=None, cookies=None):
+        if "svc/shreddit" in url or "old.reddit.com" in url:
+            session.warm_calls += 1
+            return _FakePage(200, cookies={"loid": "x"})
+        session.json_calls += 1
+        return _FakePage(429 if session.json_calls == 1 else 200)
+
+    session.get = _get  # type: ignore[method-assign]
+    holder = _FakeHolder([session])
+    token = _current_session.set(holder)
+    try:
+        result = await fetch_json("r/python/hot")
+    finally:
+        _current_session.reset(token)
+    assert result == _LISTING
+    assert holder.rotations == 0
+
+
+async def test_persistent_403_raises_blocked(monkeypatch):
+    _no_sleep(monkeypatch)
+    holder = _FakeHolder([_FakeSession(403) for _ in range(fetch._MAX_ROTATIONS + 1)])
+    token = _current_session.set(holder)
+    try:
+        raised = False
+        try:
+            await fetch_json("r/python/hot")
+        except RedditAccessBlockedError:
+            raised = True
+    finally:
+        _current_session.reset(token)
+    assert raised
+    assert holder.rotations == fetch._MAX_ROTATIONS
+
+
+class _TrackingHolder:
+    """Fake fan-out session holder that records whether it was closed."""
+
+    def __init__(self) -> None:
+        self.closed = False
+
+    async def close(self) -> None:
+        self.closed = True
+
+
+async def test_fan_out_closes_all_sessions_on_early_stop(monkeypatch):
+    holders: list[_TrackingHolder] = []
+
+    async def _fake_open():
+        h = _TrackingHolder()
+        holders.append(h)
+        return h
+
+    from contextlib import asynccontextmanager
+
+    @asynccontextmanager
+    async def _fake_bind(_holder):
+        yield _holder
+
+    monkeypatch.setattr(scraper, "open_proxy_holder", _fake_open)
+    monkeypatch.setattr(scraper, "bind_proxy_holder", _fake_bind)
+
+    async def _job(n: int) -> AsyncIterator[dict]:
+        for i in range(5):
+            yield {"job": n, "i": i}
+
+    jobs = [_job(n) for n in range(20)]
+    gen = scraper.fan_out(jobs, concurrency=4)
+    collected = []
+    async for item in gen:
+        collected.append(item)
+        if len(collected) >= 3:
+            break
+    await gen.aclose()
+
+    assert len(collected) >= 3
+    assert holders, "workers should have opened sessions"
+    assert all(h.closed for h in holders), "a worker leaked its proxy session"
+
+
+async def test_fan_out_empty_jobs_is_noop():
+    out = [x async for x in scraper.fan_out([])]
+    assert out == []

surfsense_backend/tests/unit/scrapers/reddit/test_parsers.py

@@ -0,0 +1,165 @@
+"""Offline parser tests for the Reddit scraper.
+
+Two layers:
+- Synthetic, deterministic checks of the JSON->item mapping (hand-built minimal
+  "things" — no live Reddit shapes), which run always.
+- Fixture-pinned checks against real ``.json`` captured by
+  ``scripts/e2e_reddit_scraper.py`` into ``fixtures/``; these ``skip`` when the
+  fixtures are absent (mirrors the youtube sibling). Fill in richer assertions
+  against the captured shapes during implementation.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from app.proprietary.scrapers.reddit.parsers import (
+    after,
+    children,
+    flatten_comments,
+    parse_comment,
+    parse_community,
+    parse_post,
+)
+
+_FIXTURE_DIR = Path(__file__).parent / "fixtures"
+
+
+# --- synthetic mapping (always runs) ---------------------------------------
+
+
+def test_parse_post_maps_core_fields():
+    thing = {
+        "kind": "t3",
+        "data": {
+            "name": "t3_x",
+            "id": "x",
+            "author": "alice",
+            "author_fullname": "t2_a",
+            "title": "Hello",
+            "selftext": "body text",
+            "permalink": "/r/py/comments/x/hello/",
+            "subreddit": "py",
+            "subreddit_name_prefixed": "r/py",
+            "num_comments": 3,
+            "score": 42,
+            "upvote_ratio": 0.97,
+            "over_18": False,
+            "is_self": True,
+            "created_utc": 1_700_000_000,
+            "thumbnail": "self",
+        },
+    }
+    item = parse_post(thing)
+    assert item["dataType"] == "post"
+    assert item["id"] == "t3_x"
+    assert item["parsedId"] == "x"
+    assert item["url"] == "https://www.reddit.com/r/py/comments/x/hello/"
+    assert item["upVotes"] == 42
+    assert item["numberOfComments"] == 3
+    assert item["thumbnailUrl"] is None  # 'self' sentinel is not a URL
+    assert item["createdAt"] == "2023-11-14T22:13:20.000Z"
+
+
+def test_parse_comment_strips_link_prefix():
+    thing = {
+        "kind": "t1",
+        "data": {
+            "name": "t1_c",
+            "id": "c",
+            "body": "a comment",
+            "link_id": "t3_x",
+            "parent_id": "t3_x",
+            "created_utc": 1_700_000_000,
+        },
+    }
+    item = parse_comment(thing)
+    assert item["dataType"] == "comment"
+    assert item["postId"] == "x"  # t3_ prefix stripped
+    assert item["parentId"] == "t3_x"
+
+
+def test_flatten_comments_counts_replies_and_stops_at_more():
+    tree = [
+        {
+            "kind": "t1",
+            "data": {
+                "name": "t1_1",
+                "id": "1",
+                "body": "top",
+                "created_utc": 1,
+                "replies": {
+                    "kind": "Listing",
+                    "data": {
+                        "children": [
+                            {"kind": "t1", "data": {"name": "t1_2", "id": "2",
+                                                    "body": "reply", "replies": ""}},
+                            {"kind": "more", "data": {}},  # stub -> ignored
+                        ]
+                    },
+                },
+            },
+        }
+    ]
+    flat = flatten_comments(tree, max_comments=10)
+    assert len(flat) == 2  # the 'more' stub is skipped
+    assert flat[0]["numberOfReplies"] == 1
+    assert [c["depth"] for c in flat] == [0, 1]
+
+
+def test_flatten_comments_honors_max():
+    tree = [
+        {"kind": "t1", "data": {"name": f"t1_{i}", "id": str(i), "body": "x",
+                                "replies": ""}}
+        for i in range(5)
+    ]
+    assert len(flatten_comments(tree, max_comments=2)) == 2
+
+
+def test_children_and_after():
+    listing = {"kind": "Listing", "data": {"children": [1, 2, 3], "after": "t3_next"}}
+    assert children(listing) == [1, 2, 3]
+    assert after(listing) == "t3_next"
+    assert children({}) == []
+    assert after({"data": {"after": None}}) is None
+
+
+def test_parse_community_maps_members():
+    thing = {"kind": "t5", "data": {"name": "t5_s", "id": "s",
+                                    "display_name": "py", "display_name_prefixed": "r/py",
+                                    "subscribers": 1234, "url": "/r/py/"}}
+    item = parse_community(thing)
+    assert item["dataType"] == "community"
+    assert item["numberOfMembers"] == 1234
+    assert item["url"] == "https://www.reddit.com/r/py/"
+
+
+# --- fixture-pinned (skips until e2e captures real .json) ------------------
+
+
+def _load(name: str):
+    path = _FIXTURE_DIR / name
+    if not path.exists():
+        pytest.skip(f"fixture {name} not captured yet (run e2e_reddit_scraper.py)")
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def test_parse_captured_post_fixture_if_present():
+    data = _load("sample_post.json")
+    # sample_post.json is the [postListing, commentsListing] .json shape.
+    post_children = children(data[0]) if isinstance(data, list) else []
+    assert post_children, "captured post fixture has no post child"
+    item = parse_post(post_children[0])
+    assert item["dataType"] == "post"
+    assert item["id"]
+
+
+def test_parse_captured_listing_fixture_if_present():
+    listing = _load("sample_listing.json")
+    kids = children(listing)
+    assert kids, "captured listing fixture is empty"
+    posts = [parse_post(c) for c in kids if c.get("kind") == "t3"]
+    assert posts, "no t3 posts in captured listing"

surfsense_backend/tests/unit/scrapers/reddit/test_skeleton.py

@@ -0,0 +1,73 @@
+"""Offline schema + URL-resolver tests for the Reddit scraper.
+
+Deterministic (no network, no live Reddit shapes): asserts the anonymous-only
+input surface, the flat item serialization contract, and URL classification.
+"""
+
+from __future__ import annotations
+
+from app.proprietary.scrapers.reddit.schemas import RedditItem, RedditScrapeInput
+from app.proprietary.scrapers.reddit.url_resolver import resolve_url
+
+
+def test_input_has_no_auth_fields():
+    # Anonymous-only: no auth-shaped field may exist on the input surface.
+    forbidden = {"username", "password", "token", "login", "auth", "credentials"}
+    assert forbidden.isdisjoint(RedditScrapeInput.model_fields)
+
+
+def test_input_defaults():
+    model = RedditScrapeInput()
+    assert model.sort == "new"
+    assert model.includeNSFW is True
+    assert model.maxItems == 10
+    assert model.startUrls == []
+    assert model.searches == []
+
+
+def test_input_allows_extra_inert_fields():
+    # extra="allow": unknown inert fields are accepted, not rejected.
+    model = RedditScrapeInput(debugMode=True, proxy={"provider": "custom"})
+    assert model.model_dump().get("debugMode") is True
+
+
+def test_item_to_output_keeps_none_keys():
+    out = RedditItem(dataType="post", id="t3_x").to_output()
+    assert out["dataType"] == "post"
+    assert out["id"] == "t3_x"
+    assert "numberOfComments" in out  # unsourced fields still present (None)
+    assert out["numberOfComments"] is None
+
+
+def test_resolve_post():
+    r = resolve_url("https://www.reddit.com/r/python/comments/abc123/some_title/")
+    assert r is not None
+    assert r.kind == "post"
+    assert r.value == "abc123"
+    assert r.subreddit == "python"
+
+
+def test_resolve_subreddit_with_and_without_sort():
+    bare = resolve_url("https://www.reddit.com/r/python")
+    assert bare is not None and bare.kind == "subreddit" and bare.sort is None
+    sorted_ = resolve_url("https://www.reddit.com/r/python/top")
+    assert sorted_ is not None and sorted_.sort == "top"
+
+
+def test_resolve_user_tabs():
+    overview = resolve_url("https://www.reddit.com/user/spez")
+    assert overview is not None and overview.kind == "user" and overview.content is None
+    comments = resolve_url("https://www.reddit.com/u/spez/comments")
+    assert comments is not None and comments.content == "comments"
+
+
+def test_resolve_search_global_and_in_sub():
+    global_ = resolve_url("https://www.reddit.com/search/?q=hello")
+    assert global_ is not None and global_.kind == "search" and global_.value == "hello"
+    in_sub = resolve_url("https://www.reddit.com/r/python/search/?q=async")
+    assert in_sub is not None and in_sub.subreddit == "python"
+
+
+def test_resolve_rejects_non_reddit_host():
+    assert resolve_url("https://example.com/r/python") is None
+    assert resolve_url("https://notreddit.com/user/x") is None