diff --git a/surfsense_backend/app/proprietary/scrapers/reddit/README.md b/surfsense_backend/app/proprietary/scrapers/reddit/README.md index 9f4cff5f2..58c601cf5 100644 --- a/surfsense_backend/app/proprietary/scrapers/reddit/README.md +++ b/surfsense_backend/app/proprietary/scrapers/reddit/README.md @@ -1,6 +1,6 @@ # Reddit scraper (anonymous, no browser) -Platform-native Reddit scraper, Apify "Reddit Scraper"-compatible. Standalone +Platform-native Reddit scraper (anonymous, no browser). Standalone module: it depends only on `app.utils.proxy` + `scrapling` and exposes a stable public API. It is **not** wired into routes, `connector_service.py`, ingestion, or Celery — integration later is a thin `reddit_routes.py` + one `include_router` @@ -38,7 +38,7 @@ rotation surfaces as `RedditAccessBlockedError` (mirrors google_maps' | File | Responsibility | |---|---| | `__init__.py` | Public exports: `RedditScrapeInput`, `RedditItem`, `iter_reddit`, `scrape_reddit`, `RedditAccessBlockedError`. | -| `schemas.py` | Apify-shaped `RedditScrapeInput` (`extra="allow"`, no auth fields) + single flat `RedditItem` keyed by `dataType` + `StartUrl`. | +| `schemas.py` | `RedditScrapeInput` (`extra="allow"`, no auth fields) + single flat `RedditItem` keyed by `dataType` + `StartUrl`. | | `fetch.py` | The core. Rotate-on-block sticky `_RotatingSession` + `_current_session` ContextVar + `warm_session` (loid) + `fetch_json`. No browser imports. | | `url_resolver.py` | Classify a Reddit URL → `post`/`subreddit`/`user`/`search`; non-Reddit → `None`. | | `parsers.py` | Pure JSON→item mapping (`parse_post`, `parse_comment`, `parse_community`, `flatten_comments`, `children`/`after`). I/O-free. | diff --git a/surfsense_backend/app/proprietary/scrapers/reddit/__init__.py b/surfsense_backend/app/proprietary/scrapers/reddit/__init__.py index 8b8c75f11..94a17ed75 100644 --- a/surfsense_backend/app/proprietary/scrapers/reddit/__init__.py +++ b/surfsense_backend/app/proprietary/scrapers/reddit/__init__.py @@ -1,4 +1,4 @@ -"""Platform-native Reddit scraper (Apify Reddit Scraper-compatible, anonymous).""" +"""Platform-native Reddit scraper (anonymous, no browser).""" from .fetch import RedditAccessBlockedError from .schemas import RedditItem, RedditScrapeInput diff --git a/surfsense_backend/app/proprietary/scrapers/reddit/fetch.py b/surfsense_backend/app/proprietary/scrapers/reddit/fetch.py index 02ed3ac88..ce45795e7 100644 --- a/surfsense_backend/app/proprietary/scrapers/reddit/fetch.py +++ b/surfsense_backend/app/proprietary/scrapers/reddit/fetch.py @@ -96,7 +96,7 @@ _LOID_COOKIE = "loid" def now_iso() -> str: - """UTC timestamp in the millisecond ISO shape the Apify actor stamps.""" + """UTC timestamp in the millisecond ISO shape used by scraper output.""" return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" diff --git a/surfsense_backend/app/proprietary/scrapers/reddit/schemas.py b/surfsense_backend/app/proprietary/scrapers/reddit/schemas.py index 28d322936..74560ae97 100644 --- a/surfsense_backend/app/proprietary/scrapers/reddit/schemas.py +++ b/surfsense_backend/app/proprietary/scrapers/reddit/schemas.py @@ -1,9 +1,8 @@ -# ruff: noqa: N815 - field names intentionally mirror the Apify camelCase spec -"""Apify-compatible input/output models for the Reddit scraper. +# ruff: noqa: N815 - field names intentionally use the public camelCase API +"""Input/output models for the Reddit scraper. -The models mirror the public Apify "Reddit Scraper" actor spec so the endpoint -can be a drop-in. The MVP populates a reliably-sourced subset; every other field -is accepted (input) or emitted as ``None``/``[]`` (output) so parity is additive. +The MVP populates a reliably-sourced subset of fields; every other output field +is emitted as ``None``/``[]`` so the contract can expand additively. **Anonymous only.** There is deliberately **no** authentication field on the input (no username/password/token/``login*``) — the scraper holds only Reddit's @@ -23,7 +22,7 @@ RedditDataType = Literal["post", "comment", "community", "user"] class StartUrl(BaseModel): - """A single direct URL entry (Apify passes ``{"url": ...}`` objects).""" + """A single direct URL entry (``{"url": ...}``; extra keys ignored).""" model_config = ConfigDict(extra="allow") @@ -31,7 +30,7 @@ class StartUrl(BaseModel): class RedditScrapeInput(BaseModel): - """Full Apify "Reddit Scraper" input surface (anonymous, no auth fields). + """Reddit scraper input surface (anonymous, no auth fields). Caps (``maxItems``/``maxPostCount``/...) are collector policy applied by :func:`scrape_reddit`, NOT ceilings baked into the streaming flows. Fields @@ -71,7 +70,7 @@ class RedditItem(BaseModel): """Single flat output item, keyed by ``dataType``. One model for post/comment/community/user (union of fields, unsourced - default ``None``/``[]``), matching the Apify actor's single-dataset shape. + default ``None``/``[]``), using a single flat-dataset shape. ``extra="allow"`` keeps the contract open so added fields never break consumers. """ @@ -123,5 +122,5 @@ class RedditItem(BaseModel): scrapedAt: str | None = None def to_output(self) -> dict[str, Any]: - """Serialize to the flat dict shape Apify emits (keeps extras).""" + """Serialize to the flat output dict shape (keeps extras).""" return self.model_dump(exclude_none=False) diff --git a/surfsense_backend/app/proprietary/scrapers/reddit/scraper.py b/surfsense_backend/app/proprietary/scrapers/reddit/scraper.py index 5fc4e07e9..e97cda024 100644 --- a/surfsense_backend/app/proprietary/scrapers/reddit/scraper.py +++ b/surfsense_backend/app/proprietary/scrapers/reddit/scraper.py @@ -362,7 +362,7 @@ def _capped_targets( async def iter_reddit( input_model: RedditScrapeInput, ) -> AsyncIterator[dict[str, Any]]: - """Yield Apify-shaped Reddit items. ``startUrls`` override ``searches``. + """Yield flat Reddit items. ``startUrls`` override ``searches``. Independent targets fan out concurrently; each target's ``after`` paging stays sequential. diff --git a/surfsense_backend/app/proprietary/scrapers/reddit/url_resolver.py b/surfsense_backend/app/proprietary/scrapers/reddit/url_resolver.py index 3b6d4e442..0af80b524 100644 --- a/surfsense_backend/app/proprietary/scrapers/reddit/url_resolver.py +++ b/surfsense_backend/app/proprietary/scrapers/reddit/url_resolver.py @@ -1,6 +1,6 @@ """Classify a Reddit URL into a scrape job. -Covers the ``startUrls`` shapes the Apify spec accepts: a post (permalink), a +Covers the supported ``startUrls`` shapes: a post (permalink), a subreddit listing, a user profile, and a search-results page. Non-Reddit hosts resolve to ``None`` so the orchestrator can skip them. Mirrors the frozen ``ResolvedUrl`` dataclass shape of ``../youtube/url_resolver.py``, widened with diff --git a/surfsense_backend/tests/unit/scrapers/reddit/test_skeleton.py b/surfsense_backend/tests/unit/scrapers/reddit/test_skeleton.py index 7c6829540..696944b0a 100644 --- a/surfsense_backend/tests/unit/scrapers/reddit/test_skeleton.py +++ b/surfsense_backend/tests/unit/scrapers/reddit/test_skeleton.py @@ -26,8 +26,8 @@ def test_input_defaults(): def test_input_allows_extra_inert_fields(): - # extra="allow": Apify fields we don't act on are accepted, not rejected. - model = RedditScrapeInput(debugMode=True, proxy={"useApifyProxy": True}) + # extra="allow": unknown inert fields are accepted, not rejected. + model = RedditScrapeInput(debugMode=True, proxy={"provider": "custom"}) assert model.model_dump().get("debugMode") is True