chore: fix docs

This commit is contained in:
Anish Sarkar 2026-07-04 17:45:43 +05:30
parent 0a83bf6192
commit 82141d8047
7 changed files with 16 additions and 17 deletions

View file

@ -1,6 +1,6 @@
# Reddit scraper (anonymous, no browser)
Platform-native Reddit scraper, Apify "Reddit Scraper"-compatible. Standalone
Platform-native Reddit scraper (anonymous, no browser). Standalone
module: it depends only on `app.utils.proxy` + `scrapling` and exposes a stable
public API. It is **not** wired into routes, `connector_service.py`, ingestion,
or Celery — integration later is a thin `reddit_routes.py` + one `include_router`
@ -38,7 +38,7 @@ rotation surfaces as `RedditAccessBlockedError` (mirrors google_maps'
| File | Responsibility |
|---|---|
| `__init__.py` | Public exports: `RedditScrapeInput`, `RedditItem`, `iter_reddit`, `scrape_reddit`, `RedditAccessBlockedError`. |
| `schemas.py` | Apify-shaped `RedditScrapeInput` (`extra="allow"`, no auth fields) + single flat `RedditItem` keyed by `dataType` + `StartUrl`. |
| `schemas.py` | `RedditScrapeInput` (`extra="allow"`, no auth fields) + single flat `RedditItem` keyed by `dataType` + `StartUrl`. |
| `fetch.py` | The core. Rotate-on-block sticky `_RotatingSession` + `_current_session` ContextVar + `warm_session` (loid) + `fetch_json`. No browser imports. |
| `url_resolver.py` | Classify a Reddit URL → `post`/`subreddit`/`user`/`search`; non-Reddit → `None`. |
| `parsers.py` | Pure JSON→item mapping (`parse_post`, `parse_comment`, `parse_community`, `flatten_comments`, `children`/`after`). I/O-free. |

View file

@ -1,4 +1,4 @@
"""Platform-native Reddit scraper (Apify Reddit Scraper-compatible, anonymous)."""
"""Platform-native Reddit scraper (anonymous, no browser)."""
from .fetch import RedditAccessBlockedError
from .schemas import RedditItem, RedditScrapeInput

View file

@ -96,7 +96,7 @@ _LOID_COOKIE = "loid"
def now_iso() -> str:
"""UTC timestamp in the millisecond ISO shape the Apify actor stamps."""
"""UTC timestamp in the millisecond ISO shape used by scraper output."""
return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"

View file

@ -1,9 +1,8 @@
# ruff: noqa: N815 - field names intentionally mirror the Apify camelCase spec
"""Apify-compatible input/output models for the Reddit scraper.
# ruff: noqa: N815 - field names intentionally use the public camelCase API
"""Input/output models for the Reddit scraper.
The models mirror the public Apify "Reddit Scraper" actor spec so the endpoint
can be a drop-in. The MVP populates a reliably-sourced subset; every other field
is accepted (input) or emitted as ``None``/``[]`` (output) so parity is additive.
The MVP populates a reliably-sourced subset of fields; every other output field
is emitted as ``None``/``[]`` so the contract can expand additively.
**Anonymous only.** There is deliberately **no** authentication field on the
input (no username/password/token/``login*``) the scraper holds only Reddit's
@ -23,7 +22,7 @@ RedditDataType = Literal["post", "comment", "community", "user"]
class StartUrl(BaseModel):
"""A single direct URL entry (Apify passes ``{"url": ...}`` objects)."""
"""A single direct URL entry (``{"url": ...}``; extra keys ignored)."""
model_config = ConfigDict(extra="allow")
@ -31,7 +30,7 @@ class StartUrl(BaseModel):
class RedditScrapeInput(BaseModel):
"""Full Apify "Reddit Scraper" input surface (anonymous, no auth fields).
"""Reddit scraper input surface (anonymous, no auth fields).
Caps (``maxItems``/``maxPostCount``/...) are collector policy applied by
:func:`scrape_reddit`, NOT ceilings baked into the streaming flows. Fields
@ -71,7 +70,7 @@ class RedditItem(BaseModel):
"""Single flat output item, keyed by ``dataType``.
One model for post/comment/community/user (union of fields, unsourced
default ``None``/``[]``), matching the Apify actor's single-dataset shape.
default ``None``/``[]``), using a single flat-dataset shape.
``extra="allow"`` keeps the contract open so added fields never break
consumers.
"""
@ -123,5 +122,5 @@ class RedditItem(BaseModel):
scrapedAt: str | None = None
def to_output(self) -> dict[str, Any]:
"""Serialize to the flat dict shape Apify emits (keeps extras)."""
"""Serialize to the flat output dict shape (keeps extras)."""
return self.model_dump(exclude_none=False)

View file

@ -362,7 +362,7 @@ def _capped_targets(
async def iter_reddit(
input_model: RedditScrapeInput,
) -> AsyncIterator[dict[str, Any]]:
"""Yield Apify-shaped Reddit items. ``startUrls`` override ``searches``.
"""Yield flat Reddit items. ``startUrls`` override ``searches``.
Independent targets fan out concurrently; each target's ``after`` paging
stays sequential.

View file

@ -1,6 +1,6 @@
"""Classify a Reddit URL into a scrape job.
Covers the ``startUrls`` shapes the Apify spec accepts: a post (permalink), a
Covers the supported ``startUrls`` shapes: a post (permalink), a
subreddit listing, a user profile, and a search-results page. Non-Reddit hosts
resolve to ``None`` so the orchestrator can skip them. Mirrors the frozen
``ResolvedUrl`` dataclass shape of ``../youtube/url_resolver.py``, widened with

View file

@ -26,8 +26,8 @@ def test_input_defaults():
def test_input_allows_extra_inert_fields():
# extra="allow": Apify fields we don't act on are accepted, not rejected.
model = RedditScrapeInput(debugMode=True, proxy={"useApifyProxy": True})
# extra="allow": unknown inert fields are accepted, not rejected.
model = RedditScrapeInput(debugMode=True, proxy={"provider": "custom"})
assert model.model_dump().get("debugMode") is True