mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-08 22:22:17 +02:00
chore: fix docs
This commit is contained in:
parent
0a83bf6192
commit
82141d8047
7 changed files with 16 additions and 17 deletions
|
|
@ -1,6 +1,6 @@
|
|||
# Reddit scraper (anonymous, no browser)
|
||||
|
||||
Platform-native Reddit scraper, Apify "Reddit Scraper"-compatible. Standalone
|
||||
Platform-native Reddit scraper (anonymous, no browser). Standalone
|
||||
module: it depends only on `app.utils.proxy` + `scrapling` and exposes a stable
|
||||
public API. It is **not** wired into routes, `connector_service.py`, ingestion,
|
||||
or Celery — integration later is a thin `reddit_routes.py` + one `include_router`
|
||||
|
|
@ -38,7 +38,7 @@ rotation surfaces as `RedditAccessBlockedError` (mirrors google_maps'
|
|||
| File | Responsibility |
|
||||
|---|---|
|
||||
| `__init__.py` | Public exports: `RedditScrapeInput`, `RedditItem`, `iter_reddit`, `scrape_reddit`, `RedditAccessBlockedError`. |
|
||||
| `schemas.py` | Apify-shaped `RedditScrapeInput` (`extra="allow"`, no auth fields) + single flat `RedditItem` keyed by `dataType` + `StartUrl`. |
|
||||
| `schemas.py` | `RedditScrapeInput` (`extra="allow"`, no auth fields) + single flat `RedditItem` keyed by `dataType` + `StartUrl`. |
|
||||
| `fetch.py` | The core. Rotate-on-block sticky `_RotatingSession` + `_current_session` ContextVar + `warm_session` (loid) + `fetch_json`. No browser imports. |
|
||||
| `url_resolver.py` | Classify a Reddit URL → `post`/`subreddit`/`user`/`search`; non-Reddit → `None`. |
|
||||
| `parsers.py` | Pure JSON→item mapping (`parse_post`, `parse_comment`, `parse_community`, `flatten_comments`, `children`/`after`). I/O-free. |
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
"""Platform-native Reddit scraper (Apify Reddit Scraper-compatible, anonymous)."""
|
||||
"""Platform-native Reddit scraper (anonymous, no browser)."""
|
||||
|
||||
from .fetch import RedditAccessBlockedError
|
||||
from .schemas import RedditItem, RedditScrapeInput
|
||||
|
|
|
|||
|
|
@ -96,7 +96,7 @@ _LOID_COOKIE = "loid"
|
|||
|
||||
|
||||
def now_iso() -> str:
|
||||
"""UTC timestamp in the millisecond ISO shape the Apify actor stamps."""
|
||||
"""UTC timestamp in the millisecond ISO shape used by scraper output."""
|
||||
return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,8 @@
|
|||
# ruff: noqa: N815 - field names intentionally mirror the Apify camelCase spec
|
||||
"""Apify-compatible input/output models for the Reddit scraper.
|
||||
# ruff: noqa: N815 - field names intentionally use the public camelCase API
|
||||
"""Input/output models for the Reddit scraper.
|
||||
|
||||
The models mirror the public Apify "Reddit Scraper" actor spec so the endpoint
|
||||
can be a drop-in. The MVP populates a reliably-sourced subset; every other field
|
||||
is accepted (input) or emitted as ``None``/``[]`` (output) so parity is additive.
|
||||
The MVP populates a reliably-sourced subset of fields; every other output field
|
||||
is emitted as ``None``/``[]`` so the contract can expand additively.
|
||||
|
||||
**Anonymous only.** There is deliberately **no** authentication field on the
|
||||
input (no username/password/token/``login*``) — the scraper holds only Reddit's
|
||||
|
|
@ -23,7 +22,7 @@ RedditDataType = Literal["post", "comment", "community", "user"]
|
|||
|
||||
|
||||
class StartUrl(BaseModel):
|
||||
"""A single direct URL entry (Apify passes ``{"url": ...}`` objects)."""
|
||||
"""A single direct URL entry (``{"url": ...}``; extra keys ignored)."""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
|
|
@ -31,7 +30,7 @@ class StartUrl(BaseModel):
|
|||
|
||||
|
||||
class RedditScrapeInput(BaseModel):
|
||||
"""Full Apify "Reddit Scraper" input surface (anonymous, no auth fields).
|
||||
"""Reddit scraper input surface (anonymous, no auth fields).
|
||||
|
||||
Caps (``maxItems``/``maxPostCount``/...) are collector policy applied by
|
||||
:func:`scrape_reddit`, NOT ceilings baked into the streaming flows. Fields
|
||||
|
|
@ -71,7 +70,7 @@ class RedditItem(BaseModel):
|
|||
"""Single flat output item, keyed by ``dataType``.
|
||||
|
||||
One model for post/comment/community/user (union of fields, unsourced
|
||||
default ``None``/``[]``), matching the Apify actor's single-dataset shape.
|
||||
default ``None``/``[]``), using a single flat-dataset shape.
|
||||
``extra="allow"`` keeps the contract open so added fields never break
|
||||
consumers.
|
||||
"""
|
||||
|
|
@ -123,5 +122,5 @@ class RedditItem(BaseModel):
|
|||
scrapedAt: str | None = None
|
||||
|
||||
def to_output(self) -> dict[str, Any]:
|
||||
"""Serialize to the flat dict shape Apify emits (keeps extras)."""
|
||||
"""Serialize to the flat output dict shape (keeps extras)."""
|
||||
return self.model_dump(exclude_none=False)
|
||||
|
|
|
|||
|
|
@ -362,7 +362,7 @@ def _capped_targets(
|
|||
async def iter_reddit(
|
||||
input_model: RedditScrapeInput,
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
"""Yield Apify-shaped Reddit items. ``startUrls`` override ``searches``.
|
||||
"""Yield flat Reddit items. ``startUrls`` override ``searches``.
|
||||
|
||||
Independent targets fan out concurrently; each target's ``after`` paging
|
||||
stays sequential.
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
"""Classify a Reddit URL into a scrape job.
|
||||
|
||||
Covers the ``startUrls`` shapes the Apify spec accepts: a post (permalink), a
|
||||
Covers the supported ``startUrls`` shapes: a post (permalink), a
|
||||
subreddit listing, a user profile, and a search-results page. Non-Reddit hosts
|
||||
resolve to ``None`` so the orchestrator can skip them. Mirrors the frozen
|
||||
``ResolvedUrl`` dataclass shape of ``../youtube/url_resolver.py``, widened with
|
||||
|
|
|
|||
|
|
@ -26,8 +26,8 @@ def test_input_defaults():
|
|||
|
||||
|
||||
def test_input_allows_extra_inert_fields():
|
||||
# extra="allow": Apify fields we don't act on are accepted, not rejected.
|
||||
model = RedditScrapeInput(debugMode=True, proxy={"useApifyProxy": True})
|
||||
# extra="allow": unknown inert fields are accepted, not rejected.
|
||||
model = RedditScrapeInput(debugMode=True, proxy={"provider": "custom"})
|
||||
assert model.model_dump().get("debugMode") is True
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue