mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-08 22:22:17 +02:00
feat(native-connector): added youtube scrapers
This commit is contained in:
DESKTOP-RTLN3BA\$punk
parent
f964e59686
commit
99378d7b63
19 changed files with 3959 additions and 0 deletions
|
|
@ -9,6 +9,12 @@ from fastapi import APIRouter, Depends, HTTPException, Query
|
|||
from scrapling.fetchers import AsyncFetcher
|
||||
|
||||
from app.auth.context import AuthContext
|
||||
from app.scrapers.youtube import (
|
||||
YouTubeCommentsInput,
|
||||
YouTubeScrapeInput,
|
||||
scrape_comments,
|
||||
scrape_youtube,
|
||||
)
|
||||
from app.users import require_session_context
|
||||
from app.utils.proxy import get_proxy_url
|
||||
|
||||
|
|
@ -26,6 +32,45 @@ _INNERTUBE_CLIENT = {
|
|||
}
|
||||
|
||||
|
||||
@router.post("/youtube/scrape")
|
||||
async def scrape_youtube_route(
|
||||
payload: YouTubeScrapeInput,
|
||||
_auth: AuthContext = Depends(require_session_context),
|
||||
) -> list[dict]:
|
||||
"""Scrape public YouTube data (search / video / channel / playlist / shorts).
|
||||
|
||||
Apify YouTube Scraper-compatible input/output. The scrape runs inline and is
|
||||
bounded only by the request's own ``maxResults`` / ``maxResultsShorts`` /
|
||||
``maxResultStreams`` — no separate per-request ceiling.
|
||||
"""
|
||||
try:
|
||||
return await scrape_youtube(payload)
|
||||
except Exception as e:
|
||||
logger.error("YouTube scrape failed: %s", e)
|
||||
raise HTTPException(
|
||||
status_code=502, detail=f"YouTube scrape failed: {e!s}"
|
||||
) from e
|
||||
|
||||
|
||||
@router.post("/youtube/comments")
|
||||
async def scrape_comments_route(
|
||||
payload: YouTubeCommentsInput,
|
||||
_auth: AuthContext = Depends(require_session_context),
|
||||
) -> list[dict]:
|
||||
"""Scrape YouTube comments (+ replies) for the given video URLs.
|
||||
|
||||
Apify "YouTube Comments Scraper"-compatible. Runs inline and is bounded only
|
||||
by the request's own ``maxComments`` — no separate per-request ceiling.
|
||||
"""
|
||||
try:
|
||||
return await scrape_comments(payload)
|
||||
except Exception as e:
|
||||
logger.error("YouTube comments scrape failed: %s", e)
|
||||
raise HTTPException(
|
||||
status_code=502, detail=f"YouTube comments scrape failed: {e!s}"
|
||||
) from e
|
||||
|
||||
|
||||
@router.get("/youtube/playlist-videos")
|
||||
async def get_playlist_videos(
|
||||
url: str = Query(..., description="YouTube playlist URL"),
|
||||
|
|
|
|||
1
surfsense_backend/app/scrapers/__init__.py
Normal file
1
surfsense_backend/app/scrapers/__init__.py
Normal file
|
|
@ -0,0 +1 @@
|
|||
"""Platform-native scrapers (non-connector data extraction tools)."""
|
||||
287
surfsense_backend/app/scrapers/youtube/README.md
Normal file
287
surfsense_backend/app/scrapers/youtube/README.md
Normal file
|
|
@ -0,0 +1,287 @@
|
|||
# YouTube Scraper
|
||||
|
||||
A platform-native YouTube scraper that is a **drop-in clone of the Apify
|
||||
"YouTube Scraper" and "YouTube Comments Scraper" actors** — same input surface,
|
||||
same output item shape. It talks to YouTube's internal **InnerTube** API plus
|
||||
the public watch/channel HTML, egresses through a residential proxy, and streams
|
||||
Apify-shaped dicts.
|
||||
|
||||
No API keys, no Apify account, no headless browser on the happy path.
|
||||
|
||||
---
|
||||
|
||||
## Quick start
|
||||
|
||||
```python
|
||||
from app.scrapers.youtube import (
|
||||
YouTubeScrapeInput, scrape_youtube,
|
||||
YouTubeCommentsInput, scrape_comments,
|
||||
)
|
||||
|
||||
# Videos — by search query and/or direct URLs (video/channel/playlist/hashtag/search)
|
||||
videos = await scrape_youtube(
|
||||
YouTubeScrapeInput(searchQueries=["surfsense"], maxResults=50)
|
||||
)
|
||||
videos = await scrape_youtube(
|
||||
YouTubeScrapeInput(startUrls=[{"url": "https://www.youtube.com/@SomeChannel"}],
|
||||
maxResults=20, downloadSubtitles=True)
|
||||
)
|
||||
|
||||
# Comments — one output item per top-level comment AND per reply
|
||||
comments = await scrape_comments(
|
||||
YouTubeCommentsInput(
|
||||
startUrls=[{"url": "https://www.youtube.com/watch?v=VIDEO_ID"}],
|
||||
maxComments=200, sortCommentsBy="TOP_COMMENTS",
|
||||
)
|
||||
)
|
||||
```
|
||||
|
||||
Both have a streaming twin — `iter_youtube()` / `iter_comments()` — that yields
|
||||
items as they arrive (unbounded, continuation-paged). `scrape_*` is just a
|
||||
collector with an optional `limit` guard.
|
||||
|
||||
The HTTP surface lives in `app/routes/youtube_routes.py`.
|
||||
|
||||
---
|
||||
|
||||
## Module map
|
||||
|
||||
| File | Responsibility |
|
||||
|------|----------------|
|
||||
| `__init__.py` | Public exports (entry points + schemas). |
|
||||
| `schemas.py` | Pydantic input/output models mirroring the Apify camelCase spec. `extra="allow"` on outputs keeps the contract open. |
|
||||
| `scraper.py` | Video orchestrator. Resolves URLs → per-flow async generators (`_video_flow`, `_search_flow`, `_channel_flow`, `_playlist_flow`), runs them through the `fan_out` worker pool. |
|
||||
| `comments.py` | Comments orchestrator. Watch page → comments-section continuation → `/next` paging, with concurrent per-thread reply fetching. |
|
||||
| `innertube.py` | **The network seam.** Proxy-only fetch (`fetch_html`, `post_innertube`), reusable sticky-IP sessions, reactive IP rotation, `StealthyFetcher` fallback, and the InnerTube payload builder. |
|
||||
| `parsers.py` | Pure, I/O-free JSON/HTML traversal + normalization (`find_all`/`find_first`/`dig`, `parse_video_page`, `parse_search_response`, comment/continuation token extractors, `parse_count`, …). |
|
||||
| `url_resolver.py` | Classify a URL into `video` / `channel` / `playlist` / `hashtag` / `search` and extract its id. |
|
||||
| `search_filters.py` | Encode Apify search filters into YouTube's `sp=` base64 protobuf (sort/date/type/length/feature flags), composable. |
|
||||
| `subtitles.py` | Subtitle download via `youtube-transcript-api`, shaped to Apify `subtitles[]`. |
|
||||
|
||||
Everything in `parsers.py` is deterministic and unit-tested offline; everything
|
||||
that touches the network is funneled through `innertube.py`.
|
||||
|
||||
---
|
||||
|
||||
## How it fetches (the important part)
|
||||
|
||||
All network I/O goes through **`fetch_html`** (GET watch/channel pages) and
|
||||
**`post_innertube`** (POST InnerTube `browse`/`search`/`next`). Design rules:
|
||||
|
||||
1. **Proxy-only egress.** Every request goes through the residential proxy
|
||||
(`app/utils/proxy.get_proxy_url`). We never connect directly — a direct hit
|
||||
would expose and risk-block the server IP.
|
||||
2. **Session reuse = sticky IP.** Within one flow (a continuation chain, or the
|
||||
jobs a worker pulls), a single keep-alive `FetcherSession` is reused. This
|
||||
roughly **halves warm latency** (~2.1s → ~1.0s) because only the first
|
||||
request pays the TCP+TLS handshake, and it pins one sticky exit IP instead of
|
||||
drawing a new (often slow) residential node per request.
|
||||
3. **Reactive IP rotation.** A sticky IP is kept until it's actually blocked. On
|
||||
`403`/`429` or a connection error, the session rotates to a fresh IP and
|
||||
retries, up to `_MAX_ROTATIONS` (3). A probe of 120 sequential requests on
|
||||
one IP saw zero blocks, so rotation is reactive, not proactive.
|
||||
4. **Browser fallback.** If all proxy attempts fail on an HTML page, `fetch_html`
|
||||
falls back to `StealthyFetcher` (headless, `solve_cloudflare=True`) in a
|
||||
worker thread. Optional — needs patchright browsers installed. Age-gated
|
||||
content requires login and is **not** bypassable.
|
||||
|
||||
The active session is bound to the current async task via a `ContextVar`
|
||||
(`_current_session`), so parsers and orchestrators never thread a session
|
||||
argument through every call — each concurrent flow transparently uses its own
|
||||
session/IP.
|
||||
|
||||
### InnerTube payloads
|
||||
|
||||
`build_innertube_payload(...)` builds the `WEB` client `context` payload
|
||||
(I/O-free, unit-testable). Some endpoints reject a keyless POST; `scraper._post`
|
||||
retries once with the public web key (`INNERTUBE_PUBLIC_API_KEY`) when the
|
||||
keyless call returns nothing. `hl=<lang>` on a `/next` call returns the
|
||||
creator-localized title/description (the translation flow).
|
||||
|
||||
---
|
||||
|
||||
## Concurrency model
|
||||
|
||||
Independent jobs — each `startUrl`, each `searchQuery`, each comment video — run
|
||||
concurrently through **`fan_out`**, a warm **worker pool** (`_FANOUT_CONCURRENCY
|
||||
= 16`):
|
||||
|
||||
- Each worker opens **one** proxy session and reuses it across the sequential
|
||||
jobs it pulls, so only the first job per worker pays the handshake.
|
||||
- **A bad job yields nothing rather than aborting the batch** (per-job
|
||||
try/except). One dead URL / comments-disabled video never kills the run.
|
||||
- Results stream out as each job finishes; **within** a flow, continuation
|
||||
paging stays sequential.
|
||||
- If the consumer stops early (collector hits its `limit`), workers are
|
||||
cancelled and **awaited** so every session's `finally` closes — no leaked
|
||||
keep-alive connections.
|
||||
|
||||
Comment reply threads for a page are fetched **concurrently** on the same
|
||||
multiplexed session (`asyncio.gather`), capped at the remaining budget.
|
||||
|
||||
---
|
||||
|
||||
## Data flow
|
||||
|
||||
- **Video by URL** → fetch watch HTML → `parse_video_page` (reads
|
||||
`ytInitialData` + `ytInitialPlayerResponse`) → optional subtitles + translation.
|
||||
- **Search** → InnerTube `/search` (+ `sp=` filter protobuf) → paginate via
|
||||
continuation tokens up to `maxResults`.
|
||||
- **Channel** → fetch the videos-tab seed once (reused for channel-wide metadata
|
||||
+ the About panel via `/browse`), then page `videos` / `shorts` / `streams`
|
||||
tabs, each capped independently (`maxResults` / `maxResultsShorts` /
|
||||
`maxResultStreams`). `sortVideosBy` uses the sort chips; `oldestPostDate` cuts
|
||||
off newest-first.
|
||||
- **Playlist** → `/browse` `VL<id>` → resolve each video via the video flow.
|
||||
- **Comments** → watch HTML seeds the comments-section token → `/next` returns
|
||||
comment entities + per-thread reply tokens + the page token. `maxComments`
|
||||
counts **every** emitted item (comments + replies).
|
||||
|
||||
### `commentsCount`
|
||||
|
||||
For the **comments** scraper, the authoritative total is read from the
|
||||
comments-section header (`commentsHeaderRenderer.countText`), not the watch-page
|
||||
HTML where it's lazy-loaded/absent. **Known gap:** the **video** scraper's
|
||||
`VideoItem.commentsCount` still comes from search/watch HTML and is often `null`
|
||||
— it would need an extra `/next` call to backfill (intentionally not done to
|
||||
keep the video path cheap).
|
||||
|
||||
---
|
||||
|
||||
## API spec
|
||||
|
||||
Mirrors the Apify "YouTube Scraper" and "YouTube Comments Scraper" actors
|
||||
(camelCase, `extra="allow"`). Inputs use Pydantic defaults; **every field is
|
||||
additive** — unknown inputs are accepted, unsourced outputs come back as
|
||||
`None`/`[]` — so parity grows without breaking consumers. `schemas.py` is the
|
||||
source of truth.
|
||||
|
||||
### Video scraper — input (`YouTubeScrapeInput`)
|
||||
|
||||
| Field | Type / values | Default | Notes |
|
||||
|-------|---------------|---------|-------|
|
||||
| `searchQueries` | `string[]` | `[]` | Discovery by query. Ignored when `startUrls` is set. |
|
||||
| `startUrls` | `[{ "url": string }]` | `[]` | Direct URLs: video, channel, playlist, hashtag, search. Overrides `searchQueries`. |
|
||||
| `maxResults` | `int ≥ 0` | `0` | Cap of regular videos **per query and per channel**. `0` = fetch none. |
|
||||
| `maxResultsShorts` | `int ≥ 0` | `0` | Cap of Shorts per channel. |
|
||||
| `maxResultStreams` | `int ≥ 0` | `0` | Cap of live/streams per channel. |
|
||||
| `downloadSubtitles` | `bool` | `false` | Populate `subtitles[]`. |
|
||||
| `subtitlesLanguage` | `string` | `"en"` | Also drives the translation flow when non-`en` (see `translatedTitle`). |
|
||||
| `subtitlesFormat` | `srt` \| `vtt` \| `xml` \| `plaintext` | `"srt"` | |
|
||||
| `preferAutoGeneratedSubtitles` | `bool` | `false` | |
|
||||
| `saveSubsToKVS` | `bool` | `false` | Accepted for parity; no-op (Apify key-value-store concept). |
|
||||
| `sortingOrder` | `relevance` \| `rating` \| `date` \| `views` | `null` | Search only. |
|
||||
| `dateFilter` | `hour` \| `today` \| `week` \| `month` \| `year` | `null` | Search only. |
|
||||
| `videoType` | `video` \| `movie` | `null` | Search only. |
|
||||
| `lengthFilter` | `under4` \| `between420` \| `plus20` | `null` | Search only (<4min / 4–20min / >20min). |
|
||||
| `isHD` `hasSubtitles` `hasCC` `is3D` `isLive` `isBought` `is4K` `is360` `hasLocation` `isHDR` `isVR180` | `bool` | `null` | Search feature filters (encoded into `sp=`). |
|
||||
| `oldestPostDate` | `string` (date) | `null` | Channel cutoff; day-accurate (relative times). |
|
||||
| `sortVideosBy` | `NEWEST` \| `POPULAR` \| `OLDEST` | `null` | Channel videos tab sort chip. |
|
||||
|
||||
### Video scraper — output (`VideoItem`)
|
||||
|
||||
| Field | Type | Populated? |
|
||||
|-------|------|-----------|
|
||||
| `title` `id` `url` `viewCount` `date` `duration` | str/int | yes |
|
||||
| `type` | `video` \| `shorts` \| `stream` | yes |
|
||||
| `thumbnailUrl` | str | yes |
|
||||
| `input` `fromYTUrl` `order` | str/int | yes (provenance: source query/URL, origin URL, index) |
|
||||
| `text` | str | yes (description) |
|
||||
| `descriptionLinks` | `[{ url, text }]` | yes |
|
||||
| `hashtags` | `string[]` | yes |
|
||||
| `likes` `commentsCount` `commentsTurnedOff` | int/bool | partial (often `null` on the video path — see `commentsCount` note) |
|
||||
| `location` | str | when present |
|
||||
| `collaborators` | `[{ name, username, url }]` | when present |
|
||||
| `translatedTitle` `translatedText` | str | when `subtitlesLanguage != "en"` |
|
||||
| `subtitles` | `[{ srtUrl, type, language, srt }]` | when `downloadSubtitles` |
|
||||
| `isMembersOnly` `isPaidContent` | bool | yes (default `false`) |
|
||||
| `isMonetized` `isAgeRestricted` | bool | best-effort (`null` when unknown) |
|
||||
| `channelName` `channelUrl` `channelUsername` `channelId` | str | yes |
|
||||
| `numberOfSubscribers` `channelTotalVideos` `channelTotalViews` | int | channel/deep fields |
|
||||
| `channelDescription` `channelLocation` `channelJoinedDate` | str | channel About panel |
|
||||
| `isChannelVerified` `channelBannerUrl` `channelAvatarUrl` | bool/str | channel fields |
|
||||
|
||||
### Comments scraper — input (`YouTubeCommentsInput`)
|
||||
|
||||
| Field | Type / values | Default | Notes |
|
||||
|-------|---------------|---------|-------|
|
||||
| `startUrls` | `[{ "url": string }]` | `[]` | Video URLs only (non-video URLs skipped). |
|
||||
| `maxComments` | `int ≥ 1` | `1` | Counts **every** emitted item (top-level comments **and** replies). |
|
||||
| `sortCommentsBy` | `TOP_COMMENTS` \| `NEWEST_FIRST` | `"NEWEST_FIRST"` | |
|
||||
| `oldestCommentDate` | `string` (date) | `null` | Forces newest-first and stops at the cutoff. |
|
||||
|
||||
### Comments scraper — output (`CommentItem`)
|
||||
|
||||
| Field | Type | Notes |
|
||||
|-------|------|-------|
|
||||
| `cid` | str | Comment id. |
|
||||
| `comment` | str | Text. |
|
||||
| `author` | str | |
|
||||
| `type` | `comment` \| `reply` | |
|
||||
| `replyToCid` | str | Parent `cid` (replies only). |
|
||||
| `replyCount` | int | Replies under a top-level comment. |
|
||||
| `voteCount` | int | Likes. |
|
||||
| `authorIsChannelOwner` `hasCreatorHeart` | bool | |
|
||||
| `publishedTimeText` | str | Relative time ("2 days ago"). |
|
||||
| `videoId` `pageUrl` `title` | str | Source video. |
|
||||
| `commentsCount` | int | Authoritative total from the comments header. |
|
||||
|
||||
---
|
||||
|
||||
## Configuration
|
||||
|
||||
- **Proxy** — required for real runs; configured via `app/utils/proxy.py`
|
||||
(residential rotating gateway env vars). With no proxy configured the fetchers
|
||||
fall back to one-shot direct `AsyncFetcher` calls (fine for local tests, not
|
||||
for production).
|
||||
- **Concurrency** — `scraper._FANOUT_CONCURRENCY` (16). The gateway handled 64
|
||||
parallel flows with zero failures in a ramp probe, so this leaves headroom.
|
||||
- **Rotation** — `innertube._BLOCK_STATUSES` (`403`, `429`) and
|
||||
`_MAX_ROTATIONS` (3).
|
||||
|
||||
---
|
||||
|
||||
## Testing
|
||||
|
||||
- **Offline unit tests** (no network) — run these on every change:
|
||||
```bash
|
||||
cd surfsense_backend
|
||||
.venv/Scripts/python.exe -m pytest tests/unit/scrapers/youtube/
|
||||
```
|
||||
- `test_parsers.py` — parser/normalization + filter-protobuf + URL-resolver
|
||||
cases against hand-built and (if present) captured real fixtures.
|
||||
- `test_fetch_resilience.py` — deterministic rotate-on-block (`429`/error →
|
||||
rotate → `200`, exhaustion, no-rotate on `404`, stealthy fallback) and the
|
||||
`fan_out` no-session-leak-on-early-stop guarantee, all with stubbed sessions.
|
||||
- **Live functional harness** — `scripts/e2e_youtube_scraper.py` (needs live
|
||||
network + optional proxy creds). Exercises video/search/channel/comments/
|
||||
location/collaborators/translation end to end, and **regenerates the offline
|
||||
fixtures** into `tests/unit/scrapers/youtube/fixtures/`:
|
||||
```bash
|
||||
.venv/Scripts/python.exe scripts/e2e_youtube_scraper.py
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Extending it
|
||||
|
||||
- **Add an output field** → populate it in the relevant `parsers.py` function
|
||||
and add it to `schemas.py`. Because outputs are `extra="allow"`, forgetting the
|
||||
schema line won't drop the value, but declaring it documents the contract.
|
||||
- **Add a URL kind** → extend `url_resolver.resolve_url` + add a `_*_flow` in
|
||||
`scraper.py` and a branch in `_dispatch`.
|
||||
- **Add a search filter** → add the field to `YouTubeScrapeInput` and encode it
|
||||
in `search_filters.build_search_params` (verify byte-for-byte against a real
|
||||
YouTube `sp=` token in the unit test).
|
||||
|
||||
### Known ceilings (grep `ponytail:` in the source for the live list)
|
||||
|
||||
- Hashtag pages are routed through search (`#tag` query), not a dedicated
|
||||
hashtag browse.
|
||||
- `oldestPostDate` / `oldestCommentDate` cutoffs are day-accurate at best
|
||||
(channel/list pages only expose coarse relative times like "2 years ago").
|
||||
- Keyless-vs-keyed InnerTube retry does one extra request on the keyed path
|
||||
instead of remembering which worked.
|
||||
- Video-path `commentsCount` (see above).
|
||||
- Playlist scraping reads only the first `/browse` page (~100 videos); paging
|
||||
the continuation token for longer playlists is not yet wired.
|
||||
16
surfsense_backend/app/scrapers/youtube/__init__.py
Normal file
16
surfsense_backend/app/scrapers/youtube/__init__.py
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
"""Platform-native YouTube scraper (Apify YouTube Scraper-compatible)."""
|
||||
|
||||
from .comments import iter_comments, scrape_comments
|
||||
from .schemas import CommentItem, VideoItem, YouTubeCommentsInput, YouTubeScrapeInput
|
||||
from .scraper import iter_youtube, scrape_youtube
|
||||
|
||||
__all__ = [
|
||||
"CommentItem",
|
||||
"VideoItem",
|
||||
"YouTubeCommentsInput",
|
||||
"YouTubeScrapeInput",
|
||||
"iter_comments",
|
||||
"iter_youtube",
|
||||
"scrape_comments",
|
||||
"scrape_youtube",
|
||||
]
|
||||
184
surfsense_backend/app/scrapers/youtube/comments.py
Normal file
184
surfsense_backend/app/scrapers/youtube/comments.py
Normal file
|
|
@ -0,0 +1,184 @@
|
|||
"""Orchestrator for the YouTube Comments scraper (Apify-compatible).
|
||||
|
||||
Distinct from the video scraper: one output item per top-level comment *and*
|
||||
per reply, sourced from the InnerTube ``/next`` endpoint. The watch page seeds a
|
||||
comments-section continuation; ``/next`` then returns comment entity payloads,
|
||||
per-thread reply tokens, and the paging token. ``maxComments`` counts every
|
||||
emitted item (comments + replies), matching Apify.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any
|
||||
|
||||
from .innertube import INNERTUBE_NEXT_URL, build_innertube_payload, fetch_html
|
||||
from .parsers import (
|
||||
comment_next_token,
|
||||
comment_reply_tokens,
|
||||
comment_section_token,
|
||||
comment_sort_tokens,
|
||||
dig,
|
||||
extract_yt_initial_data,
|
||||
find_first,
|
||||
parse_comment_entities,
|
||||
parse_count,
|
||||
parse_video_page,
|
||||
)
|
||||
from .scraper import _post, _published_date, fan_out
|
||||
from .schemas import CommentItem, YouTubeCommentsInput
|
||||
from .url_resolver import resolve_url
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Apify sort value -> YouTube sort-menu label. The section token loads "Top" by
|
||||
# default, so only "Newest" needs an explicit switch request.
|
||||
_SORT_LABELS = {"TOP_COMMENTS": "Top", "NEWEST_FIRST": "Newest"}
|
||||
|
||||
|
||||
async def _post_next(token: str) -> dict[str, Any] | None:
|
||||
return await _post(INNERTUBE_NEXT_URL, build_innertube_payload(continuation_token=token))
|
||||
|
||||
|
||||
def _finalize(partial: dict[str, Any], base: dict[str, Any], reply_to: str | None) -> dict:
|
||||
return CommentItem(**{**base, **partial, "replyToCid": reply_to}).to_output()
|
||||
|
||||
|
||||
def _older(entity: dict[str, Any], cutoff) -> bool:
|
||||
when = _published_date(entity.get("publishedTimeText"))
|
||||
return when is not None and when < cutoff
|
||||
|
||||
|
||||
async def _collect_replies(
|
||||
token: str, *, parent_cid: str, base: dict[str, Any], cutoff
|
||||
) -> list[dict]:
|
||||
"""Collect one thread's replies (``replyLevel`` 1) until exhausted."""
|
||||
replies: list[dict] = []
|
||||
while token:
|
||||
data = await _post_next(token)
|
||||
if not data:
|
||||
break
|
||||
for entity in parse_comment_entities(data):
|
||||
if cutoff and _older(entity, cutoff):
|
||||
continue
|
||||
replies.append(_finalize(entity, base, parent_cid))
|
||||
token = comment_next_token(data)
|
||||
return replies
|
||||
|
||||
|
||||
async def _comments_for_video(
|
||||
video_id: str, input_model: YouTubeCommentsInput
|
||||
) -> AsyncIterator[dict]:
|
||||
limit = input_model.maxComments
|
||||
page_url = f"https://www.youtube.com/watch?v={video_id}"
|
||||
html = await fetch_html(page_url)
|
||||
if not html:
|
||||
return
|
||||
initial = extract_yt_initial_data(html)
|
||||
if not initial:
|
||||
return
|
||||
section = comment_section_token(initial)
|
||||
if not section: # comments disabled or absent
|
||||
return
|
||||
|
||||
meta = parse_video_page(html) or {}
|
||||
base = {
|
||||
"videoId": video_id,
|
||||
"pageUrl": page_url,
|
||||
"title": meta.get("title"),
|
||||
"commentsCount": meta.get("commentsCount"),
|
||||
}
|
||||
|
||||
data = await _post_next(section)
|
||||
if not data:
|
||||
return
|
||||
|
||||
# Authoritative total lives in the comments-section header (exact integer),
|
||||
# not the watch-page HTML where the count is lazy-loaded / absent.
|
||||
header = find_first(data, "commentsHeaderRenderer")
|
||||
if header:
|
||||
base["commentsCount"] = parse_count(dig(header, "countText", "runs", 0, "text"))
|
||||
|
||||
# oldestCommentDate forces newest-first (Apify behavior).
|
||||
cutoff = _published_date(input_model.oldestCommentDate)
|
||||
sort = "NEWEST_FIRST" if cutoff else input_model.sortCommentsBy
|
||||
label = _SORT_LABELS[sort]
|
||||
if label == "Newest":
|
||||
sort_tokens = comment_sort_tokens(data)
|
||||
if label in sort_tokens:
|
||||
switched = await _post_next(sort_tokens[label])
|
||||
if switched:
|
||||
data = switched
|
||||
|
||||
count = 0
|
||||
while data:
|
||||
reply_tokens = comment_reply_tokens(data)
|
||||
entities = parse_comment_entities(data)
|
||||
|
||||
# Fetch this page's reply threads concurrently (the reused session
|
||||
# multiplexes requests), then emit in Apify order: comment, its replies.
|
||||
# ponytail: prefetch is capped at the remaining budget — each thread
|
||||
# yields >=1 item, so more threads than budget can't all be needed.
|
||||
pending = [
|
||||
(e["cid"], reply_tokens[e["cid"]])
|
||||
for e in entities
|
||||
if e["cid"] in reply_tokens
|
||||
][: max(limit - count, 0)]
|
||||
fetched = await asyncio.gather(
|
||||
*(
|
||||
_collect_replies(tok, parent_cid=cid, base=base, cutoff=cutoff)
|
||||
for cid, tok in pending
|
||||
)
|
||||
)
|
||||
replies_by_cid = {cid: r for (cid, _), r in zip(pending, fetched)}
|
||||
|
||||
for entity in entities:
|
||||
if count >= limit:
|
||||
return
|
||||
# Newest-first: the first too-old comment means the rest are older.
|
||||
if cutoff and _older(entity, cutoff):
|
||||
return
|
||||
yield _finalize(entity, base, None)
|
||||
count += 1
|
||||
|
||||
for reply in replies_by_cid.get(entity["cid"], []):
|
||||
if count >= limit:
|
||||
return
|
||||
yield reply
|
||||
count += 1
|
||||
|
||||
token = comment_next_token(data)
|
||||
if not token or count >= limit:
|
||||
return
|
||||
data = await _post_next(token)
|
||||
|
||||
|
||||
async def iter_comments(input_model: YouTubeCommentsInput) -> AsyncIterator[dict]:
|
||||
"""Yield Apify-shaped comment items for every startUrl video.
|
||||
|
||||
Videos are scraped concurrently (each video's own comment paging stays
|
||||
sequential), which is the dominant speedup when reviewing many videos.
|
||||
"""
|
||||
jobs = []
|
||||
for entry in input_model.startUrls:
|
||||
resolved = resolve_url(entry.url)
|
||||
if not resolved or resolved.kind != "video":
|
||||
logger.warning("Comments: not a video URL, skipping: %s", entry.url)
|
||||
continue
|
||||
jobs.append(_comments_for_video(resolved.value, input_model))
|
||||
async for comment in fan_out(jobs):
|
||||
yield comment
|
||||
|
||||
|
||||
async def scrape_comments(
|
||||
input_model: YouTubeCommentsInput, *, limit: int | None = None
|
||||
) -> list[dict]:
|
||||
"""Collect :func:`iter_comments` into a list, honoring an optional guard."""
|
||||
results: list[dict] = []
|
||||
async for comment in iter_comments(input_model):
|
||||
results.append(comment)
|
||||
if limit is not None and len(results) >= limit:
|
||||
break
|
||||
return results
|
||||
315
surfsense_backend/app/scrapers/youtube/innertube.py
Normal file
315
surfsense_backend/app/scrapers/youtube/innertube.py
Normal file
|
|
@ -0,0 +1,315 @@
|
|||
"""Proxy-aware fetch seam + InnerTube payload builder for the YouTube scraper.
|
||||
|
||||
All network I/O for the scraper flows through :func:`fetch_html` and
|
||||
:func:`post_innertube`, and always egresses through the residential proxy (never
|
||||
a direct connection — that would expose and risk-block the server IP). Within a
|
||||
continuation chain, :func:`proxy_session` opens ONE reused keep-alive session so
|
||||
sequential pages share a connection and a sticky exit IP (roughly halving warm
|
||||
latency vs a fresh rotating IP per request). HTML pages fall back to a
|
||||
``StealthyFetcher`` browser tier for anti-bot walls. Parsers and the orchestrator
|
||||
never see which tier served the bytes.
|
||||
|
||||
The ``AsyncFetcher.post(..., json=...)`` + ``page.json()`` / ``page.html_content``
|
||||
pattern is already proven in ``app/routes/youtube_routes.py``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from contextlib import asynccontextmanager
|
||||
from contextvars import ContextVar
|
||||
from typing import Any
|
||||
|
||||
from scrapling.fetchers import AsyncFetcher, FetcherSession
|
||||
|
||||
from app.utils.proxy import get_proxy_url
|
||||
|
||||
try: # browser tier is optional (needs patchright browsers installed)
|
||||
from scrapling.fetchers import StealthyFetcher
|
||||
except Exception: # pragma: no cover - import guard
|
||||
StealthyFetcher = None # type: ignore[assignment]
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Per-flow proxy session (set by ``proxy_session`` around one continuation
|
||||
# chain). Reusing one keep-alive connection halves warm latency AND pins a
|
||||
# single sticky exit IP on the rotating gateway, instead of paying a fresh
|
||||
# TCP+TLS handshake and drawing a new (often slow) residential node per request.
|
||||
# A ContextVar keeps each concurrent fan-out flow on its own session/IP without
|
||||
# threading a param through every parser call.
|
||||
_current_session: ContextVar[_RotatingSession | None] = ContextVar(
|
||||
"yt_proxy_session", default=None
|
||||
)
|
||||
|
||||
# Statuses that mean "this IP is throttled/blocked" -> rotate to a fresh one.
|
||||
# A probe of 120 sequential requests on one sticky IP saw zero blocks, so we
|
||||
# stay sticky for speed and rotate only *reactively*, up to this many times per
|
||||
# request. ponytail upgrade path: also rotate proactively every N requests if a
|
||||
# future run ever trips these.
|
||||
_BLOCK_STATUSES = frozenset({403, 429})
|
||||
_MAX_ROTATIONS = 3
|
||||
|
||||
|
||||
class _RotatingSession:
|
||||
"""Owns one live ``FetcherSession`` (sticky IP) and can swap it for a fresh one.
|
||||
|
||||
``rotate()`` closes the current keep-alive connection and opens a new one, so
|
||||
the rotating gateway hands out a different residential exit IP. Used
|
||||
sequentially within a single flow (never shared across concurrent tasks), so
|
||||
no locking is needed. ``session`` is ``None`` only when no proxy is configured.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._cm: Any | None = None
|
||||
self.session: Any | None = None
|
||||
self.rotations = 0
|
||||
|
||||
async def _open(self) -> None:
|
||||
proxy = get_proxy_url()
|
||||
if proxy is None:
|
||||
self._cm = self.session = None
|
||||
return
|
||||
self._cm = FetcherSession(
|
||||
proxy=proxy, stealthy_headers=True, impersonate="chrome"
|
||||
)
|
||||
self.session = await self._cm.__aenter__()
|
||||
|
||||
async def close(self) -> None:
|
||||
if self._cm is not None:
|
||||
try:
|
||||
await self._cm.__aexit__(None, None, None)
|
||||
except Exception: # best-effort teardown
|
||||
pass
|
||||
self._cm = self.session = None
|
||||
|
||||
async def rotate(self) -> Any | None:
|
||||
"""Drop the current IP and connect through a fresh one. Returns new session."""
|
||||
await self.close()
|
||||
self.rotations += 1
|
||||
await self._open()
|
||||
logger.info("[youtube] rotated proxy session (rotation #%d)", self.rotations)
|
||||
return self.session
|
||||
|
||||
|
||||
async def open_proxy_holder() -> _RotatingSession:
|
||||
"""Open a warm rotate-on-block session holder (caller owns ``close()``)."""
|
||||
holder = _RotatingSession()
|
||||
await holder._open()
|
||||
return holder
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def bind_proxy_holder(holder: _RotatingSession):
|
||||
"""Route this task's fetches through ``holder`` for the enclosed block.
|
||||
|
||||
Does NOT close the holder — enables pooling warm sessions across sequential
|
||||
jobs so each job skips the ~2s proxy TCP+TLS handshake.
|
||||
"""
|
||||
token = _current_session.set(holder)
|
||||
try:
|
||||
yield holder
|
||||
finally:
|
||||
_current_session.reset(token)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def proxy_session():
|
||||
"""Open one reused, rotate-on-block proxy session for a continuation chain."""
|
||||
holder = await open_proxy_holder()
|
||||
try:
|
||||
async with bind_proxy_holder(holder):
|
||||
yield holder
|
||||
finally:
|
||||
await holder.close()
|
||||
|
||||
# Consent cookies to dodge the EU consent interstitial that otherwise returns a
|
||||
# page with no ``ytInitialData``. Mirrors app/routes/youtube_routes.py.
|
||||
CONSENT_COOKIES = {
|
||||
"CONSENT": "PENDING+999",
|
||||
"SOCS": "CAISNQgDEitib3FfaWRlbnRpdHlmcm9udGVuZHVpc2VydmVyXzIwMjMwODI5LjA3X3AxGgJlbiADGgYIgOa_pgY",
|
||||
}
|
||||
|
||||
# Public InnerTube "WEB" client block. The web API key below is YouTube's
|
||||
# long-standing public key; ``search``/``next`` may reject a keyless POST where
|
||||
# ``browse`` accepts it, so the builder can attach both key and visitorData.
|
||||
INNERTUBE_PUBLIC_API_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"
|
||||
|
||||
INNERTUBE_BROWSE_URL = "https://www.youtube.com/youtubei/v1/browse"
|
||||
INNERTUBE_SEARCH_URL = "https://www.youtube.com/youtubei/v1/search"
|
||||
INNERTUBE_NEXT_URL = "https://www.youtube.com/youtubei/v1/next"
|
||||
|
||||
|
||||
def build_innertube_payload(
|
||||
*,
|
||||
continuation_token: str | None = None,
|
||||
browse_id: str | None = None,
|
||||
search_query: str | None = None,
|
||||
search_params: str | None = None,
|
||||
visitor_data: str | None = None,
|
||||
video_id: str | None = None,
|
||||
hl: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Build the standard InnerTube ``context.client`` payload.
|
||||
|
||||
Ported from the Scrapfly reference ``call_youtube_api`` but I/O-free so it is
|
||||
unit-testable and reusable across browse/search/next/player endpoints.
|
||||
``hl`` selects the response language: a ``/next`` call with a non-default
|
||||
``hl`` returns the video's creator-localized title/description.
|
||||
"""
|
||||
client: dict[str, Any] = {
|
||||
"hl": hl or "en",
|
||||
"gl": "US",
|
||||
"clientName": "WEB",
|
||||
"clientVersion": "2.20241111.07.00",
|
||||
"platform": "DESKTOP",
|
||||
"userInterfaceTheme": "USER_INTERFACE_THEME_DARK",
|
||||
}
|
||||
if visitor_data:
|
||||
client["visitorData"] = visitor_data
|
||||
|
||||
payload: dict[str, Any] = {
|
||||
"context": {
|
||||
"client": client,
|
||||
"user": {"lockedSafetyMode": False},
|
||||
"request": {"useSsl": True},
|
||||
}
|
||||
}
|
||||
if browse_id is not None:
|
||||
payload["browseId"] = browse_id
|
||||
if video_id is not None:
|
||||
payload["videoId"] = video_id
|
||||
if search_query is not None:
|
||||
payload["query"] = search_query
|
||||
if search_params is not None:
|
||||
payload["params"] = search_params
|
||||
if continuation_token is not None:
|
||||
payload["continuation"] = continuation_token
|
||||
return payload
|
||||
|
||||
|
||||
async def post_innertube(
|
||||
base_url: str,
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
api_key: str | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
"""POST an InnerTube payload and return parsed JSON (or ``None`` on failure).
|
||||
|
||||
Always egresses through the proxy (a reused per-flow session when one is
|
||||
open, else a one-shot ``AsyncFetcher``). Never connects directly — a direct
|
||||
hit would expose and risk-block the server IP.
|
||||
"""
|
||||
url = f"{base_url}?key={api_key}" if api_key else base_url
|
||||
holder = _current_session.get()
|
||||
for attempt in range(_MAX_ROTATIONS + 1):
|
||||
session = holder.session if holder else None
|
||||
try:
|
||||
started = time.perf_counter()
|
||||
if session is not None:
|
||||
page = await session.post(url, json=payload)
|
||||
else:
|
||||
page = await AsyncFetcher.post(
|
||||
url, json=payload, proxy=get_proxy_url(), stealthy_headers=True
|
||||
)
|
||||
fetch_ms = (time.perf_counter() - started) * 1000
|
||||
logger.info(
|
||||
"[youtube][perf] source=innertube reused=%s url=%s status=%s fetch_ms=%.1f",
|
||||
session is not None,
|
||||
base_url,
|
||||
page.status,
|
||||
fetch_ms,
|
||||
)
|
||||
if page.status == 200:
|
||||
return page.json()
|
||||
logger.warning("InnerTube POST %s returned %s", base_url, page.status)
|
||||
if not (holder and page.status in _BLOCK_STATUSES and attempt < _MAX_ROTATIONS):
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.warning("InnerTube POST %s failed: %s", base_url, e)
|
||||
if not (holder and attempt < _MAX_ROTATIONS):
|
||||
return None
|
||||
await holder.rotate() # blocked/errored on a proxy session: try a fresh IP
|
||||
return None
|
||||
|
||||
|
||||
async def fetch_html(url: str, *, cookies: dict[str, str] | None = None) -> str | None:
|
||||
"""GET a YouTube page and return raw HTML (or ``None`` on failure).
|
||||
|
||||
Fetches through the proxy (reused per-flow session when open, else a one-shot
|
||||
``AsyncFetcher``); falls back to the ``StealthyFetcher`` browser tier for
|
||||
anti-bot walls. Sets consent cookies by default so EU-egress fetches don't
|
||||
hit the consent wall and return a page without ``ytInitialData``.
|
||||
"""
|
||||
merged = {**CONSENT_COOKIES, **(cookies or {})}
|
||||
headers = {"Accept-Language": "en-US,en;q=0.9"}
|
||||
holder = _current_session.get()
|
||||
for attempt in range(_MAX_ROTATIONS + 1):
|
||||
session = holder.session if holder else None
|
||||
try:
|
||||
started = time.perf_counter()
|
||||
if session is not None:
|
||||
page = await session.get(url, headers=headers, cookies=merged)
|
||||
else:
|
||||
page = await AsyncFetcher.get(
|
||||
url,
|
||||
headers=headers,
|
||||
cookies=merged,
|
||||
proxy=get_proxy_url(),
|
||||
stealthy_headers=True,
|
||||
)
|
||||
fetch_ms = (time.perf_counter() - started) * 1000
|
||||
logger.info(
|
||||
"[youtube][perf] source=html reused=%s url=%s status=%s fetch_ms=%.1f",
|
||||
session is not None,
|
||||
url,
|
||||
page.status,
|
||||
fetch_ms,
|
||||
)
|
||||
if page.status == 200:
|
||||
return page.html_content
|
||||
logger.warning("HTML GET %s returned %s", url, page.status)
|
||||
if not (holder and page.status in _BLOCK_STATUSES and attempt < _MAX_ROTATIONS):
|
||||
break
|
||||
except Exception as e:
|
||||
logger.warning("HTML GET %s failed: %s", url, e)
|
||||
if not (holder and attempt < _MAX_ROTATIONS):
|
||||
break
|
||||
await holder.rotate() # blocked/errored on a proxy session: try a fresh IP
|
||||
# All proxy attempts blocked/failed -> last-resort browser tier.
|
||||
return await _fetch_html_stealthy(url)
|
||||
|
||||
|
||||
async def _fetch_html_stealthy(url: str) -> str | None:
|
||||
"""Last-resort browser fetch for anti-bot walls (mirrors the crawler tier).
|
||||
|
||||
``StealthyFetcher.fetch`` is synchronous, so it runs in a worker thread to
|
||||
keep the event loop free. Returns ``None`` when the browser tier is
|
||||
unavailable or still blocked.
|
||||
"""
|
||||
if StealthyFetcher is None:
|
||||
return None
|
||||
try:
|
||||
started = time.perf_counter()
|
||||
page = await asyncio.to_thread(
|
||||
StealthyFetcher.fetch,
|
||||
url,
|
||||
headless=True,
|
||||
network_idle=True,
|
||||
solve_cloudflare=True,
|
||||
proxy=get_proxy_url(),
|
||||
)
|
||||
fetch_ms = (time.perf_counter() - started) * 1000
|
||||
logger.info(
|
||||
"[youtube][perf] source=html tier=stealthy url=%s status=%s fetch_ms=%.1f",
|
||||
url,
|
||||
page.status,
|
||||
fetch_ms,
|
||||
)
|
||||
if page.status == 200:
|
||||
return page.html_content
|
||||
logger.warning("HTML GET %s tier=stealthy returned %s", url, page.status)
|
||||
except Exception as e:
|
||||
logger.warning("HTML GET %s tier=stealthy failed: %s", url, e)
|
||||
return None
|
||||
812
surfsense_backend/app/scrapers/youtube/parsers.py
Normal file
812
surfsense_backend/app/scrapers/youtube/parsers.py
Normal file
|
|
@ -0,0 +1,812 @@
|
|||
"""Pure parsing/normalization for YouTube page + InnerTube JSON.
|
||||
|
||||
I/O-free and dependency-free (no ``jmespath``/``jsonpath-ng``): all traversal
|
||||
uses two tiny helpers — :func:`find_all` (the ``$..key`` equivalent, same style
|
||||
as ``_extract_playlist_video_ids`` in ``app/routes/youtube_routes.py``) and
|
||||
:func:`dig` (null-safe positional get). Ported from the Scrapfly reference
|
||||
``references/scrapfly-scrapers/youtube-scraper/youtube.py`` with all brittle deep
|
||||
jmespath paths rewritten and the throwing ``convert_to_number`` replaced by a
|
||||
robust :func:`parse_count`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Traversal helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def find_all(obj: Any, key: str) -> list[Any]:
|
||||
"""Collect every value stored under ``key`` anywhere in a nested structure."""
|
||||
out: list[Any] = []
|
||||
|
||||
def _walk(node: Any) -> None:
|
||||
if isinstance(node, dict):
|
||||
for k, v in node.items():
|
||||
if k == key:
|
||||
out.append(v)
|
||||
_walk(v)
|
||||
elif isinstance(node, list):
|
||||
for item in node:
|
||||
_walk(item)
|
||||
|
||||
_walk(obj)
|
||||
return out
|
||||
|
||||
|
||||
def find_first(obj: Any, key: str) -> Any:
|
||||
"""First value stored under ``key`` anywhere in the structure, else ``None``."""
|
||||
matches = find_all(obj, key)
|
||||
return matches[0] if matches else None
|
||||
|
||||
|
||||
def find_last(obj: Any, key: str) -> Any:
|
||||
"""Last value stored under ``key`` (continuation tokens want the newest)."""
|
||||
matches = find_all(obj, key)
|
||||
return matches[-1] if matches else None
|
||||
|
||||
|
||||
def dig(obj: Any, *path: Any) -> Any:
|
||||
"""Null-safe positional get through dict keys / list indices."""
|
||||
cur = obj
|
||||
for step in path:
|
||||
if isinstance(step, int) and isinstance(cur, list):
|
||||
if -len(cur) <= step < len(cur):
|
||||
cur = cur[step]
|
||||
else:
|
||||
return None
|
||||
elif isinstance(cur, dict):
|
||||
cur = cur.get(step)
|
||||
else:
|
||||
return None
|
||||
if cur is None:
|
||||
return None
|
||||
return cur
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Value normalization
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_COUNT_RE = re.compile(r"([\d,.]+)\s*([KMB]?)", re.IGNORECASE)
|
||||
_MULTIPLIER = {"": 1, "K": 1_000, "M": 1_000_000, "B": 1_000_000_000}
|
||||
|
||||
|
||||
def parse_count(value: Any) -> int | None:
|
||||
"""Normalize a YouTube count string to an int, else ``None``.
|
||||
|
||||
Handles ``"451K views"``, ``"1.2M"``, ``"1,234"``, plain ints, and returns
|
||||
``None`` for ``"No views"`` / missing (the reference's ``convert_to_number``
|
||||
throws on all of these).
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, (int, float)):
|
||||
return int(value)
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
match = _COUNT_RE.search(value.strip())
|
||||
if not match:
|
||||
return None
|
||||
number, suffix = match.group(1), match.group(2).upper()
|
||||
number = number.replace(",", "")
|
||||
if number in ("", "."):
|
||||
return None
|
||||
try:
|
||||
return int(float(number) * _MULTIPLIER[suffix])
|
||||
except (ValueError, KeyError):
|
||||
return None
|
||||
|
||||
|
||||
def parse_date(microformat: dict | None) -> str | None:
|
||||
"""Prefer the real publish date from a video's microformat renderer.
|
||||
|
||||
``playerMicroformatRenderer.publishDate`` is an actual date (``2024-08-27``),
|
||||
unlike the relative ``"7 days ago"`` strings on list pages.
|
||||
"""
|
||||
if not microformat:
|
||||
return None
|
||||
return microformat.get("publishDate") or microformat.get("uploadDate")
|
||||
|
||||
|
||||
def seconds_to_duration(seconds: Any) -> str | None:
|
||||
"""Format a length in seconds as ``HH:MM:SS`` (Apify ``duration`` shape)."""
|
||||
total = parse_count(seconds)
|
||||
if total is None:
|
||||
return None
|
||||
hours, rem = divmod(total, 3600)
|
||||
minutes, secs = divmod(rem, 60)
|
||||
return f"{hours:02d}:{minutes:02d}:{secs:02d}"
|
||||
|
||||
|
||||
def _best_thumbnail(thumbnails: Any) -> str | None:
|
||||
"""Return the highest-resolution thumbnail URL from a thumbnails list."""
|
||||
if not isinstance(thumbnails, list) or not thumbnails:
|
||||
return None
|
||||
best = None
|
||||
best_area = -1
|
||||
for t in thumbnails:
|
||||
if not isinstance(t, dict) or "url" not in t:
|
||||
continue
|
||||
area = (t.get("width") or 0) * (t.get("height") or 0)
|
||||
if area >= best_area:
|
||||
best_area = area
|
||||
best = t["url"]
|
||||
return best
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Embedded page-JSON extraction (brace matcher, from youtube_routes.py)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _extract_json_object(html: str, patterns: list[re.Pattern[str]]) -> dict | None:
|
||||
start = -1
|
||||
for pattern in patterns:
|
||||
match = pattern.search(html)
|
||||
if match:
|
||||
start = match.end()
|
||||
break
|
||||
if start == -1:
|
||||
return None
|
||||
|
||||
depth = 0
|
||||
i = start
|
||||
while i < len(html):
|
||||
ch = html[i]
|
||||
if ch == "{":
|
||||
depth += 1
|
||||
elif ch == "}":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
break
|
||||
elif ch == '"':
|
||||
i += 1
|
||||
while i < len(html) and html[i] != '"':
|
||||
if html[i] == "\\":
|
||||
i += 1
|
||||
i += 1
|
||||
i += 1
|
||||
|
||||
try:
|
||||
return json.loads(html[start : i + 1])
|
||||
except (json.JSONDecodeError, IndexError):
|
||||
return None
|
||||
|
||||
|
||||
def extract_yt_initial_data(html: str) -> dict | None:
|
||||
"""Extract the ``ytInitialData`` object embedded in a YouTube page."""
|
||||
return _extract_json_object(
|
||||
html,
|
||||
[
|
||||
re.compile(r"var\s+ytInitialData\s*=\s*"),
|
||||
re.compile(r'window\["ytInitialData"\]\s*=\s*'),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def extract_yt_initial_player_response(html: str) -> dict | None:
|
||||
"""Extract the ``ytInitialPlayerResponse`` object from a video page."""
|
||||
return _extract_json_object(
|
||||
html,
|
||||
[
|
||||
re.compile(r"var\s+ytInitialPlayerResponse\s*=\s*"),
|
||||
re.compile(r'window\["ytInitialPlayerResponse"\]\s*=\s*'),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Page / API parsers -> partial VideoItem dicts
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _unwrap_redirect(url: str) -> str:
|
||||
"""Resolve a ``youtube.com/redirect?...&q=<target>`` link to its target."""
|
||||
if "/redirect" in url and "q=" in url:
|
||||
q = parse_qs(urlparse(url).query).get("q")
|
||||
if q:
|
||||
return q[0]
|
||||
if url.startswith("/"):
|
||||
return f"https://www.youtube.com{url}"
|
||||
return url
|
||||
|
||||
|
||||
def parse_description_links(initial: dict) -> list[dict[str, str]] | None:
|
||||
"""Extract ``{url, text}`` links from the video's attributed description.
|
||||
|
||||
Each ``commandRun`` marks a linked span (``startIndex``/``length``) into the
|
||||
description ``content``; external links are wrapped in a YouTube redirect
|
||||
whose real target is the ``q`` query param.
|
||||
"""
|
||||
ad = find_first(initial, "attributedDescription")
|
||||
if not isinstance(ad, dict):
|
||||
return None
|
||||
content = ad.get("content") or ""
|
||||
# YouTube's startIndex/length are UTF-16 code-unit offsets (JS string
|
||||
# semantics); emoji in the description are surrogate pairs, so slice on the
|
||||
# UTF-16-LE bytes to keep spans aligned instead of Python code points.
|
||||
units = content.encode("utf-16-le")
|
||||
out: list[dict[str, str]] = []
|
||||
for run in ad.get("commandRuns") or []:
|
||||
start, length = run.get("startIndex"), run.get("length")
|
||||
if not isinstance(start, int) or not isinstance(length, int):
|
||||
continue
|
||||
cmd = dig(run, "onTap", "innertubeCommand")
|
||||
url = dig(cmd, "urlEndpoint", "url") or dig(
|
||||
cmd, "commandMetadata", "webCommandMetadata", "url"
|
||||
)
|
||||
if not url:
|
||||
continue
|
||||
text = units[start * 2 : (start + length) * 2].decode("utf-16-le", "ignore")
|
||||
out.append({"url": _unwrap_redirect(url), "text": text})
|
||||
return out or None
|
||||
|
||||
|
||||
def _comments_turned_off(initial: dict) -> bool | None:
|
||||
"""Detect a disabled comments section, else ``None`` if no section is present.
|
||||
|
||||
An enabled section carries a ``continuationItemRenderer`` (the token that
|
||||
loads the comments); a disabled one keeps the section but drops the token.
|
||||
"""
|
||||
sections = [
|
||||
s
|
||||
for s in find_all(initial, "itemSectionRenderer")
|
||||
if isinstance(s, dict) and s.get("sectionIdentifier") == "comment-item-section"
|
||||
]
|
||||
if not sections:
|
||||
return None
|
||||
return not any(find_all(s, "continuationItemRenderer") for s in sections)
|
||||
|
||||
|
||||
def _is_members_only(player: dict, initial: dict) -> bool:
|
||||
"""True for members-only videos (a members badge or a members-only offer)."""
|
||||
for badge in find_all(initial, "metadataBadgeRenderer"):
|
||||
if isinstance(badge, dict) and badge.get("style") == "BADGE_STYLE_TYPE_MEMBERS_ONLY":
|
||||
return True
|
||||
status = player.get("playabilityStatus") or {}
|
||||
return find_first(status, "offerId") == "sponsors_only_video"
|
||||
|
||||
|
||||
def _is_age_restricted(player: dict, microformat: dict) -> bool | None:
|
||||
"""True when the player response signals an age gate, else ``None``.
|
||||
|
||||
``playabilityStatus`` is authoritative (age-gated videos report a legacy age
|
||||
gate reason or an "age"-mentioning reason); ``isFamilySafe`` is the fallback.
|
||||
"""
|
||||
status = player.get("playabilityStatus")
|
||||
if isinstance(status, dict):
|
||||
reason = (status.get("reason") or "").lower()
|
||||
if status.get("desktopLegacyAgeGateReason") or "age" in reason:
|
||||
return True
|
||||
if microformat.get("isFamilySafe") is False:
|
||||
return True
|
||||
return None
|
||||
|
||||
|
||||
def _channel_from_byline(renderer: dict) -> dict[str, Any]:
|
||||
"""Pull channel name/id/url from a list item's owner byline runs.
|
||||
|
||||
Search ``videoRenderer``s carry the uploader in ``ownerText`` /
|
||||
``longBylineText``; the run's ``browseEndpoint`` holds the channel id and the
|
||||
canonical ``/@handle`` (or ``/channel/UC...``) url.
|
||||
"""
|
||||
run = dig(renderer, "ownerText", "runs", 0) or dig(
|
||||
renderer, "longBylineText", "runs", 0
|
||||
)
|
||||
if not isinstance(run, dict):
|
||||
return {}
|
||||
browse = dig(run, "navigationEndpoint", "browseEndpoint") or {}
|
||||
cid = browse.get("browseId")
|
||||
base = browse.get("canonicalBaseUrl")
|
||||
out: dict[str, Any] = {}
|
||||
if run.get("text"):
|
||||
out["channelName"] = run["text"]
|
||||
if cid:
|
||||
out["channelId"] = cid
|
||||
if isinstance(base, str) and base:
|
||||
out["channelUrl"] = f"https://www.youtube.com{base}"
|
||||
if base.startswith("/@"):
|
||||
out["channelUsername"] = base[2:] or None
|
||||
elif cid:
|
||||
out["channelUrl"] = f"https://www.youtube.com/channel/{cid}"
|
||||
return out
|
||||
|
||||
|
||||
def parse_channel_metadata(initial_data: dict) -> dict[str, Any]:
|
||||
"""Channel identity from ``channelMetadataRenderer`` (in the channel HTML).
|
||||
|
||||
Zero extra fetch: name/id/url/avatar/description are already in the channel
|
||||
page's ``ytInitialData`` and apply to every video the channel flow yields.
|
||||
"""
|
||||
meta = find_first(initial_data, "channelMetadataRenderer")
|
||||
if not isinstance(meta, dict):
|
||||
return {}
|
||||
out: dict[str, Any] = {}
|
||||
if meta.get("title"):
|
||||
out["channelName"] = meta["title"]
|
||||
if meta.get("externalId"):
|
||||
out["channelId"] = meta["externalId"]
|
||||
if meta.get("description"):
|
||||
out["channelDescription"] = meta["description"]
|
||||
url = meta.get("vanityChannelUrl") or meta.get("channelUrl")
|
||||
if url:
|
||||
out["channelUrl"] = url
|
||||
avatar = _best_thumbnail(dig(meta, "avatar", "thumbnails"))
|
||||
if avatar:
|
||||
out["channelAvatarUrl"] = avatar
|
||||
banner = _best_thumbnail(
|
||||
dig(
|
||||
find_first(initial_data, "pageHeaderViewModel"),
|
||||
"banner",
|
||||
"imageBannerViewModel",
|
||||
"image",
|
||||
"sources",
|
||||
)
|
||||
)
|
||||
if banner:
|
||||
out["channelBannerUrl"] = banner
|
||||
return out
|
||||
|
||||
|
||||
def channel_about_tokens(initial_data: dict) -> list[str]:
|
||||
"""Continuation tokens for the channel's engagement panels (one is About).
|
||||
|
||||
The About panel isn't embedded in the channel HTML; it's fetched via a
|
||||
``/browse`` continuation. Panels aren't labeled, so callers try each token
|
||||
and keep the response that yields an ``aboutChannelViewModel``.
|
||||
"""
|
||||
tokens: list[str] = []
|
||||
for ep in find_all(initial_data, "showEngagementPanelEndpoint"):
|
||||
token = dig(find_first(ep, "continuationCommand"), "token")
|
||||
if token:
|
||||
tokens.append(token)
|
||||
return tokens
|
||||
|
||||
|
||||
def parse_channel_about(about: dict) -> dict[str, Any]:
|
||||
"""Deep channel fields from an ``aboutChannelViewModel``."""
|
||||
out: dict[str, Any] = {}
|
||||
if about.get("description"):
|
||||
out["channelDescription"] = about["description"]
|
||||
if about.get("country"):
|
||||
out["channelLocation"] = about["country"]
|
||||
subs = parse_count(about.get("subscriberCountText"))
|
||||
if subs is not None:
|
||||
out["numberOfSubscribers"] = subs
|
||||
views = parse_count(about.get("viewCountText"))
|
||||
if views is not None:
|
||||
out["channelTotalViews"] = views
|
||||
videos = parse_count(about.get("videoCountText"))
|
||||
if videos is not None:
|
||||
out["channelTotalVideos"] = videos
|
||||
joined = dig(about, "joinedDateText", "content") or about.get("joinedDateText")
|
||||
if isinstance(joined, str) and joined:
|
||||
# "Joined Feb 8, 2005" -> keep the date portion.
|
||||
out["channelJoinedDate"] = joined.replace("Joined", "").strip()
|
||||
return out
|
||||
|
||||
|
||||
# The primary-info super-title links to a geo-restricted search for tagged
|
||||
# videos; its a11y label is the reliable "is this a location?" signal (the same
|
||||
# slot holds hashtags otherwise). The label is English (hl=en client).
|
||||
_GEO_TAG_RE = re.compile(r"geo tagged with (.+)", re.IGNORECASE)
|
||||
|
||||
|
||||
def parse_location(initial: dict) -> str | None:
|
||||
"""Video geo-tag (place name) from ``videoPrimaryInfoRenderer.superTitleLink``.
|
||||
|
||||
Returns ``None`` when the super-title is hashtags (or absent) rather than a
|
||||
location. The place name is read from the a11y label ("...geo tagged with
|
||||
Rome"), which is proper-cased where the visible runs are upper-cased.
|
||||
"""
|
||||
st = dig(find_first(initial, "videoPrimaryInfoRenderer"), "superTitleLink")
|
||||
if not isinstance(st, dict):
|
||||
return None
|
||||
label = dig(st, "accessibility", "accessibilityData", "label") or ""
|
||||
match = _GEO_TAG_RE.search(label)
|
||||
return match.group(1).strip() if match else None
|
||||
|
||||
|
||||
def parse_translation(next_data: dict) -> tuple[str | None, str | None]:
|
||||
"""(title, description) from a ``/next`` response fetched with a target ``hl``.
|
||||
|
||||
YouTube renders the creator-localized title/description into
|
||||
``videoPrimaryInfoRenderer`` / ``videoSecondaryInfoRenderer`` for the request
|
||||
language. Videos without a localization return their original text.
|
||||
"""
|
||||
vpir = find_first(next_data, "videoPrimaryInfoRenderer") or {}
|
||||
title = "".join(r.get("text", "") for r in (dig(vpir, "title", "runs") or [])) or None
|
||||
|
||||
vsir = find_first(next_data, "videoSecondaryInfoRenderer") or {}
|
||||
description = dig(vsir, "attributedDescription", "content")
|
||||
if description is None:
|
||||
description = (
|
||||
"".join(r.get("text", "") for r in (dig(vsir, "description", "runs") or []))
|
||||
or None
|
||||
)
|
||||
return title, description
|
||||
|
||||
|
||||
def parse_collaborators(initial: dict) -> list[dict[str, str | None]] | None:
|
||||
"""Collaborator channels from the multi-owner "Collaborators" dialog.
|
||||
|
||||
Collaboration videos replace the owner's plain ``title`` with an
|
||||
``attributedTitle`` whose tap opens a dialog listing each credited channel
|
||||
(``listItemViewModel``). Returns ``None`` for ordinary single-owner videos.
|
||||
"""
|
||||
owner = dig(find_first(initial, "videoSecondaryInfoRenderer"), "owner", "videoOwnerRenderer")
|
||||
attributed = owner.get("attributedTitle") if isinstance(owner, dict) else None
|
||||
if not isinstance(attributed, dict):
|
||||
return None
|
||||
|
||||
# The dialog's own list holds one row per channel. Read those rows directly
|
||||
# (not find_all) — each row's subscribe button nests its own menu items.
|
||||
dialog = find_first(attributed, "dialogViewModel")
|
||||
rows = dig(find_first(dialog, "listViewModel"), "listItems") or []
|
||||
|
||||
collaborators: list[dict[str, str | None]] = []
|
||||
for row in rows:
|
||||
item = row.get("listItemViewModel") if isinstance(row, dict) else None
|
||||
name = dig(item, "title", "content")
|
||||
if not name:
|
||||
continue
|
||||
base = dig(find_first(item, "browseEndpoint"), "canonicalBaseUrl")
|
||||
collaborators.append(
|
||||
{
|
||||
"name": name,
|
||||
"username": base[2:] if isinstance(base, str) and base.startswith("/@") else None,
|
||||
"url": f"https://www.youtube.com{base}" if isinstance(base, str) and base else None,
|
||||
}
|
||||
)
|
||||
return collaborators or None
|
||||
|
||||
|
||||
def parse_video_page(html: str) -> dict[str, Any] | None:
|
||||
"""Parse a video/shorts watch page into a partial VideoItem dict.
|
||||
|
||||
Merges ``ytInitialPlayerResponse.videoDetails`` (title, views, length,
|
||||
description, thumbnails) with ``ytInitialData`` (likes, comment count,
|
||||
channel info) and the microformat renderer (real publish date).
|
||||
"""
|
||||
player = extract_yt_initial_player_response(html)
|
||||
initial = extract_yt_initial_data(html)
|
||||
if not player:
|
||||
return None
|
||||
|
||||
details = player.get("videoDetails") or {}
|
||||
microformat = dig(player, "microformat", "playerMicroformatRenderer") or {}
|
||||
video_id = details.get("videoId")
|
||||
|
||||
result: dict[str, Any] = {
|
||||
"id": video_id,
|
||||
"url": f"https://www.youtube.com/watch?v={video_id}" if video_id else None,
|
||||
"title": details.get("title"),
|
||||
"text": details.get("shortDescription"),
|
||||
"viewCount": parse_count(details.get("viewCount")),
|
||||
"duration": seconds_to_duration(details.get("lengthSeconds")),
|
||||
"date": parse_date(microformat),
|
||||
"thumbnailUrl": _best_thumbnail(dig(details, "thumbnail", "thumbnails")),
|
||||
"hashtags": details.get("keywords") or [],
|
||||
"channelName": details.get("author"),
|
||||
"channelId": details.get("channelId"),
|
||||
"isAgeRestricted": _is_age_restricted(player, microformat),
|
||||
# Monetized ⇔ the player response carries ad slots. ponytail: absence
|
||||
# isn't proof (a logged-out fetch may omit ads), so False→None, not False.
|
||||
"isMonetized": bool(player.get("adPlacements") or player.get("playerAds"))
|
||||
or None,
|
||||
"isMembersOnly": _is_members_only(player, initial or {}),
|
||||
# "Includes paid promotion" disclosure overlay.
|
||||
"isPaidContent": find_first(player, "paidContentOverlayRenderer") is not None,
|
||||
}
|
||||
|
||||
if initial:
|
||||
# Likes: the LIKE button's label carries the count string.
|
||||
likes = [
|
||||
b.get("title")
|
||||
for b in find_all(initial, "buttonViewModel")
|
||||
if isinstance(b, dict) and b.get("iconName") == "LIKE"
|
||||
]
|
||||
result["likes"] = parse_count(likes[0]) if likes else None
|
||||
|
||||
# Comment count lives in the engagement panel contextual info.
|
||||
contextual = find_first(initial, "contextualInfo")
|
||||
result["commentsCount"] = parse_count(dig(contextual, "runs", 0, "text"))
|
||||
result["commentsTurnedOff"] = _comments_turned_off(initial)
|
||||
result["descriptionLinks"] = parse_description_links(initial)
|
||||
result["location"] = parse_location(initial)
|
||||
result["collaborators"] = parse_collaborators(initial)
|
||||
|
||||
# Channel handle / URL from the canonical base url ("/@Handle").
|
||||
base = find_first(initial, "canonicalBaseUrl")
|
||||
if isinstance(base, str) and base:
|
||||
result["channelUrl"] = f"https://www.youtube.com{base}"
|
||||
# Only "/@handle" yields a username; "/channel/UC..." is an id, not a handle.
|
||||
if base.startswith("/@"):
|
||||
result["channelUsername"] = base[2:] or None
|
||||
|
||||
subs = find_first(initial, "subscriberCountText")
|
||||
result["numberOfSubscribers"] = parse_count(
|
||||
subs.get("simpleText") if isinstance(subs, dict) else subs
|
||||
)
|
||||
|
||||
# Verified badge on the channel owner.
|
||||
badges = find_all(initial, "metadataBadgeRenderer")
|
||||
result["isChannelVerified"] = any(
|
||||
isinstance(b, dict) and b.get("tooltip") == "Verified" for b in badges
|
||||
) or None
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _view_count_from_renderer(renderer: dict) -> int | None:
|
||||
for key in ("shortViewCountText", "viewCountText"):
|
||||
node = renderer.get(key)
|
||||
if isinstance(node, dict):
|
||||
text = node.get("simpleText") or dig(node, "runs", 0, "text")
|
||||
count = parse_count(text)
|
||||
if count is not None:
|
||||
return count
|
||||
return None
|
||||
|
||||
|
||||
def parse_search_response(data: dict) -> tuple[list[dict[str, Any]], str | None]:
|
||||
"""Parse an InnerTube search response into partial VideoItem dicts + token."""
|
||||
results: list[dict[str, Any]] = []
|
||||
for r in find_all(data, "videoRenderer"):
|
||||
if not isinstance(r, dict) or "videoId" not in r:
|
||||
continue
|
||||
vid = r["videoId"]
|
||||
results.append(
|
||||
{
|
||||
"id": vid,
|
||||
"url": f"https://www.youtube.com/watch?v={vid}",
|
||||
"title": dig(r, "title", "runs", 0, "text"),
|
||||
"text": dig(
|
||||
r, "detailedMetadataSnippets", 0, "snippetText", "runs", 0, "text"
|
||||
),
|
||||
"date": None, # list pages only expose relative time
|
||||
"publishedTimeText": dig(r, "publishedTimeText", "simpleText"),
|
||||
"duration": dig(r, "lengthText", "simpleText"),
|
||||
"viewCount": _view_count_from_renderer(r),
|
||||
"thumbnailUrl": _best_thumbnail(dig(r, "thumbnail", "thumbnails")),
|
||||
**_channel_from_byline(r),
|
||||
}
|
||||
)
|
||||
return results, _continuation_token(data)
|
||||
|
||||
|
||||
def _continuation_token(data: dict) -> str | None:
|
||||
tokens = [
|
||||
t.get("token")
|
||||
for t in find_all(data, "continuationCommand")
|
||||
if isinstance(t, dict) and t.get("token")
|
||||
]
|
||||
return tokens[-1] if tokens else None
|
||||
|
||||
|
||||
def parse_playlist_video_ids(data: dict) -> list[str]:
|
||||
"""Ordered, de-duped video ids from a playlist ``/browse`` response.
|
||||
|
||||
Playlist entries are ``lockupViewModel`` (the old ``playlistVideoRenderer``
|
||||
is retired); ``contentId`` holds the id. The 11-char guard keeps only videos
|
||||
and drops any playlist/channel lockup (e.g. the sidebar self-lockup).
|
||||
"""
|
||||
seen: set[str] = set()
|
||||
ids: list[str] = []
|
||||
for lockup in find_all(data, "lockupViewModel"):
|
||||
vid = lockup.get("contentId") if isinstance(lockup, dict) else None
|
||||
if vid and len(vid) == 11 and vid not in seen:
|
||||
seen.add(vid)
|
||||
ids.append(vid)
|
||||
return ids
|
||||
|
||||
|
||||
def parse_channel_videos(data: dict) -> tuple[list[dict[str, Any]], str | None]:
|
||||
"""Parse an InnerTube channel-videos browse response (richItem lockups)."""
|
||||
results: list[dict[str, Any]] = []
|
||||
for item in find_all(data, "richItemRenderer"):
|
||||
lockup = dig(item, "content", "lockupViewModel")
|
||||
if not isinstance(lockup, dict):
|
||||
continue
|
||||
vid = lockup.get("contentId")
|
||||
if not vid:
|
||||
continue
|
||||
meta = dig(lockup, "metadata", "lockupMetadataViewModel")
|
||||
rows = dig(meta, "metadata", "contentMetadataViewModel", "metadataRows") or []
|
||||
parts = dig(rows, 0, "metadataParts") or []
|
||||
results.append(
|
||||
{
|
||||
"id": vid,
|
||||
"url": f"https://www.youtube.com/watch?v={vid}",
|
||||
"title": dig(meta, "title", "content"),
|
||||
"viewCount": parse_count(dig(parts, 0, "text", "content")),
|
||||
"date": None,
|
||||
"publishedTimeText": dig(parts, 1, "text", "content"),
|
||||
"duration": _lockup_duration(lockup),
|
||||
"thumbnailUrl": _best_thumbnail(
|
||||
dig(lockup, "contentImage", "thumbnailViewModel", "image", "sources")
|
||||
),
|
||||
}
|
||||
)
|
||||
return results, _continuation_token(data)
|
||||
|
||||
|
||||
def parse_channel_shorts(data: dict) -> tuple[list[dict[str, Any]], str | None]:
|
||||
"""Parse a channel Shorts browse/seed response (``shortsLockupViewModel``).
|
||||
|
||||
Shorts use a different lockup than videos/streams: the id is on the reel
|
||||
watch endpoint, and title/views live in ``overlayMetadata``.
|
||||
"""
|
||||
results: list[dict[str, Any]] = []
|
||||
for s in find_all(data, "shortsLockupViewModel"):
|
||||
if not isinstance(s, dict):
|
||||
continue
|
||||
vid = dig(s, "onTap", "innertubeCommand", "reelWatchEndpoint", "videoId")
|
||||
if not vid:
|
||||
entity = s.get("entityId") or ""
|
||||
vid = entity.rsplit("-", 1)[-1] if entity else None
|
||||
if not vid:
|
||||
continue
|
||||
results.append(
|
||||
{
|
||||
"id": vid,
|
||||
"url": f"https://www.youtube.com/shorts/{vid}",
|
||||
"title": dig(s, "overlayMetadata", "primaryText", "content"),
|
||||
"viewCount": parse_count(
|
||||
dig(s, "overlayMetadata", "secondaryText", "content")
|
||||
),
|
||||
"date": None,
|
||||
"thumbnailUrl": _best_thumbnail(
|
||||
dig(s, "thumbnailViewModel", "image", "sources")
|
||||
),
|
||||
}
|
||||
)
|
||||
return results, _continuation_token(data)
|
||||
|
||||
|
||||
def _lockup_duration(lockup: dict) -> str | None:
|
||||
overlays = dig(
|
||||
lockup, "contentImage", "thumbnailViewModel", "overlays"
|
||||
) or []
|
||||
for overlay in overlays:
|
||||
text = dig(
|
||||
overlay,
|
||||
"thumbnailBottomOverlayViewModel",
|
||||
"badges",
|
||||
0,
|
||||
"thumbnailBadgeViewModel",
|
||||
"text",
|
||||
)
|
||||
if text:
|
||||
return text
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Comments (/next endpoint)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def comment_section_token(initial_data: dict) -> str | None:
|
||||
"""The watch page's comments-section continuation (seeds the /next call)."""
|
||||
for section in find_all(initial_data, "itemSectionRenderer"):
|
||||
if (
|
||||
isinstance(section, dict)
|
||||
and section.get("sectionIdentifier") == "comment-item-section"
|
||||
):
|
||||
token = dig(
|
||||
find_first(section, "continuationItemRenderer"),
|
||||
"continuationEndpoint",
|
||||
"continuationCommand",
|
||||
"token",
|
||||
)
|
||||
if token:
|
||||
return token
|
||||
return None
|
||||
|
||||
|
||||
def comment_sort_tokens(data: dict) -> dict[str, str]:
|
||||
"""Map the comment sort labels ("Top"/"Newest") to their continuations."""
|
||||
tokens: dict[str, str] = {}
|
||||
sub = find_first(data, "sortFilterSubMenuRenderer")
|
||||
for item in dig(sub, "subMenuItems") or []:
|
||||
title = item.get("title") if isinstance(item, dict) else None
|
||||
token = dig(item, "serviceEndpoint", "continuationCommand", "token")
|
||||
if title and token:
|
||||
tokens[title] = token
|
||||
return tokens
|
||||
|
||||
|
||||
def _comment_continuation_items(data: dict) -> list[Any]:
|
||||
items: list[Any] = []
|
||||
for key in ("reloadContinuationItemsCommand", "appendContinuationItemsAction"):
|
||||
for action in find_all(data, key):
|
||||
if isinstance(action, dict):
|
||||
items += action.get("continuationItems") or []
|
||||
return items
|
||||
|
||||
|
||||
def comment_next_token(data: dict) -> str | None:
|
||||
"""Pagination token: the trailing bare ``continuationItemRenderer``.
|
||||
|
||||
Reply loaders are nested inside ``commentThreadRenderer``s; the page's own
|
||||
"load more" token is the last top-level continuation item, so scan the
|
||||
action's ``continuationItems`` from the end.
|
||||
"""
|
||||
for item in reversed(_comment_continuation_items(data)):
|
||||
if isinstance(item, dict) and "continuationItemRenderer" in item:
|
||||
token = dig(
|
||||
item,
|
||||
"continuationItemRenderer",
|
||||
"continuationEndpoint",
|
||||
"continuationCommand",
|
||||
"token",
|
||||
)
|
||||
if token:
|
||||
return token
|
||||
return None
|
||||
|
||||
|
||||
def comment_reply_tokens(data: dict) -> dict[str, str]:
|
||||
"""Map each top-level comment id to its replies continuation token."""
|
||||
tokens: dict[str, str] = {}
|
||||
for thread in find_all(data, "commentThreadRenderer"):
|
||||
cid = dig(thread, "commentViewModel", "commentViewModel", "commentId")
|
||||
token = dig(
|
||||
find_first(thread, "continuationItemRenderer"),
|
||||
"continuationEndpoint",
|
||||
"continuationCommand",
|
||||
"token",
|
||||
)
|
||||
if cid and token:
|
||||
tokens[cid] = token
|
||||
return tokens
|
||||
|
||||
|
||||
def parse_comment_entity(cep: dict) -> dict[str, Any]:
|
||||
"""Map one ``commentEntityPayload`` to a partial CommentItem dict."""
|
||||
props = cep.get("properties") or {}
|
||||
author = cep.get("author") or {}
|
||||
toolbar = cep.get("toolbar") or {}
|
||||
return {
|
||||
"cid": props.get("commentId"),
|
||||
"comment": dig(props, "content", "content"),
|
||||
"author": author.get("displayName"),
|
||||
"publishedTimeText": props.get("publishedTime"),
|
||||
"voteCount": parse_count(toolbar.get("likeCountNotliked")),
|
||||
"replyCount": parse_count(toolbar.get("replyCount")),
|
||||
"authorIsChannelOwner": bool(author.get("isCreator")),
|
||||
# A creator heart attaches the channel owner's avatar to the toolbar.
|
||||
"hasCreatorHeart": bool(toolbar.get("creatorThumbnailUrl")),
|
||||
"type": "comment" if (props.get("replyLevel") or 0) == 0 else "reply",
|
||||
}
|
||||
|
||||
|
||||
def parse_comment_entities(data: dict) -> list[dict[str, Any]]:
|
||||
"""All comment payloads in a /next response, in display order."""
|
||||
return [
|
||||
parse_comment_entity(cep)
|
||||
for cep in find_all(data, "commentEntityPayload")
|
||||
if isinstance(cep, dict)
|
||||
]
|
||||
|
||||
|
||||
def parse_channel_sort_tokens(initial_data: dict) -> dict[str, str]:
|
||||
"""Map channel-videos sort labels ("Latest"/"Popular"/"Oldest") to tokens."""
|
||||
tokens: dict[str, str] = {}
|
||||
for chip in find_all(initial_data, "chipViewModel"):
|
||||
if not isinstance(chip, dict):
|
||||
continue
|
||||
label = chip.get("text")
|
||||
token = dig(
|
||||
chip, "tapCommand", "innertubeCommand", "continuationCommand", "token"
|
||||
)
|
||||
if label and token:
|
||||
tokens[label] = token
|
||||
return tokens
|
||||
210
surfsense_backend/app/scrapers/youtube/schemas.py
Normal file
210
surfsense_backend/app/scrapers/youtube/schemas.py
Normal file
|
|
@ -0,0 +1,210 @@
|
|||
# ruff: noqa: N815 - field names intentionally mirror the Apify camelCase spec
|
||||
"""Apify-compatible input/output models for the YouTube scraper.
|
||||
|
||||
The models mirror the public Apify "YouTube Scraper" actor spec so the endpoint
|
||||
can be a drop-in. The MVP populates a reliably-sourced subset; every other field
|
||||
is accepted (input) or emitted as ``None``/``[]`` (output) so parity is additive.
|
||||
|
||||
``VideoItem`` uses ``extra="allow"`` on purpose: the Apify spec explicitly says
|
||||
"Other properties may be included", and it lets us grow the output shape without
|
||||
breaking existing consumers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
SubtitlesFormat = Literal["srt", "vtt", "xml", "plaintext"]
|
||||
SortingOrder = Literal["relevance", "rating", "date", "views"]
|
||||
DateFilter = Literal["hour", "today", "week", "month", "year"]
|
||||
VideoTypeFilter = Literal["video", "movie"]
|
||||
LengthFilter = Literal["under4", "between420", "plus20"]
|
||||
SortVideosBy = Literal["NEWEST", "POPULAR", "OLDEST"]
|
||||
VideoContentType = Literal["video", "shorts", "stream"]
|
||||
SortCommentsBy = Literal["TOP_COMMENTS", "NEWEST_FIRST"]
|
||||
CommentType = Literal["comment", "reply"]
|
||||
|
||||
|
||||
class StartUrl(BaseModel):
|
||||
"""A single direct URL entry (Apify passes ``{"url": ...}`` objects)."""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
url: str
|
||||
|
||||
|
||||
class YouTubeScrapeInput(BaseModel):
|
||||
"""Full Apify input surface.
|
||||
|
||||
``maxResults*`` semantics follow Apify: ``0`` means "fetch none of that type"
|
||||
(see the ``startUrls`` note in the spec), NOT unlimited. A caller wanting an
|
||||
unbounded run leaves the value high; the request-time guard lives in the
|
||||
route, not here.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
# Discovery
|
||||
searchQueries: list[str] = Field(default_factory=list)
|
||||
startUrls: list[StartUrl] = Field(default_factory=list)
|
||||
|
||||
# Per-type caps (independent, applied per search term and per channel)
|
||||
maxResults: int = Field(default=0, ge=0, le=999999)
|
||||
maxResultsShorts: int = Field(default=0, ge=0, le=999999)
|
||||
maxResultStreams: int = Field(default=0, ge=0, le=999999)
|
||||
|
||||
# Subtitles
|
||||
downloadSubtitles: bool = False
|
||||
subtitlesLanguage: str = "en"
|
||||
subtitlesFormat: SubtitlesFormat = "srt"
|
||||
saveSubsToKVS: bool = False
|
||||
preferAutoGeneratedSubtitles: bool = False
|
||||
|
||||
# Search filters
|
||||
sortingOrder: SortingOrder | None = None
|
||||
dateFilter: DateFilter | None = None
|
||||
videoType: VideoTypeFilter | None = None
|
||||
lengthFilter: LengthFilter | None = None
|
||||
isHD: bool | None = None
|
||||
hasSubtitles: bool | None = None
|
||||
hasCC: bool | None = None
|
||||
is3D: bool | None = None
|
||||
isLive: bool | None = None
|
||||
isBought: bool | None = None
|
||||
is4K: bool | None = None
|
||||
is360: bool | None = None
|
||||
hasLocation: bool | None = None
|
||||
isHDR: bool | None = None
|
||||
isVR180: bool | None = None
|
||||
|
||||
# Channel/date scoping
|
||||
oldestPostDate: str | None = None
|
||||
sortVideosBy: SortVideosBy | None = None
|
||||
|
||||
|
||||
class DescriptionLink(BaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
url: str | None = None
|
||||
text: str | None = None
|
||||
|
||||
|
||||
class SubtitleTrack(BaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
srtUrl: str | None = None
|
||||
type: str | None = None
|
||||
language: str | None = None
|
||||
srt: str | None = None
|
||||
|
||||
|
||||
class Collaborator(BaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
name: str | None = None
|
||||
username: str | None = None
|
||||
url: str | None = None
|
||||
|
||||
|
||||
class VideoItem(BaseModel):
|
||||
"""Apify output item. Unsourced fields default to ``None``/``[]``.
|
||||
|
||||
``extra="allow"`` keeps the contract open (Apify: "Other properties may be
|
||||
included") so gradually-added fields never break existing consumers.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
# Core video
|
||||
title: str | None = None
|
||||
id: str | None = None
|
||||
url: str | None = None
|
||||
viewCount: int | None = None
|
||||
date: str | None = None
|
||||
duration: str | None = None
|
||||
type: VideoContentType = "video"
|
||||
thumbnailUrl: str | None = None
|
||||
input: str | None = None
|
||||
fromYTUrl: str | None = None
|
||||
order: int | None = None
|
||||
|
||||
# Engagement / content
|
||||
likes: int | None = None
|
||||
commentsCount: int | None = None
|
||||
commentsTurnedOff: bool | None = None
|
||||
text: str | None = None
|
||||
descriptionLinks: list[DescriptionLink] | None = None
|
||||
subtitles: list[SubtitleTrack] | None = None
|
||||
hashtags: list[str] = Field(default_factory=list)
|
||||
location: str | None = None
|
||||
|
||||
# Translations
|
||||
translatedTitle: str | None = None
|
||||
translatedText: str | None = None
|
||||
|
||||
# Video flags
|
||||
isMonetized: bool | None = None
|
||||
isMembersOnly: bool = False
|
||||
isPaidContent: bool = False
|
||||
isAgeRestricted: bool | None = None
|
||||
collaborators: list[Collaborator] | None = None
|
||||
|
||||
# Channel
|
||||
channelName: str | None = None
|
||||
channelUrl: str | None = None
|
||||
channelUsername: str | None = None
|
||||
channelId: str | None = None
|
||||
numberOfSubscribers: int | None = None
|
||||
channelTotalVideos: int | None = None
|
||||
channelDescription: str | None = None
|
||||
channelLocation: str | None = None
|
||||
channelJoinedDate: str | None = None
|
||||
channelTotalViews: int | None = None
|
||||
isChannelVerified: bool | None = None
|
||||
channelBannerUrl: str | None = None
|
||||
channelAvatarUrl: str | None = None
|
||||
|
||||
def to_output(self) -> dict[str, Any]:
|
||||
"""Serialize to the flat dict shape Apify emits (keeps extras)."""
|
||||
return self.model_dump(exclude_none=False)
|
||||
|
||||
|
||||
class YouTubeCommentsInput(BaseModel):
|
||||
"""Apify "YouTube Comments Scraper" input surface.
|
||||
|
||||
``maxComments`` counts every emitted item (top-level comments and replies).
|
||||
Setting ``oldestCommentDate`` forces newest-first sort, matching Apify.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
startUrls: list[StartUrl] = Field(default_factory=list)
|
||||
maxComments: int = Field(default=1, ge=1, le=999999)
|
||||
sortCommentsBy: SortCommentsBy = "NEWEST_FIRST"
|
||||
oldestCommentDate: str | None = None
|
||||
|
||||
|
||||
class CommentItem(BaseModel):
|
||||
"""Apify comment output item (one per top-level comment or reply)."""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
author: str | None = None
|
||||
comment: str | None = None
|
||||
cid: str | None = None
|
||||
replyCount: int | None = None
|
||||
replyToCid: str | None = None
|
||||
voteCount: int | None = None
|
||||
authorIsChannelOwner: bool | None = None
|
||||
publishedTimeText: str | None = None
|
||||
type: CommentType | None = None
|
||||
hasCreatorHeart: bool | None = None
|
||||
videoId: str | None = None
|
||||
pageUrl: str | None = None
|
||||
commentsCount: int | None = None
|
||||
title: str | None = None
|
||||
|
||||
def to_output(self) -> dict[str, Any]:
|
||||
return self.model_dump(exclude_none=False)
|
||||
493
surfsense_backend/app/scrapers/youtube/scraper.py
Normal file
493
surfsense_backend/app/scrapers/youtube/scraper.py
Normal file
|
|
@ -0,0 +1,493 @@
|
|||
"""Orchestrator for the YouTube scraper.
|
||||
|
||||
The core is the async generator :func:`iter_youtube` (unbounded / continuation
|
||||
paged); :func:`scrape_youtube` is a thin collector with a caller-supplied
|
||||
``limit`` guard. Per-type counters (regular / shorts / streams) are applied
|
||||
independently per search term and per channel, matching Apify semantics. Any cap
|
||||
is caller policy, never baked into flow logic.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
from .innertube import (
|
||||
INNERTUBE_BROWSE_URL,
|
||||
INNERTUBE_NEXT_URL,
|
||||
INNERTUBE_PUBLIC_API_KEY,
|
||||
INNERTUBE_SEARCH_URL,
|
||||
bind_proxy_holder,
|
||||
build_innertube_payload,
|
||||
fetch_html,
|
||||
open_proxy_holder,
|
||||
post_innertube,
|
||||
)
|
||||
from .parsers import (
|
||||
channel_about_tokens,
|
||||
extract_yt_initial_data,
|
||||
find_first,
|
||||
parse_channel_about,
|
||||
parse_channel_metadata,
|
||||
parse_channel_shorts,
|
||||
parse_channel_sort_tokens,
|
||||
parse_channel_videos,
|
||||
parse_playlist_video_ids,
|
||||
parse_search_response,
|
||||
parse_translation,
|
||||
parse_video_page,
|
||||
)
|
||||
from .schemas import VideoItem, YouTubeScrapeInput
|
||||
from .search_filters import build_search_params
|
||||
from .subtitles import fetch_subtitles
|
||||
from .url_resolver import ResolvedUrl, resolve_url
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SORT_LABELS = {"NEWEST": "Latest", "POPULAR": "Popular", "OLDEST": "Oldest"}
|
||||
|
||||
# Independent jobs (one per startUrl / search query / video) run concurrently on
|
||||
# a pool of warm proxy sessions (sticky IPs). A ramp probe on the gateway ran 64
|
||||
# parallel flows with zero failures, so the proxy is not the ceiling; 16 workers
|
||||
# saturate typical job counts while leaving gateway headroom for other callers.
|
||||
_FANOUT_CONCURRENCY = 16
|
||||
|
||||
|
||||
async def fan_out(
|
||||
jobs: list[AsyncIterator[dict[str, Any]]], *, concurrency: int = _FANOUT_CONCURRENCY
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
"""Stream items from independent async-iterator jobs via a warm worker pool.
|
||||
|
||||
Each worker opens ONE proxy session and reuses it across the sequential jobs
|
||||
it pulls, so only the first job per worker pays the ~2s proxy TCP+TLS
|
||||
handshake. A bad job yields nothing rather than aborting the batch; results
|
||||
stream out as each job finishes. Workers are cancelled if the consumer stops
|
||||
early (e.g. the collector hits its limit).
|
||||
"""
|
||||
if not jobs:
|
||||
return
|
||||
job_queue: asyncio.Queue[AsyncIterator[dict[str, Any]]] = asyncio.Queue()
|
||||
for job in jobs:
|
||||
job_queue.put_nowait(job)
|
||||
results: asyncio.Queue[list[dict[str, Any]]] = asyncio.Queue()
|
||||
|
||||
async def worker() -> None:
|
||||
holder = None
|
||||
try:
|
||||
holder = await open_proxy_holder()
|
||||
except Exception as e: # no session: jobs still run via one-shot fetches
|
||||
logger.warning("[youtube] proxy session open failed: %s", e)
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
job = job_queue.get_nowait()
|
||||
except asyncio.QueueEmpty:
|
||||
return
|
||||
items: list[dict[str, Any]] = []
|
||||
try:
|
||||
if holder is not None:
|
||||
async with bind_proxy_holder(holder):
|
||||
items = [item async for item in job]
|
||||
else:
|
||||
items = [item async for item in job]
|
||||
except Exception as e: # one bad video/URL must not kill the run
|
||||
logger.warning("[youtube] fan-out job failed: %s", e)
|
||||
await results.put(items)
|
||||
finally:
|
||||
if holder is not None:
|
||||
await holder.close()
|
||||
|
||||
tasks = [
|
||||
asyncio.create_task(worker()) for _ in range(min(concurrency, len(jobs)))
|
||||
]
|
||||
try:
|
||||
for _ in range(len(jobs)):
|
||||
for item in await results.get():
|
||||
yield item
|
||||
finally:
|
||||
for task in tasks:
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
# Await cancellation so each worker's finally closes its session before
|
||||
# we return — no leaked keep-alive connections when the consumer stops
|
||||
# early (e.g. the collector hit its limit).
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
|
||||
async def _post(url: str, payload: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""POST to InnerTube, retrying with the public web key if keyless fails.
|
||||
|
||||
ponytail: retries with the key only when the keyless call returns nothing;
|
||||
could remember which one worked to avoid the extra request.
|
||||
"""
|
||||
data = await post_innertube(url, payload)
|
||||
if data is None:
|
||||
data = await post_innertube(url, payload, api_key=INNERTUBE_PUBLIC_API_KEY)
|
||||
return data
|
||||
|
||||
|
||||
async def _finalize(
|
||||
partial: dict[str, Any],
|
||||
*,
|
||||
input_model: YouTubeScrapeInput,
|
||||
source_input: str | None,
|
||||
from_url: str | None,
|
||||
order: int,
|
||||
content_type: str,
|
||||
) -> dict[str, Any]:
|
||||
item = VideoItem(**partial)
|
||||
item.type = content_type # type: ignore[assignment]
|
||||
item.input = source_input
|
||||
item.fromYTUrl = from_url
|
||||
item.order = order
|
||||
if input_model.downloadSubtitles and item.id:
|
||||
item.subtitles = await fetch_subtitles(
|
||||
item.id,
|
||||
language=input_model.subtitlesLanguage,
|
||||
fmt=input_model.subtitlesFormat,
|
||||
prefer_generated=input_model.preferAutoGeneratedSubtitles,
|
||||
)
|
||||
# translatedTitle/Text: one extra /next in the requested language. ponytail:
|
||||
# gated on a non-English subtitlesLanguage so default runs pay nothing; costs
|
||||
# one request per item when a translation language is set.
|
||||
lang = input_model.subtitlesLanguage
|
||||
if item.id and lang and lang != "en":
|
||||
data = await _post(
|
||||
INNERTUBE_NEXT_URL, build_innertube_payload(video_id=item.id, hl=lang)
|
||||
)
|
||||
if data:
|
||||
item.translatedTitle, item.translatedText = parse_translation(data)
|
||||
return item.to_output()
|
||||
|
||||
|
||||
async def _video_flow(
|
||||
video_id: str,
|
||||
*,
|
||||
input_model: YouTubeScrapeInput,
|
||||
source_input: str | None,
|
||||
from_url: str | None,
|
||||
order: int,
|
||||
content_type: str,
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
url = f"https://www.youtube.com/watch?v={video_id}"
|
||||
html = await fetch_html(url)
|
||||
if not html:
|
||||
return
|
||||
partial = parse_video_page(html)
|
||||
if not partial:
|
||||
return
|
||||
yield await _finalize(
|
||||
partial,
|
||||
input_model=input_model,
|
||||
source_input=source_input,
|
||||
from_url=from_url,
|
||||
order=order,
|
||||
content_type=content_type,
|
||||
)
|
||||
|
||||
|
||||
async def _search_flow(
|
||||
query: str,
|
||||
*,
|
||||
input_model: YouTubeScrapeInput,
|
||||
source_input: str,
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
limit = input_model.maxResults
|
||||
if limit <= 0:
|
||||
return
|
||||
from_url = f"https://www.youtube.com/results?search_query={quote(query)}"
|
||||
params = build_search_params(input_model)
|
||||
payload = build_innertube_payload(search_query=query, search_params=params)
|
||||
data = await _post(INNERTUBE_SEARCH_URL, payload)
|
||||
|
||||
order = 0
|
||||
while data:
|
||||
items, token = parse_search_response(data)
|
||||
for it in items:
|
||||
if order >= limit:
|
||||
return
|
||||
yield await _finalize(
|
||||
it,
|
||||
input_model=input_model,
|
||||
source_input=source_input,
|
||||
from_url=from_url,
|
||||
order=order,
|
||||
content_type="video",
|
||||
)
|
||||
order += 1
|
||||
if not token or order >= limit:
|
||||
return
|
||||
data = await _post(
|
||||
INNERTUBE_SEARCH_URL, build_innertube_payload(continuation_token=token)
|
||||
)
|
||||
|
||||
|
||||
def _channel_tab_url(handle: str, tab: str) -> str:
|
||||
if handle.startswith("UC") and len(handle) > 10:
|
||||
return f"https://www.youtube.com/channel/{handle}/{tab}"
|
||||
return f"https://www.youtube.com/@{handle}/{tab}"
|
||||
|
||||
|
||||
# tab path -> (content type, list parser). Streams share the video lockup shape.
|
||||
_CHANNEL_TABS = {
|
||||
"videos": ("video", parse_channel_videos),
|
||||
"shorts": ("shorts", parse_channel_shorts),
|
||||
"streams": ("stream", parse_channel_videos),
|
||||
}
|
||||
|
||||
|
||||
async def _fetch_channel_about(initial: dict) -> dict[str, Any]:
|
||||
"""One ``/browse`` call for the About panel (deep channel fields).
|
||||
|
||||
Panels are unlabeled, so try each engagement token and keep the first that
|
||||
returns an ``aboutChannelViewModel``. ponytail: worst case is one extra
|
||||
no-op browse before the hit; a labeled-panel signal would remove it.
|
||||
"""
|
||||
for token in channel_about_tokens(initial):
|
||||
data = await _post(
|
||||
INNERTUBE_BROWSE_URL, build_innertube_payload(continuation_token=token)
|
||||
)
|
||||
about = find_first(data, "aboutChannelViewModel") if data else None
|
||||
if about:
|
||||
return parse_channel_about(about)
|
||||
return {}
|
||||
|
||||
|
||||
def _published_date(text: str | None):
|
||||
"""Parse a relative/absolute time string to a ``date`` (best-effort).
|
||||
|
||||
ponytail: channel list pages only expose coarse relative times ("2 years
|
||||
ago"), so the ``oldestPostDate`` cutoff is day-accurate at best.
|
||||
"""
|
||||
if not text:
|
||||
return None
|
||||
import dateparser
|
||||
|
||||
dt = dateparser.parse(text)
|
||||
return dt.date() if dt else None
|
||||
|
||||
|
||||
async def _channel_tab_flow(
|
||||
handle: str,
|
||||
tab: str,
|
||||
*,
|
||||
limit: int,
|
||||
input_model: YouTubeScrapeInput,
|
||||
source_input: str,
|
||||
channel_meta: dict[str, Any],
|
||||
initial: dict | None = None,
|
||||
cutoff=None,
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
"""Page one channel tab (videos/shorts/streams) up to ``limit`` items.
|
||||
|
||||
Videos honor ``sortVideosBy`` via the sort chips (re-fetch sorted from the
|
||||
start); shorts/streams page straight from the seed's first page + its
|
||||
continuation, since those tabs don't expose the same sort chips. ``initial``
|
||||
may be prefetched (videos tab) to avoid re-downloading the seed.
|
||||
"""
|
||||
if limit <= 0:
|
||||
return
|
||||
content_type, parse_fn = _CHANNEL_TABS[tab]
|
||||
from_url = _channel_tab_url(handle, tab)
|
||||
if initial is None:
|
||||
seed_html = await fetch_html(from_url)
|
||||
if not seed_html:
|
||||
return
|
||||
initial = extract_yt_initial_data(seed_html)
|
||||
if not initial:
|
||||
return
|
||||
|
||||
# Videos: prefer a sort chip token (fetches page 1 sorted). Otherwise parse
|
||||
# the seed's first page directly and follow its continuation.
|
||||
items: list[dict[str, Any]] = []
|
||||
token: str | None = None
|
||||
if tab == "videos":
|
||||
tokens = parse_channel_sort_tokens(initial)
|
||||
label = _SORT_LABELS.get(input_model.sortVideosBy or "NEWEST", "Latest")
|
||||
token = tokens.get(label) or next(iter(tokens.values()), None)
|
||||
if token is None:
|
||||
items, token = parse_fn(initial)
|
||||
|
||||
order = 0
|
||||
while order < limit:
|
||||
for it in items:
|
||||
if order >= limit:
|
||||
return
|
||||
# Newest-first ordering: once we pass the cutoff, the rest are older.
|
||||
if cutoff is not None:
|
||||
item_date = _published_date(it.get("publishedTimeText"))
|
||||
if item_date is not None and item_date < cutoff:
|
||||
return
|
||||
it.setdefault("channelUsername", handle)
|
||||
for key, value in channel_meta.items():
|
||||
it.setdefault(key, value)
|
||||
yield await _finalize(
|
||||
it,
|
||||
input_model=input_model,
|
||||
source_input=source_input,
|
||||
from_url=from_url,
|
||||
order=order,
|
||||
content_type=content_type,
|
||||
)
|
||||
order += 1
|
||||
if not token:
|
||||
return
|
||||
data = await _post(
|
||||
INNERTUBE_BROWSE_URL, build_innertube_payload(continuation_token=token)
|
||||
)
|
||||
if not data:
|
||||
return
|
||||
items, token = parse_fn(data)
|
||||
if not items:
|
||||
return
|
||||
|
||||
|
||||
async def _channel_flow(
|
||||
handle: str,
|
||||
*,
|
||||
input_model: YouTubeScrapeInput,
|
||||
source_input: str,
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
"""Scrape a channel's videos, shorts, and streams — each capped independently.
|
||||
|
||||
The videos seed is fetched once and reused to derive channel-wide metadata
|
||||
(identity, banner, and the About panel's deep fields) stamped on every item.
|
||||
"""
|
||||
videos_seed = await fetch_html(_channel_tab_url(handle, "videos"))
|
||||
initial = extract_yt_initial_data(videos_seed) if videos_seed else None
|
||||
channel_meta: dict[str, Any] = {}
|
||||
if initial:
|
||||
channel_meta = parse_channel_metadata(initial)
|
||||
channel_meta.update(await _fetch_channel_about(initial))
|
||||
|
||||
cutoff = _published_date(input_model.oldestPostDate)
|
||||
|
||||
for tab, limit in (
|
||||
("videos", input_model.maxResults),
|
||||
("shorts", input_model.maxResultsShorts),
|
||||
("streams", input_model.maxResultStreams),
|
||||
):
|
||||
async for item in _channel_tab_flow(
|
||||
handle,
|
||||
tab,
|
||||
limit=limit,
|
||||
input_model=input_model,
|
||||
source_input=source_input,
|
||||
channel_meta=channel_meta,
|
||||
initial=initial if tab == "videos" else None,
|
||||
cutoff=cutoff,
|
||||
):
|
||||
yield item
|
||||
|
||||
|
||||
async def _playlist_flow(
|
||||
playlist_id: str,
|
||||
*,
|
||||
input_model: YouTubeScrapeInput,
|
||||
source_input: str,
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
limit = input_model.maxResults
|
||||
if limit <= 0:
|
||||
return
|
||||
data = await _post(
|
||||
INNERTUBE_BROWSE_URL, build_innertube_payload(browse_id=f"VL{playlist_id}")
|
||||
)
|
||||
if not data:
|
||||
return
|
||||
# ponytail: reads the first browse page only (~100 items). Upgrade path:
|
||||
# follow the continuation token for playlists longer than one page.
|
||||
video_ids = parse_playlist_video_ids(data)
|
||||
|
||||
for order, vid in enumerate(video_ids):
|
||||
if order >= limit:
|
||||
return
|
||||
async for item in _video_flow(
|
||||
vid,
|
||||
input_model=input_model,
|
||||
source_input=source_input,
|
||||
from_url=source_input,
|
||||
order=order,
|
||||
content_type="video",
|
||||
):
|
||||
yield item
|
||||
|
||||
|
||||
async def _dispatch(
|
||||
resolved: ResolvedUrl, input_model: YouTubeScrapeInput
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
if resolved.kind == "video":
|
||||
content_type = "shorts" if "/shorts/" in resolved.url else "video"
|
||||
async for item in _video_flow(
|
||||
resolved.value,
|
||||
input_model=input_model,
|
||||
source_input=resolved.url,
|
||||
from_url=resolved.url,
|
||||
order=0,
|
||||
content_type=content_type,
|
||||
):
|
||||
yield item
|
||||
elif resolved.kind == "channel":
|
||||
async for item in _channel_flow(
|
||||
resolved.value, input_model=input_model, source_input=resolved.url
|
||||
):
|
||||
yield item
|
||||
elif resolved.kind == "playlist":
|
||||
async for item in _playlist_flow(
|
||||
resolved.value, input_model=input_model, source_input=resolved.url
|
||||
):
|
||||
yield item
|
||||
elif resolved.kind in ("search", "hashtag"):
|
||||
# ponytail: hashtag pages have their own layout; MVP routes the tag
|
||||
# through search (query "#tag"). Upgrade path: dedicated hashtag browse.
|
||||
query = resolved.value if resolved.kind == "search" else f"#{resolved.value}"
|
||||
async for item in _search_flow(
|
||||
query, input_model=input_model, source_input=resolved.url
|
||||
):
|
||||
yield item
|
||||
|
||||
|
||||
async def iter_youtube(
|
||||
input_model: YouTubeScrapeInput,
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
"""Yield Apify-shaped video items. startUrls override searchQueries.
|
||||
|
||||
Independent startUrls / queries fan out concurrently; each flow's own
|
||||
continuation paging stays sequential.
|
||||
"""
|
||||
if input_model.startUrls:
|
||||
jobs = []
|
||||
for entry in input_model.startUrls:
|
||||
resolved = resolve_url(entry.url)
|
||||
if not resolved:
|
||||
logger.warning("Unrecognized YouTube URL: %s", entry.url)
|
||||
continue
|
||||
jobs.append(_dispatch(resolved, input_model))
|
||||
async for item in fan_out(jobs):
|
||||
yield item
|
||||
return
|
||||
|
||||
jobs = [
|
||||
_search_flow(query, input_model=input_model, source_input=query)
|
||||
for query in input_model.searchQueries
|
||||
]
|
||||
async for item in fan_out(jobs):
|
||||
yield item
|
||||
|
||||
|
||||
async def scrape_youtube(
|
||||
input_model: YouTubeScrapeInput, *, limit: int | None = None
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Collect :func:`iter_youtube` into a list, honoring an optional ``limit``.
|
||||
|
||||
``limit`` is a request-time policy guard (used by the route), NOT a ceiling
|
||||
in the streaming core.
|
||||
"""
|
||||
results: list[dict[str, Any]] = []
|
||||
async for item in iter_youtube(input_model):
|
||||
results.append(item)
|
||||
if limit is not None and len(results) >= limit:
|
||||
break
|
||||
return results
|
||||
92
surfsense_backend/app/scrapers/youtube/search_filters.py
Normal file
92
surfsense_backend/app/scrapers/youtube/search_filters.py
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
"""Encode Apify search filters into YouTube's ``sp=`` (InnerTube ``params``).
|
||||
|
||||
YouTube encodes search filters as a base64 protobuf. The message is::
|
||||
|
||||
SearchParams {
|
||||
1: sortOrder (varint enum)
|
||||
2: Filters {
|
||||
1: uploadDate (varint enum)
|
||||
2: type (varint enum)
|
||||
3: duration (varint enum)
|
||||
<feature fields>: bool (varint 1)
|
||||
}
|
||||
}
|
||||
|
||||
Rather than a lookup table of single tokens, we build the protobuf directly so
|
||||
arbitrary combinations (sort + date + type + length + any boolean features)
|
||||
compose correctly. The field numbers/enums below are verified to reproduce
|
||||
YouTube's own standalone tokens byte-for-byte (see the unit test).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
|
||||
_SORT_ORDER = {"relevance": 0, "rating": 1, "date": 2, "views": 3}
|
||||
_UPLOAD_DATE = {"hour": 1, "today": 2, "week": 3, "month": 4, "year": 5}
|
||||
_TYPE = {"video": 1, "channel": 2, "playlist": 3, "movie": 4}
|
||||
_DURATION = {"under4": 1, "plus20": 2, "between420": 3}
|
||||
|
||||
# Apify boolean flag -> Filters feature field number.
|
||||
_FEATURES = {
|
||||
"isHD": 4,
|
||||
"hasSubtitles": 5,
|
||||
"hasCC": 6,
|
||||
"is3D": 7,
|
||||
"isLive": 8,
|
||||
"isBought": 9,
|
||||
"is4K": 14,
|
||||
"is360": 15,
|
||||
"hasLocation": 23,
|
||||
"isHDR": 25,
|
||||
"isVR180": 26,
|
||||
}
|
||||
|
||||
|
||||
def _varint(value: int) -> bytes:
|
||||
out = bytearray()
|
||||
while True:
|
||||
byte = value & 0x7F
|
||||
value >>= 7
|
||||
out.append(byte | (0x80 if value else 0))
|
||||
if not value:
|
||||
return bytes(out)
|
||||
|
||||
|
||||
def _tag_varint(field: int, value: int) -> bytes:
|
||||
"""A protobuf ``field: varint`` entry (wire type 0)."""
|
||||
return _varint(field << 3) + _varint(value)
|
||||
|
||||
|
||||
def _tag_message(field: int, data: bytes) -> bytes:
|
||||
"""A protobuf ``field: message`` entry (wire type 2, length-delimited)."""
|
||||
return _varint((field << 3) | 2) + _varint(len(data)) + data
|
||||
|
||||
|
||||
def build_search_params(input_model) -> str | None:
|
||||
"""Encode the input's filters into a base64 ``sp=`` token, or ``None``.
|
||||
|
||||
All set filters compose into a single protobuf; ``None`` when nothing is set
|
||||
(so the caller omits ``params`` entirely).
|
||||
"""
|
||||
filters = bytearray()
|
||||
if input_model.dateFilter in _UPLOAD_DATE:
|
||||
filters += _tag_varint(1, _UPLOAD_DATE[input_model.dateFilter])
|
||||
if input_model.videoType in _TYPE:
|
||||
filters += _tag_varint(2, _TYPE[input_model.videoType])
|
||||
if input_model.lengthFilter in _DURATION:
|
||||
filters += _tag_varint(3, _DURATION[input_model.lengthFilter])
|
||||
for flag, field in _FEATURES.items():
|
||||
if getattr(input_model, flag, None):
|
||||
filters += _tag_varint(field, 1)
|
||||
|
||||
message = bytearray()
|
||||
sort = _SORT_ORDER.get(input_model.sortingOrder or "")
|
||||
if sort: # relevance (0) is the default, so it needs no bytes
|
||||
message += _tag_varint(1, sort)
|
||||
if filters:
|
||||
message += _tag_message(2, bytes(filters))
|
||||
|
||||
if not message:
|
||||
return None
|
||||
return base64.b64encode(bytes(message)).decode("ascii")
|
||||
98
surfsense_backend/app/scrapers/youtube/subtitles.py
Normal file
98
surfsense_backend/app/scrapers/youtube/subtitles.py
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
"""Subtitle download via ``youtube-transcript-api``, shaped to Apify ``subtitles[]``.
|
||||
|
||||
Uses the library's built-in formatters (no hand-rolled srt/vtt conversion). The
|
||||
transcript API has no XML formatter, so ``xml`` falls back to the raw snippet
|
||||
data. Fetch/proxy wiring mirrors ``youtube_processor.py``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from fake_useragent import UserAgent
|
||||
from requests import Session
|
||||
from youtube_transcript_api import YouTubeTranscriptApi
|
||||
from youtube_transcript_api.formatters import (
|
||||
SRTFormatter,
|
||||
TextFormatter,
|
||||
WebVTTFormatter,
|
||||
)
|
||||
|
||||
from app.utils.proxy import get_requests_proxies
|
||||
|
||||
from .schemas import SubtitleTrack
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_FORMATTERS = {
|
||||
"srt": SRTFormatter,
|
||||
"vtt": WebVTTFormatter,
|
||||
"plaintext": TextFormatter,
|
||||
}
|
||||
|
||||
|
||||
def _build_client() -> YouTubeTranscriptApi:
|
||||
http_client = Session()
|
||||
http_client.headers.update({"User-Agent": UserAgent().random})
|
||||
proxies = get_requests_proxies()
|
||||
if proxies:
|
||||
http_client.proxies.update(proxies)
|
||||
return YouTubeTranscriptApi(http_client=http_client)
|
||||
|
||||
|
||||
def _select_transcript(transcript_list, language: str, prefer_generated: bool):
|
||||
"""Pick a transcript honoring language + generated/manual preference."""
|
||||
# ``any`` means take whatever the video offers first.
|
||||
if language == "any":
|
||||
return next(iter(transcript_list))
|
||||
|
||||
codes = [language]
|
||||
if prefer_generated:
|
||||
try:
|
||||
return transcript_list.find_generated_transcript(codes)
|
||||
except Exception:
|
||||
return transcript_list.find_transcript(codes)
|
||||
return transcript_list.find_transcript(codes)
|
||||
|
||||
|
||||
def _fetch_subtitles_sync(video_id: str, language: str, fmt: str, prefer_generated: bool):
|
||||
api = _build_client()
|
||||
transcript_list = api.list(video_id)
|
||||
transcript = _select_transcript(transcript_list, language, prefer_generated)
|
||||
fetched = transcript.fetch()
|
||||
|
||||
if fmt == "xml":
|
||||
# No XML formatter in the library; emit the raw snippet data as text.
|
||||
body = str(fetched.to_raw_data())
|
||||
else:
|
||||
formatter_cls = _FORMATTERS.get(fmt, SRTFormatter)
|
||||
body = formatter_cls().format_transcript(fetched)
|
||||
|
||||
return SubtitleTrack(
|
||||
srtUrl=None,
|
||||
type="auto_generated" if transcript.is_generated else "user_generated",
|
||||
language=transcript.language_code,
|
||||
srt=body,
|
||||
)
|
||||
|
||||
|
||||
async def fetch_subtitles(
|
||||
video_id: str,
|
||||
*,
|
||||
language: str = "en",
|
||||
fmt: str = "srt",
|
||||
prefer_generated: bool = False,
|
||||
) -> list[SubtitleTrack] | None:
|
||||
"""Return the Apify ``subtitles[]`` list for a video, or ``None`` if none.
|
||||
|
||||
Runs the blocking transcript API in a worker thread to stay async-friendly.
|
||||
"""
|
||||
try:
|
||||
track = await asyncio.to_thread(
|
||||
_fetch_subtitles_sync, video_id, language, fmt, prefer_generated
|
||||
)
|
||||
return [track]
|
||||
except Exception as e:
|
||||
logger.info("No subtitles for video %s (%s): %s", video_id, language, e)
|
||||
return None
|
||||
85
surfsense_backend/app/scrapers/youtube/url_resolver.py
Normal file
85
surfsense_backend/app/scrapers/youtube/url_resolver.py
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
"""Classify a YouTube URL and extract its identifier.
|
||||
|
||||
Covers the ``startUrls`` shapes the Apify spec accepts: video, channel,
|
||||
playlist, hashtag, and search-results pages. Video-ID extraction reuses the
|
||||
logic already in ``app/tasks/document_processors/youtube_processor.py``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from typing import Literal
|
||||
from urllib.parse import parse_qs, unquote, urlparse
|
||||
|
||||
ResolvedKind = Literal["video", "channel", "playlist", "hashtag", "search"]
|
||||
|
||||
_PLAYLIST_ID_RE = re.compile(r"[?&]list=([\w-]+)")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResolvedUrl:
|
||||
kind: ResolvedKind
|
||||
value: str # video id, channel handle/id, playlist id, hashtag, or query
|
||||
url: str
|
||||
|
||||
|
||||
def get_youtube_video_id(url: str) -> str | None:
|
||||
"""Extract a video ID from watch/youtu.be/embed/shorts URL formats."""
|
||||
parsed = urlparse(url)
|
||||
hostname = parsed.hostname or ""
|
||||
if hostname == "youtu.be":
|
||||
return parsed.path[1:] or None
|
||||
if hostname in ("www.youtube.com", "youtube.com", "m.youtube.com"):
|
||||
if parsed.path == "/watch":
|
||||
return parse_qs(parsed.query).get("v", [None])[0]
|
||||
for prefix in ("/embed/", "/v/", "/shorts/"):
|
||||
if parsed.path.startswith(prefix):
|
||||
return parsed.path[len(prefix) :].split("/")[0] or None
|
||||
return None
|
||||
|
||||
|
||||
def resolve_url(url: str) -> ResolvedUrl | None:
|
||||
"""Classify a YouTube URL into a scrape job, or ``None`` if unrecognized."""
|
||||
parsed = urlparse(url)
|
||||
path = parsed.path or ""
|
||||
|
||||
# Shorts are videos with their own path.
|
||||
if "/shorts/" in path:
|
||||
vid = path.split("/shorts/")[1].split("/")[0]
|
||||
return ResolvedUrl("video", vid, url) if vid else None
|
||||
|
||||
video_id = get_youtube_video_id(url)
|
||||
if video_id:
|
||||
return ResolvedUrl("video", video_id, url)
|
||||
|
||||
# Playlist (either a /playlist page or any URL carrying ?list=).
|
||||
playlist_match = _PLAYLIST_ID_RE.search(url)
|
||||
if path.startswith("/playlist") and playlist_match:
|
||||
return ResolvedUrl("playlist", playlist_match.group(1), url)
|
||||
|
||||
# Search results page.
|
||||
if path == "/results":
|
||||
query = parse_qs(parsed.query).get("search_query", [None])[0]
|
||||
if query:
|
||||
return ResolvedUrl("search", unquote(query), url)
|
||||
return None
|
||||
|
||||
# Hashtag page (/hashtag/<tag>).
|
||||
if path.startswith("/hashtag/"):
|
||||
tag = path[len("/hashtag/") :].split("/")[0]
|
||||
return ResolvedUrl("hashtag", unquote(tag), url) if tag else None
|
||||
|
||||
# Channel: /@handle, /channel/UC..., /c/Name, /user/Name.
|
||||
if path.startswith("/@"):
|
||||
return ResolvedUrl("channel", path[2:].split("/")[0], url)
|
||||
for prefix in ("/channel/", "/c/", "/user/"):
|
||||
if path.startswith(prefix):
|
||||
handle = path[len(prefix) :].split("/")[0]
|
||||
return ResolvedUrl("channel", handle, url) if handle else None
|
||||
|
||||
# A trailing ?list= without an explicit /playlist path.
|
||||
if playlist_match:
|
||||
return ResolvedUrl("playlist", playlist_match.group(1), url)
|
||||
|
||||
return None
|
||||
198
surfsense_backend/scripts/e2e_youtube_scraper.py
Normal file
198
surfsense_backend/scripts/e2e_youtube_scraper.py
Normal file
|
|
@ -0,0 +1,198 @@
|
|||
"""Manual functional e2e for the YouTube scraper (app/scrapers/youtube).
|
||||
|
||||
Run from the backend directory:
|
||||
cd surfsense_backend
|
||||
uv run python scripts/e2e_youtube_scraper.py
|
||||
# or: .\\.venv\\Scripts\\python.exe scripts/e2e_youtube_scraper.py
|
||||
|
||||
This is NOT a pytest test (it needs live network + optional proxy creds). It:
|
||||
|
||||
Step 0 — validates the keyless InnerTube ``search`` POST works; if it 400s the
|
||||
scraper transparently retries with the public web key (proven here).
|
||||
Step 1 — scrapes a known video URL (metadata + optional subtitles).
|
||||
Step 2 — runs a search query and prints the first few results.
|
||||
Step 3 — scrapes a small channel's latest videos.
|
||||
Step 4 — dumps trimmed raw ytInitialData / ytInitialPlayerResponse fixtures to
|
||||
tests/unit/scrapers/youtube/fixtures/ for the offline parser test.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# --- bootstrap: load .env and put the backend root on sys.path before app.* ---
|
||||
_BACKEND_ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(_BACKEND_ROOT))
|
||||
for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"):
|
||||
if _candidate.exists():
|
||||
load_dotenv(_candidate)
|
||||
break
|
||||
|
||||
from app.scrapers.youtube import ( # noqa: E402
|
||||
YouTubeCommentsInput,
|
||||
YouTubeScrapeInput,
|
||||
scrape_comments,
|
||||
scrape_youtube,
|
||||
)
|
||||
from app.scrapers.youtube.innertube import ( # noqa: E402
|
||||
INNERTUBE_PUBLIC_API_KEY,
|
||||
INNERTUBE_SEARCH_URL,
|
||||
build_innertube_payload,
|
||||
fetch_html,
|
||||
post_innertube,
|
||||
)
|
||||
from app.scrapers.youtube.parsers import ( # noqa: E402
|
||||
extract_yt_initial_data,
|
||||
extract_yt_initial_player_response,
|
||||
)
|
||||
|
||||
_VIDEO_URL = "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
|
||||
_SEARCH_TERM = "web scraping tutorials"
|
||||
_CHANNEL_URL = "https://www.youtube.com/@YouTube"
|
||||
# A geo-tagged walking tour + a multi-owner collaboration video (long-tail fields).
|
||||
_LOCATION_VIDEO_URL = "https://www.youtube.com/watch?v=bhJU_fVHMmY"
|
||||
_COLLAB_VIDEO_URL = "https://www.youtube.com/watch?v=AI2BwwLX_7s"
|
||||
# MrBeast localizes titles/descriptions into many languages (translation flow).
|
||||
_TRANSLATED_VIDEO_URL = "https://www.youtube.com/watch?v=iYlODtkyw_I"
|
||||
|
||||
_FIXTURE_DIR = _BACKEND_ROOT / "tests" / "unit" / "scrapers" / "youtube" / "fixtures"
|
||||
|
||||
|
||||
def _hr(title: str) -> None:
|
||||
print(f"\n{'=' * 70}\n{title}\n{'=' * 70}")
|
||||
|
||||
|
||||
def _check(label: str, ok: bool, detail: str = "") -> bool:
|
||||
print(f" [{'PASS' if ok else 'FAIL'}] {label}{f' — {detail}' if detail else ''}")
|
||||
return ok
|
||||
|
||||
|
||||
async def step0_validate_innertube() -> bool:
|
||||
_hr("STEP 0 — InnerTube search POST (keyless, then public-key fallback)")
|
||||
payload = build_innertube_payload(search_query=_SEARCH_TERM)
|
||||
keyless = await post_innertube(INNERTUBE_SEARCH_URL, payload)
|
||||
if keyless is not None:
|
||||
return _check("keyless search POST", True, "keyless works")
|
||||
keyed = await post_innertube(
|
||||
INNERTUBE_SEARCH_URL, payload, api_key=INNERTUBE_PUBLIC_API_KEY
|
||||
)
|
||||
return _check("public-key search POST", keyed is not None, "keyless 400 -> key")
|
||||
|
||||
|
||||
async def step1_video() -> bool:
|
||||
_hr("STEP 1 — scrape a known video")
|
||||
inp = YouTubeScrapeInput(startUrls=[{"url": _VIDEO_URL}], downloadSubtitles=True)
|
||||
items = await scrape_youtube(inp)
|
||||
print(json.dumps(items[:1], indent=2)[:2000])
|
||||
ok = bool(items) and items[0].get("id") == "dQw4w9WgXcQ"
|
||||
return _check("video scraped with id + title", ok and bool(items[0].get("title")))
|
||||
|
||||
|
||||
async def step2_search() -> bool:
|
||||
_hr("STEP 2 — search query")
|
||||
inp = YouTubeScrapeInput(searchQueries=[_SEARCH_TERM], maxResults=5)
|
||||
items = await scrape_youtube(inp)
|
||||
for it in items[:5]:
|
||||
print(f" - {it.get('id')} | {it.get('title')}")
|
||||
return _check("search returned results", len(items) > 0, f"{len(items)} items")
|
||||
|
||||
|
||||
async def step3_channel() -> bool:
|
||||
_hr("STEP 3 — channel latest videos")
|
||||
inp = YouTubeScrapeInput(startUrls=[{"url": _CHANNEL_URL}], maxResults=3)
|
||||
items = await scrape_youtube(inp)
|
||||
for it in items[:3]:
|
||||
print(f" - {it.get('id')} | {it.get('title')}")
|
||||
return _check("channel returned videos", len(items) > 0, f"{len(items)} items")
|
||||
|
||||
|
||||
async def step4_dump_fixtures() -> bool:
|
||||
_hr("STEP 4 — dump raw fixtures for offline test")
|
||||
html = await fetch_html(_VIDEO_URL)
|
||||
if not html:
|
||||
return _check("fetched video HTML", False)
|
||||
initial = extract_yt_initial_data(html)
|
||||
player = extract_yt_initial_player_response(html)
|
||||
_FIXTURE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
if player:
|
||||
(_FIXTURE_DIR / "video_player_response.json").write_text(
|
||||
json.dumps(player), encoding="utf-8"
|
||||
)
|
||||
if initial:
|
||||
(_FIXTURE_DIR / "video_initial_data.json").write_text(
|
||||
json.dumps(initial), encoding="utf-8"
|
||||
)
|
||||
return _check(
|
||||
"dumped fixtures", bool(player), f"-> {_FIXTURE_DIR}"
|
||||
)
|
||||
|
||||
|
||||
async def step5_comments() -> bool:
|
||||
_hr("STEP 5 — comments (+ replies) for a video")
|
||||
inp = YouTubeCommentsInput(
|
||||
startUrls=[{"url": _VIDEO_URL}], maxComments=6, sortCommentsBy="NEWEST_FIRST"
|
||||
)
|
||||
items = await scrape_comments(inp)
|
||||
for it in items[:6]:
|
||||
print(f" - [{it.get('type')}] {it.get('author')} | {it.get('voteCount')} votes")
|
||||
ok = bool(items) and all(it.get("cid") and it.get("videoId") for it in items)
|
||||
has_reply = any(it.get("type") == "reply" and it.get("replyToCid") for it in items)
|
||||
return _check("comments scraped (cid+videoId, reply linkage)", ok and has_reply,
|
||||
f"{len(items)} items")
|
||||
|
||||
|
||||
async def step6_location_collaborators() -> bool:
|
||||
_hr("STEP 6 — long-tail fields: location + collaborators")
|
||||
loc_items = await scrape_youtube(
|
||||
YouTubeScrapeInput(startUrls=[{"url": _LOCATION_VIDEO_URL}])
|
||||
)
|
||||
location = loc_items[0].get("location") if loc_items else None
|
||||
print(f" location: {location!r}")
|
||||
collab_items = await scrape_youtube(
|
||||
YouTubeScrapeInput(startUrls=[{"url": _COLLAB_VIDEO_URL}])
|
||||
)
|
||||
collaborators = collab_items[0].get("collaborators") if collab_items else None
|
||||
print(f" collaborators: {collaborators}")
|
||||
return _check(
|
||||
"location + collaborators populated",
|
||||
bool(location) and bool(collaborators) and len(collaborators) >= 2,
|
||||
)
|
||||
|
||||
|
||||
async def step7_translation() -> bool:
|
||||
_hr("STEP 7 — translatedTitle/translatedText (subtitlesLanguage=es)")
|
||||
items = await scrape_youtube(
|
||||
YouTubeScrapeInput(startUrls=[{"url": _TRANSLATED_VIDEO_URL}], subtitlesLanguage="es")
|
||||
)
|
||||
it = items[0] if items else {}
|
||||
print(f" title : {it.get('title')}")
|
||||
print(f" translatedTitle: {it.get('translatedTitle')}")
|
||||
# A localized video's translated title differs from the canonical English one.
|
||||
ok = bool(it.get("translatedTitle")) and it.get("translatedTitle") != it.get("title")
|
||||
return _check("translatedTitle differs from original", ok)
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
results = []
|
||||
results.append(await step0_validate_innertube())
|
||||
if not results[-1]:
|
||||
print("\nInnerTube unreachable — aborting remaining steps.")
|
||||
return 1
|
||||
results.append(await step1_video())
|
||||
results.append(await step2_search())
|
||||
results.append(await step3_channel())
|
||||
results.append(await step4_dump_fixtures())
|
||||
results.append(await step5_comments())
|
||||
results.append(await step6_location_collaborators())
|
||||
results.append(await step7_translation())
|
||||
|
||||
_hr("SUMMARY")
|
||||
print(f" {sum(results)}/{len(results)} steps passed")
|
||||
return 0 if all(results) else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(asyncio.run(main()))
|
||||
0
surfsense_backend/tests/unit/scrapers/__init__.py
Normal file
0
surfsense_backend/tests/unit/scrapers/__init__.py
Normal file
|
|
@ -0,0 +1,199 @@
|
|||
"""Offline resilience tests for the fetch seam and fan-out worker pool.
|
||||
|
||||
No network. Stubs the proxy session so the rotate-on-block path (which never
|
||||
fires in practice because live runs return 200s) is exercised deterministically,
|
||||
and asserts the worker pool closes every session when the consumer stops early.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
import pytest
|
||||
|
||||
from app.scrapers.youtube import innertube, scraper
|
||||
from app.scrapers.youtube.innertube import (
|
||||
INNERTUBE_SEARCH_URL,
|
||||
_current_session,
|
||||
fetch_html,
|
||||
post_innertube,
|
||||
)
|
||||
|
||||
|
||||
class _FakePage:
|
||||
def __init__(self, status: int) -> None:
|
||||
self.status = status
|
||||
|
||||
def json(self) -> dict:
|
||||
return {"status": self.status}
|
||||
|
||||
@property
|
||||
def html_content(self) -> str:
|
||||
return "<html>ok</html>"
|
||||
|
||||
|
||||
class _FakeSession:
|
||||
"""One 'IP': returns ``status`` for every request, or raises if ``exc``."""
|
||||
|
||||
def __init__(self, status: int = 200, *, exc: bool = False) -> None:
|
||||
self.status = status
|
||||
self.exc = exc
|
||||
self.calls = 0
|
||||
|
||||
async def post(self, url, json=None): # noqa: A002 - mirror scrapling API
|
||||
self.calls += 1
|
||||
if self.exc:
|
||||
raise ConnectionError("boom")
|
||||
return _FakePage(self.status)
|
||||
|
||||
async def get(self, url, headers=None, cookies=None):
|
||||
self.calls += 1
|
||||
if self.exc:
|
||||
raise ConnectionError("boom")
|
||||
return _FakePage(self.status)
|
||||
|
||||
|
||||
class _FakeHolder:
|
||||
"""Holder whose ``rotate()`` advances to the next fake session (a new IP)."""
|
||||
|
||||
def __init__(self, sessions: list[_FakeSession]) -> None:
|
||||
self._sessions = sessions
|
||||
self.session = sessions[0]
|
||||
self.rotations = 0
|
||||
|
||||
async def rotate(self):
|
||||
self.rotations += 1
|
||||
self.session = self._sessions[min(self.rotations, len(self._sessions) - 1)]
|
||||
return self.session
|
||||
|
||||
|
||||
def _payload() -> dict:
|
||||
return {"context": {}}
|
||||
|
||||
|
||||
async def test_post_rotates_on_429_then_succeeds():
|
||||
holder = _FakeHolder([_FakeSession(429), _FakeSession(200)])
|
||||
token = _current_session.set(holder)
|
||||
try:
|
||||
result = await post_innertube(INNERTUBE_SEARCH_URL, _payload())
|
||||
finally:
|
||||
_current_session.reset(token)
|
||||
assert result == {"status": 200}
|
||||
assert holder.rotations == 1 # rotated exactly once to the healthy IP
|
||||
|
||||
|
||||
async def test_post_rotates_on_connection_error_then_succeeds():
|
||||
holder = _FakeHolder([_FakeSession(exc=True), _FakeSession(200)])
|
||||
token = _current_session.set(holder)
|
||||
try:
|
||||
result = await post_innertube(INNERTUBE_SEARCH_URL, _payload())
|
||||
finally:
|
||||
_current_session.reset(token)
|
||||
assert result == {"status": 200}
|
||||
assert holder.rotations == 1
|
||||
|
||||
|
||||
async def test_post_gives_up_after_max_rotations():
|
||||
# Every IP is blocked -> rotate up to the cap, then return None.
|
||||
holder = _FakeHolder([_FakeSession(429) for _ in range(innertube._MAX_ROTATIONS + 1)])
|
||||
token = _current_session.set(holder)
|
||||
try:
|
||||
result = await post_innertube(INNERTUBE_SEARCH_URL, _payload())
|
||||
finally:
|
||||
_current_session.reset(token)
|
||||
assert result is None
|
||||
assert holder.rotations == innertube._MAX_ROTATIONS
|
||||
|
||||
|
||||
async def test_post_does_not_rotate_on_non_block_status():
|
||||
# 404 is not a block: fail fast, no wasted IP rotations.
|
||||
holder = _FakeHolder([_FakeSession(404), _FakeSession(200)])
|
||||
token = _current_session.set(holder)
|
||||
try:
|
||||
result = await post_innertube(INNERTUBE_SEARCH_URL, _payload())
|
||||
finally:
|
||||
_current_session.reset(token)
|
||||
assert result is None
|
||||
assert holder.rotations == 0
|
||||
|
||||
|
||||
async def test_fetch_html_rotates_then_succeeds():
|
||||
holder = _FakeHolder([_FakeSession(403), _FakeSession(200)])
|
||||
token = _current_session.set(holder)
|
||||
try:
|
||||
html = await fetch_html("https://www.youtube.com/watch?v=abc")
|
||||
finally:
|
||||
_current_session.reset(token)
|
||||
assert html == "<html>ok</html>"
|
||||
assert holder.rotations == 1
|
||||
|
||||
|
||||
async def test_fetch_html_falls_back_to_stealthy_when_all_blocked(monkeypatch):
|
||||
called: dict[str, str] = {}
|
||||
|
||||
async def _fake_stealthy(url: str):
|
||||
called["url"] = url
|
||||
return "<html>stealthy</html>"
|
||||
|
||||
monkeypatch.setattr(innertube, "_fetch_html_stealthy", _fake_stealthy)
|
||||
holder = _FakeHolder([_FakeSession(429) for _ in range(innertube._MAX_ROTATIONS + 1)])
|
||||
token = _current_session.set(holder)
|
||||
try:
|
||||
html = await fetch_html("https://www.youtube.com/watch?v=zzz")
|
||||
finally:
|
||||
_current_session.reset(token)
|
||||
assert html == "<html>stealthy</html>"
|
||||
assert called["url"].endswith("v=zzz")
|
||||
|
||||
|
||||
class _TrackingHolder:
|
||||
"""Fake fan-out session holder that records whether it was closed."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.closed = False
|
||||
|
||||
async def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
|
||||
async def test_fan_out_closes_all_sessions_on_early_stop(monkeypatch):
|
||||
holders: list[_TrackingHolder] = []
|
||||
|
||||
async def _fake_open():
|
||||
h = _TrackingHolder()
|
||||
holders.append(h)
|
||||
return h
|
||||
|
||||
# No real session binding needed; jobs just yield.
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
@asynccontextmanager
|
||||
async def _fake_bind(_holder):
|
||||
yield _holder
|
||||
|
||||
monkeypatch.setattr(scraper, "open_proxy_holder", _fake_open)
|
||||
monkeypatch.setattr(scraper, "bind_proxy_holder", _fake_bind)
|
||||
|
||||
async def _job(n: int) -> AsyncIterator[dict]:
|
||||
for i in range(5):
|
||||
yield {"job": n, "i": i}
|
||||
|
||||
jobs = [_job(n) for n in range(20)]
|
||||
|
||||
gen = scraper.fan_out(jobs, concurrency=4)
|
||||
collected = []
|
||||
async for item in gen:
|
||||
collected.append(item)
|
||||
if len(collected) >= 3: # consumer stops early (like a limit)
|
||||
break
|
||||
await gen.aclose() # deterministically run fan_out's teardown
|
||||
|
||||
assert len(collected) >= 3
|
||||
assert holders, "workers should have opened sessions"
|
||||
# Every opened session must be closed after the generator is torn down.
|
||||
assert all(h.closed for h in holders), "a worker leaked its proxy session"
|
||||
|
||||
|
||||
async def test_fan_out_empty_jobs_is_noop():
|
||||
out = [x async for x in scraper.fan_out([])]
|
||||
assert out == []
|
||||
852
surfsense_backend/tests/unit/scrapers/youtube/test_parsers.py
Normal file
852
surfsense_backend/tests/unit/scrapers/youtube/test_parsers.py
Normal file
|
|
@ -0,0 +1,852 @@
|
|||
"""Offline unit tests for the YouTube scraper parsers.
|
||||
|
||||
No network. Uses small hand-built raw ``ytInitialData`` / ``ytInitialPlayerResponse``
|
||||
shapes (mirroring real nesting) plus direct normalization cases — the smallest
|
||||
things that fail if the dict-walk or normalization logic breaks. If the e2e
|
||||
script has captured real fixtures into ``fixtures/``, they are exercised too.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.scrapers.youtube.parsers import (
|
||||
channel_about_tokens,
|
||||
comment_next_token,
|
||||
comment_reply_tokens,
|
||||
comment_section_token,
|
||||
comment_sort_tokens,
|
||||
dig,
|
||||
find_all,
|
||||
parse_channel_about,
|
||||
parse_collaborators,
|
||||
parse_comment_entities,
|
||||
parse_description_links,
|
||||
parse_location,
|
||||
parse_translation,
|
||||
parse_channel_metadata,
|
||||
parse_channel_shorts,
|
||||
parse_playlist_video_ids,
|
||||
parse_count,
|
||||
parse_date,
|
||||
parse_search_response,
|
||||
parse_video_page,
|
||||
seconds_to_duration,
|
||||
)
|
||||
from app.scrapers.youtube.schemas import YouTubeScrapeInput
|
||||
from app.scrapers.youtube.search_filters import build_search_params
|
||||
from app.scrapers.youtube.url_resolver import resolve_url
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
_FIXTURE_DIR = Path(__file__).parent / "fixtures"
|
||||
|
||||
|
||||
# --- normalization -----------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw,expected",
|
||||
[
|
||||
("451K views", 451_000),
|
||||
("1.2M", 1_200_000),
|
||||
("1,234", 1_234),
|
||||
("100", 100),
|
||||
("2.5B", 2_500_000_000),
|
||||
(500, 500),
|
||||
("No views", None),
|
||||
("", None),
|
||||
(None, None),
|
||||
],
|
||||
)
|
||||
def test_parse_count(raw, expected):
|
||||
assert parse_count(raw) == expected
|
||||
|
||||
|
||||
def test_parse_date_prefers_publish_date():
|
||||
mf = {"publishDate": "2024-08-27", "uploadDate": "2024-08-20"}
|
||||
assert parse_date(mf) == "2024-08-27"
|
||||
assert parse_date({"uploadDate": "2024-08-20"}) == "2024-08-20"
|
||||
assert parse_date(None) is None
|
||||
|
||||
|
||||
def test_seconds_to_duration():
|
||||
assert seconds_to_duration("207") == "00:03:27"
|
||||
assert seconds_to_duration(3661) == "01:01:01"
|
||||
assert seconds_to_duration(None) is None
|
||||
|
||||
|
||||
# --- traversal helpers -------------------------------------------------------
|
||||
|
||||
|
||||
def test_find_all_and_dig():
|
||||
data = {"a": {"b": [{"k": 1}, {"k": 2}]}, "k": 3}
|
||||
assert find_all(data, "k") == [1, 2, 3]
|
||||
assert dig(data, "a", "b", 0, "k") == 1
|
||||
assert dig(data, "a", "b", 5, "k") is None
|
||||
assert dig(data, "missing", "x") is None
|
||||
|
||||
|
||||
# --- page parsing ------------------------------------------------------------
|
||||
|
||||
|
||||
def _video_html() -> str:
|
||||
player = {
|
||||
"videoDetails": {
|
||||
"videoId": "abc123",
|
||||
"title": "Test Title",
|
||||
"lengthSeconds": "207",
|
||||
"viewCount": "100",
|
||||
"author": "TechReviewer",
|
||||
"channelId": "UC123",
|
||||
"shortDescription": "desc",
|
||||
"keywords": ["#tech"],
|
||||
"thumbnail": {
|
||||
"thumbnails": [
|
||||
{"url": "https://i.ytimg.com/vi/abc123/hq.jpg", "width": 480, "height": 360}
|
||||
]
|
||||
},
|
||||
},
|
||||
"microformat": {
|
||||
"playerMicroformatRenderer": {"publishDate": "2024-08-27", "isFamilySafe": True}
|
||||
},
|
||||
"adPlacements": [{"adPlacementRenderer": {}}],
|
||||
}
|
||||
initial = {
|
||||
"like": {"buttonViewModel": {"iconName": "LIKE", "title": "15K"}},
|
||||
"comments": {
|
||||
"itemSectionRenderer": {
|
||||
"sectionIdentifier": "comment-item-section",
|
||||
"contents": [{"continuationItemRenderer": {"trigger": "x"}}],
|
||||
}
|
||||
},
|
||||
"ctx": {"contextualInfo": {"runs": [{"text": "1,250"}]}},
|
||||
"chan": {"canonicalBaseUrl": "/@Apify"},
|
||||
"subs": {"subscriberCountText": {"simpleText": "500K subscribers"}},
|
||||
"badge": {"metadataBadgeRenderer": {"tooltip": "Verified"}},
|
||||
}
|
||||
return (
|
||||
"<html><script>var ytInitialPlayerResponse = "
|
||||
+ json.dumps(player)
|
||||
+ ";</script><script>var ytInitialData = "
|
||||
+ json.dumps(initial)
|
||||
+ ";</script></html>"
|
||||
)
|
||||
|
||||
|
||||
def test_parse_video_page():
|
||||
result = parse_video_page(_video_html())
|
||||
assert result is not None
|
||||
assert result["id"] == "abc123"
|
||||
assert result["url"] == "https://www.youtube.com/watch?v=abc123"
|
||||
assert result["title"] == "Test Title"
|
||||
assert result["viewCount"] == 100
|
||||
assert result["duration"] == "00:03:27"
|
||||
assert result["date"] == "2024-08-27"
|
||||
assert result["thumbnailUrl"] == "https://i.ytimg.com/vi/abc123/hq.jpg"
|
||||
assert result["hashtags"] == ["#tech"]
|
||||
assert result["channelName"] == "TechReviewer"
|
||||
assert result["channelId"] == "UC123"
|
||||
assert result["likes"] == 15_000
|
||||
assert result["commentsCount"] == 1_250
|
||||
assert result["channelUrl"] == "https://www.youtube.com/@Apify"
|
||||
assert result["channelUsername"] == "Apify"
|
||||
assert result["numberOfSubscribers"] == 500_000
|
||||
assert result["isChannelVerified"] is True
|
||||
assert result["isMonetized"] is True # adPlacements present
|
||||
assert result["isAgeRestricted"] is None # family safe, no age gate
|
||||
assert result["commentsTurnedOff"] is False # section has a continuation
|
||||
|
||||
|
||||
def test_parse_video_page_comments_turned_off():
|
||||
player = {"videoDetails": {"videoId": "x", "title": "t"}}
|
||||
initial = {
|
||||
"itemSectionRenderer": {
|
||||
"sectionIdentifier": "comment-item-section",
|
||||
"contents": [{"messageRenderer": {"text": {"simpleText": "off"}}}],
|
||||
}
|
||||
}
|
||||
html = (
|
||||
"<script>var ytInitialPlayerResponse = "
|
||||
+ json.dumps(player)
|
||||
+ ";</script><script>var ytInitialData = "
|
||||
+ json.dumps(initial)
|
||||
+ ";</script>"
|
||||
)
|
||||
result = parse_video_page(html)
|
||||
assert result is not None
|
||||
assert result["commentsTurnedOff"] is True
|
||||
assert result["commentsCount"] is None
|
||||
|
||||
|
||||
def test_parse_video_page_age_restricted():
|
||||
player = {
|
||||
"videoDetails": {"videoId": "x", "title": "t"},
|
||||
"playabilityStatus": {
|
||||
"status": "LOGIN_REQUIRED",
|
||||
"desktopLegacyAgeGateReason": 1,
|
||||
},
|
||||
}
|
||||
html = "<script>var ytInitialPlayerResponse = " + json.dumps(player) + ";</script>"
|
||||
result = parse_video_page(html)
|
||||
assert result is not None
|
||||
assert result["isAgeRestricted"] is True
|
||||
assert result["isMonetized"] is None # no ad slots → can't confirm
|
||||
|
||||
|
||||
def test_parse_video_page_members_only_and_paid():
|
||||
player = {
|
||||
"videoDetails": {"videoId": "m", "title": "t"},
|
||||
"playabilityStatus": {"errorScreen": {"x": {"offerId": "sponsors_only_video"}}},
|
||||
"paidContentOverlay": {"paidContentOverlayRenderer": {"text": "Includes paid promotion"}},
|
||||
}
|
||||
initial = {
|
||||
"badges": [
|
||||
{
|
||||
"metadataBadgeRenderer": {
|
||||
"style": "BADGE_STYLE_TYPE_MEMBERS_ONLY",
|
||||
"label": "Members only",
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
html = (
|
||||
"<script>var ytInitialPlayerResponse = "
|
||||
+ json.dumps(player)
|
||||
+ ";</script><script>var ytInitialData = "
|
||||
+ json.dumps(initial)
|
||||
+ ";</script>"
|
||||
)
|
||||
result = parse_video_page(html)
|
||||
assert result is not None
|
||||
assert result["isMembersOnly"] is True
|
||||
assert result["isPaidContent"] is True
|
||||
|
||||
|
||||
def test_parse_video_page_returns_none_without_player():
|
||||
assert parse_video_page("<html>no data here</html>") is None
|
||||
|
||||
|
||||
def test_parse_channel_metadata():
|
||||
initial = {
|
||||
"metadata": {
|
||||
"channelMetadataRenderer": {
|
||||
"title": "Apify",
|
||||
"externalId": "UCabc123",
|
||||
"description": "We scrape the web.",
|
||||
"vanityChannelUrl": "https://www.youtube.com/@Apify",
|
||||
"avatar": {"thumbnails": [{"url": "https://a/avatar.jpg", "width": 88, "height": 88}]},
|
||||
}
|
||||
},
|
||||
"header": {
|
||||
"pageHeaderViewModel": {
|
||||
"banner": {
|
||||
"imageBannerViewModel": {
|
||||
"image": {
|
||||
"sources": [
|
||||
{"url": "https://a/banner.jpg", "width": 1060, "height": 175}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
meta = parse_channel_metadata(initial)
|
||||
assert meta["channelName"] == "Apify"
|
||||
assert meta["channelId"] == "UCabc123"
|
||||
assert meta["channelDescription"] == "We scrape the web."
|
||||
assert meta["channelUrl"] == "https://www.youtube.com/@Apify"
|
||||
assert meta["channelAvatarUrl"] == "https://a/avatar.jpg"
|
||||
assert meta["channelBannerUrl"] == "https://a/banner.jpg"
|
||||
assert parse_channel_metadata({}) == {}
|
||||
|
||||
|
||||
def test_parse_channel_about():
|
||||
about = {
|
||||
"description": "Official channel.",
|
||||
"country": "United States",
|
||||
"subscriberCountText": "45.6M subscribers",
|
||||
"viewCountText": "1,132,903,744 views",
|
||||
"videoCountText": "1,471 videos",
|
||||
"joinedDateText": {"content": "Joined Feb 8, 2005"},
|
||||
}
|
||||
out = parse_channel_about(about)
|
||||
assert out["channelDescription"] == "Official channel."
|
||||
assert out["channelLocation"] == "United States"
|
||||
assert out["numberOfSubscribers"] == 45_600_000
|
||||
assert out["channelTotalViews"] == 1_132_903_744
|
||||
assert out["channelTotalVideos"] == 1_471
|
||||
assert out["channelJoinedDate"] == "Feb 8, 2005"
|
||||
|
||||
|
||||
def test_parse_description_links():
|
||||
initial = {
|
||||
"attributedDescription": {
|
||||
"content": "See #tag and my site here",
|
||||
"commandRuns": [
|
||||
{
|
||||
"startIndex": 4,
|
||||
"length": 4,
|
||||
"onTap": {"innertubeCommand": {"urlEndpoint": {"url": "/hashtag/tag"}}},
|
||||
},
|
||||
{
|
||||
"startIndex": 21,
|
||||
"length": 4,
|
||||
"onTap": {
|
||||
"innertubeCommand": {
|
||||
"commandMetadata": {
|
||||
"webCommandMetadata": {
|
||||
"url": "https://www.youtube.com/redirect?q=https%3A%2F%2Fexample.com%2Fp&v=x"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
links = parse_description_links(initial)
|
||||
assert links == [
|
||||
{"url": "https://www.youtube.com/hashtag/tag", "text": "#tag"},
|
||||
{"url": "https://example.com/p", "text": "here"},
|
||||
]
|
||||
assert parse_description_links({}) is None
|
||||
|
||||
|
||||
def test_channel_about_tokens():
|
||||
initial = {
|
||||
"a": {"showEngagementPanelEndpoint": {"x": {"continuationCommand": {"token": "T1"}}}},
|
||||
"b": {"showEngagementPanelEndpoint": {"y": {"continuationCommand": {"token": "T2"}}}},
|
||||
}
|
||||
assert channel_about_tokens(initial) == ["T1", "T2"]
|
||||
assert channel_about_tokens({}) == []
|
||||
|
||||
|
||||
def test_parse_search_response():
|
||||
data = {
|
||||
"contents": [
|
||||
{
|
||||
"videoRenderer": {
|
||||
"videoId": "mx3g7XoPVNQ",
|
||||
"title": {"runs": [{"text": "A bad day"}]},
|
||||
"detailedMetadataSnippets": [
|
||||
{"snippetText": {"runs": [{"text": "become an engineer"}]}}
|
||||
],
|
||||
"publishedTimeText": {"simpleText": "7 days ago"},
|
||||
"lengthText": {"simpleText": "8:39"},
|
||||
"shortViewCountText": {"simpleText": "451K views"},
|
||||
"thumbnail": {
|
||||
"thumbnails": [
|
||||
{"url": "https://i.ytimg.com/vi/x/hq.jpg", "width": 360, "height": 202}
|
||||
]
|
||||
},
|
||||
"ownerText": {
|
||||
"runs": [
|
||||
{
|
||||
"text": "Life of Boris",
|
||||
"navigationEndpoint": {
|
||||
"browseEndpoint": {
|
||||
"browseId": "UCS5tt2z_DFvG7-39J3aE-bQ",
|
||||
"canonicalBaseUrl": "/@LifeofBoris",
|
||||
}
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
}
|
||||
}
|
||||
],
|
||||
"continuationCommand": {"token": "TOKEN123"},
|
||||
}
|
||||
items, token = parse_search_response(data)
|
||||
assert token == "TOKEN123"
|
||||
assert len(items) == 1
|
||||
item = items[0]
|
||||
assert item["id"] == "mx3g7XoPVNQ"
|
||||
assert item["title"] == "A bad day"
|
||||
assert item["viewCount"] == 451_000
|
||||
assert item["duration"] == "8:39"
|
||||
assert item["publishedTimeText"] == "7 days ago"
|
||||
assert item["date"] is None # list pages have no real date
|
||||
assert item["channelName"] == "Life of Boris"
|
||||
assert item["channelId"] == "UCS5tt2z_DFvG7-39J3aE-bQ"
|
||||
assert item["channelUrl"] == "https://www.youtube.com/@LifeofBoris"
|
||||
assert item["channelUsername"] == "LifeofBoris"
|
||||
|
||||
|
||||
def test_parse_channel_shorts():
|
||||
data = {
|
||||
"richItemRenderer": {
|
||||
"content": {
|
||||
"shortsLockupViewModel": {
|
||||
"entityId": "shorts-shelf-item-0UfLo9SOUeE",
|
||||
"onTap": {
|
||||
"innertubeCommand": {
|
||||
"reelWatchEndpoint": {"videoId": "0UfLo9SOUeE"}
|
||||
}
|
||||
},
|
||||
"overlayMetadata": {
|
||||
"primaryText": {"content": "is the west coast the best?"},
|
||||
"secondaryText": {"content": "812 views"},
|
||||
},
|
||||
"thumbnailViewModel": {
|
||||
"image": {
|
||||
"sources": [
|
||||
{"url": "https://i.ytimg.com/s.jpg", "width": 405, "height": 720}
|
||||
]
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
"continuationCommand": {"token": "SHORTSTOKEN"},
|
||||
}
|
||||
items, token = parse_channel_shorts(data)
|
||||
assert token == "SHORTSTOKEN"
|
||||
assert len(items) == 1
|
||||
it = items[0]
|
||||
assert it["id"] == "0UfLo9SOUeE"
|
||||
assert it["url"] == "https://www.youtube.com/shorts/0UfLo9SOUeE"
|
||||
assert it["title"] == "is the west coast the best?"
|
||||
assert it["viewCount"] == 812
|
||||
assert it["thumbnailUrl"] == "https://i.ytimg.com/s.jpg"
|
||||
|
||||
|
||||
def test_parse_playlist_video_ids():
|
||||
# Playlists now return lockupViewModel with contentId (not playlistVideoRenderer).
|
||||
# Guards against the exact renderer-migration that silently broke the flow:
|
||||
# keep videos (11-char ids) in order, dedup, and drop non-video lockups.
|
||||
data = {
|
||||
"contents": {
|
||||
"items": [
|
||||
{"lockupViewModel": {"contentId": "fNk_zzaMoSs", "contentType": "VIDEO"}},
|
||||
{"lockupViewModel": {"contentId": "k7RM-ot2NWY", "contentType": "VIDEO"}},
|
||||
{"lockupViewModel": {"contentId": "fNk_zzaMoSs", "contentType": "VIDEO"}}, # dup
|
||||
]
|
||||
},
|
||||
# a playlist self-lockup (non-video, longer id) that must be ignored
|
||||
"sidebar": {"lockupViewModel": {"contentId": "PLZHQObOWTQDPD3MizzM2xVFitgF8hE_ab"}},
|
||||
}
|
||||
assert parse_playlist_video_ids(data) == ["fNk_zzaMoSs", "k7RM-ot2NWY"]
|
||||
|
||||
|
||||
# --- search filter (sp=) encoder --------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"kwargs,expected",
|
||||
[
|
||||
({}, None),
|
||||
({"sortingOrder": "relevance"}, None), # default sort → no bytes
|
||||
({"sortingOrder": "rating"}, "CAE="),
|
||||
({"sortingOrder": "date"}, "CAI="),
|
||||
({"sortingOrder": "views"}, "CAM="),
|
||||
({"videoType": "video"}, "EgIQAQ=="),
|
||||
({"videoType": "movie"}, "EgIQBA=="),
|
||||
({"dateFilter": "hour"}, "EgIIAQ=="),
|
||||
({"dateFilter": "year"}, "EgIIBQ=="),
|
||||
({"lengthFilter": "under4"}, "EgIYAQ=="),
|
||||
({"lengthFilter": "between420"}, "EgIYAw=="),
|
||||
({"lengthFilter": "plus20"}, "EgIYAg=="),
|
||||
],
|
||||
)
|
||||
def test_build_search_params_matches_youtube_tokens(kwargs, expected):
|
||||
assert build_search_params(YouTubeScrapeInput(**kwargs)) == expected
|
||||
|
||||
|
||||
def test_build_search_params_combines_filters():
|
||||
# sort=date + upload=week + type=video + HD + subtitles, all in one token.
|
||||
token = build_search_params(
|
||||
YouTubeScrapeInput(
|
||||
sortingOrder="date",
|
||||
dateFilter="week",
|
||||
videoType="video",
|
||||
isHD=True,
|
||||
hasSubtitles=True,
|
||||
)
|
||||
)
|
||||
raw = base64.b64decode(token)
|
||||
# top-level: field1 (sort=2), field2 (Filters submessage)
|
||||
assert raw[:2] == b"\x08\x02" # sort = date
|
||||
assert b"\x12" in raw # Filters message tag present
|
||||
# Filters payload contains upload_date=week(3), type=video(1), hd, subtitles
|
||||
assert b"\x08\x03" in raw # uploadDate = week
|
||||
assert b"\x10\x01" in raw # type = video
|
||||
assert b"\x20\x01" in raw # feature 4 (HD) = 1
|
||||
assert b"\x28\x01" in raw # feature 5 (subtitles) = 1
|
||||
|
||||
|
||||
# --- location & collaborators ------------------------------------------------
|
||||
|
||||
|
||||
def _primary_info(super_title_link):
|
||||
return {
|
||||
"contents": {
|
||||
"twoColumnWatchNextResults": {
|
||||
"results": {
|
||||
"results": {
|
||||
"contents": [
|
||||
{"videoPrimaryInfoRenderer": {"superTitleLink": super_title_link}}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def test_parse_location_from_geo_tag_label():
|
||||
initial = _primary_info(
|
||||
{
|
||||
"runs": [{"text": "ROME"}],
|
||||
"accessibility": {
|
||||
"accessibilityData": {
|
||||
"label": "Link to a location restricted search for videos geo tagged with Rome"
|
||||
}
|
||||
},
|
||||
}
|
||||
)
|
||||
assert parse_location(initial) == "Rome"
|
||||
|
||||
|
||||
def test_parse_location_hashtag_supertitle_is_none():
|
||||
# Same slot, but hashtags carry no geo-tag a11y label.
|
||||
initial = _primary_info({"runs": [{"text": "#music"}, {"text": "#video"}]})
|
||||
assert parse_location(initial) is None
|
||||
|
||||
|
||||
def test_parse_location_absent_is_none():
|
||||
assert parse_location({}) is None
|
||||
|
||||
|
||||
def _owner(video_owner_renderer):
|
||||
return {
|
||||
"contents": {
|
||||
"twoColumnWatchNextResults": {
|
||||
"results": {
|
||||
"results": {
|
||||
"contents": [
|
||||
{
|
||||
"videoSecondaryInfoRenderer": {
|
||||
"owner": {"videoOwnerRenderer": video_owner_renderer}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def _collab_row(name, base_url):
|
||||
return {
|
||||
"listItemViewModel": {
|
||||
"title": {
|
||||
"content": name,
|
||||
"commandRuns": [
|
||||
{
|
||||
"onTap": {
|
||||
"innertubeCommand": {
|
||||
"browseEndpoint": {"browseId": "UCx", "canonicalBaseUrl": base_url}
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
},
|
||||
# A nested subscribe submenu — must NOT be picked up as a collaborator.
|
||||
"subscribeMenu": {"listItemViewModel": {"title": {"content": "Unsubscribe"}}},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def test_parse_collaborators_from_dialog():
|
||||
initial = _owner(
|
||||
{
|
||||
"attributedTitle": {
|
||||
"content": "Alice and Bob",
|
||||
"commandRuns": [
|
||||
{
|
||||
"onTap": {
|
||||
"innertubeCommand": {
|
||||
"showDialogCommand": {
|
||||
"panelLoadingStrategy": {
|
||||
"inlineContent": {
|
||||
"dialogViewModel": {
|
||||
"header": {
|
||||
"dialogHeaderViewModel": {
|
||||
"headline": {"content": "Collaborators"}
|
||||
}
|
||||
},
|
||||
"customContent": {
|
||||
"listViewModel": {
|
||||
"listItems": [
|
||||
_collab_row("Alice", "/@alice"),
|
||||
_collab_row(
|
||||
"Bob", "/channel/UCbob123"
|
||||
),
|
||||
]
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
}
|
||||
}
|
||||
)
|
||||
collaborators = parse_collaborators(initial)
|
||||
assert collaborators == [
|
||||
{"name": "Alice", "username": "alice", "url": "https://www.youtube.com/@alice"},
|
||||
{"name": "Bob", "username": None, "url": "https://www.youtube.com/channel/UCbob123"},
|
||||
]
|
||||
|
||||
|
||||
def test_parse_collaborators_single_owner_is_none():
|
||||
# Ordinary videos use title.runs, not attributedTitle.
|
||||
initial = _owner({"title": {"runs": [{"text": "Some Channel"}]}})
|
||||
assert parse_collaborators(initial) is None
|
||||
|
||||
|
||||
def test_parse_translation_from_next():
|
||||
next_data = {
|
||||
"contents": {
|
||||
"twoColumnWatchNextResults": {
|
||||
"results": {
|
||||
"results": {
|
||||
"contents": [
|
||||
{"videoPrimaryInfoRenderer": {"title": {"runs": [{"text": "Título "}, {"text": "traducido"}]}}},
|
||||
{"videoSecondaryInfoRenderer": {"attributedDescription": {"content": "Descripción traducida"}}},
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
title, description = parse_translation(next_data)
|
||||
assert title == "Título traducido"
|
||||
assert description == "Descripción traducida"
|
||||
|
||||
|
||||
def test_parse_translation_description_runs_fallback():
|
||||
next_data = {
|
||||
"videoSecondaryInfoRenderer": {"description": {"runs": [{"text": "old "}, {"text": "style"}]}}
|
||||
}
|
||||
title, description = parse_translation(next_data)
|
||||
assert title is None
|
||||
assert description == "old style"
|
||||
|
||||
|
||||
# --- comments ----------------------------------------------------------------
|
||||
|
||||
|
||||
def _comment_cep(cid, *, level=0, hearted=False, owner=False, replies="5"):
|
||||
"""Mirror a real commentEntityPayload's relevant fields."""
|
||||
toolbar = {"likeCountNotliked": "1.2K", "replyCount": replies}
|
||||
if hearted:
|
||||
toolbar["creatorThumbnailUrl"] = "https://yt3.ggpht.com/heart"
|
||||
return {
|
||||
"properties": {
|
||||
"commentId": cid,
|
||||
"content": {"content": f"text {cid}"},
|
||||
"publishedTime": "2 days ago",
|
||||
"replyLevel": level,
|
||||
},
|
||||
"author": {"displayName": f"@user{cid}", "isCreator": owner},
|
||||
"toolbar": toolbar,
|
||||
}
|
||||
|
||||
|
||||
def _next_comments_response():
|
||||
"""A /next comments response: sort menu, two threads, a page token."""
|
||||
def _reply_loader(token):
|
||||
return {
|
||||
"continuationItemRenderer": {
|
||||
"continuationEndpoint": {"continuationCommand": {"token": token}}
|
||||
}
|
||||
}
|
||||
|
||||
def _thread(cid, reply_token):
|
||||
return {
|
||||
"commentThreadRenderer": {
|
||||
"commentViewModel": {"commentViewModel": {"commentId": cid}},
|
||||
"replies": {
|
||||
"commentRepliesRenderer": {"contents": [_reply_loader(reply_token)]}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
"frameworkUpdates": {
|
||||
"entityBatchUpdate": {
|
||||
"mutations": [
|
||||
{"payload": {"commentEntityPayload": _comment_cep("C1", hearted=True, owner=True)}},
|
||||
{"payload": {"commentEntityPayload": _comment_cep("C2")}},
|
||||
]
|
||||
}
|
||||
},
|
||||
"onResponseReceivedEndpoints": [
|
||||
{
|
||||
"reloadContinuationItemsCommand": {
|
||||
"continuationItems": [
|
||||
_thread("C1", "REPLYTOK1"),
|
||||
_thread("C2", "REPLYTOK2"),
|
||||
{
|
||||
"continuationItemRenderer": {
|
||||
"continuationEndpoint": {
|
||||
"continuationCommand": {"token": "PAGE2"}
|
||||
}
|
||||
}
|
||||
},
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"header": {
|
||||
"sortFilterSubMenuRenderer": {
|
||||
"subMenuItems": [
|
||||
{"title": "Top", "serviceEndpoint": {"continuationCommand": {"token": "TOPTOK"}}},
|
||||
{"title": "Newest", "serviceEndpoint": {"continuationCommand": {"token": "NEWTOK"}}},
|
||||
]
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_parse_comment_entities():
|
||||
data = _next_comments_response()
|
||||
comments = parse_comment_entities(data)
|
||||
assert len(comments) == 2
|
||||
first = comments[0]
|
||||
assert first["cid"] == "C1"
|
||||
assert first["comment"] == "text C1"
|
||||
assert first["author"] == "@userC1"
|
||||
assert first["voteCount"] == 1200 # "1.2K"
|
||||
assert first["replyCount"] == 5
|
||||
assert first["type"] == "comment"
|
||||
assert first["hasCreatorHeart"] is True
|
||||
assert first["authorIsChannelOwner"] is True
|
||||
# second is a plain, non-hearted comment
|
||||
assert comments[1]["hasCreatorHeart"] is False
|
||||
assert comments[1]["authorIsChannelOwner"] is False
|
||||
|
||||
|
||||
def test_comment_entity_reply_level_and_empty_reply_count():
|
||||
reply = parse_comment_entities(
|
||||
{
|
||||
"frameworkUpdates": {
|
||||
"entityBatchUpdate": {
|
||||
"mutations": [
|
||||
{"payload": {"commentEntityPayload": _comment_cep("R1", level=1, replies="")}}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
)[0]
|
||||
assert reply["type"] == "reply"
|
||||
assert reply["replyCount"] is None # empty string -> None
|
||||
|
||||
|
||||
def test_comment_sort_tokens():
|
||||
tokens = comment_sort_tokens(_next_comments_response())
|
||||
assert tokens == {"Top": "TOPTOK", "Newest": "NEWTOK"}
|
||||
|
||||
|
||||
def test_comment_reply_tokens():
|
||||
tokens = comment_reply_tokens(_next_comments_response())
|
||||
assert tokens == {"C1": "REPLYTOK1", "C2": "REPLYTOK2"}
|
||||
|
||||
|
||||
def test_comment_next_token_is_trailing_bare_continuation():
|
||||
# The page token is the last top-level continuationItemRenderer, not a
|
||||
# reply loader nested inside a thread.
|
||||
assert comment_next_token(_next_comments_response()) == "PAGE2"
|
||||
|
||||
|
||||
def test_comment_section_token_from_watch_page():
|
||||
initial = {
|
||||
"contents": {
|
||||
"twoColumnWatchNextResults": {
|
||||
"results": {
|
||||
"results": {
|
||||
"contents": [
|
||||
{
|
||||
"itemSectionRenderer": {
|
||||
"sectionIdentifier": "comment-item-section",
|
||||
"contents": [
|
||||
{
|
||||
"continuationItemRenderer": {
|
||||
"continuationEndpoint": {
|
||||
"continuationCommand": {"token": "SECTIONTOK"}
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
assert comment_section_token(initial) == "SECTIONTOK"
|
||||
|
||||
|
||||
# --- url resolution ----------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"url,kind,value",
|
||||
[
|
||||
("https://www.youtube.com/watch?v=dQw4w9WgXcQ", "video", "dQw4w9WgXcQ"),
|
||||
("https://youtu.be/dQw4w9WgXcQ", "video", "dQw4w9WgXcQ"),
|
||||
("https://www.youtube.com/shorts/abc123", "video", "abc123"),
|
||||
("https://www.youtube.com/@Apify", "channel", "Apify"),
|
||||
("https://www.youtube.com/channel/UC123456789abc/videos", "channel", "UC123456789abc"),
|
||||
("https://www.youtube.com/playlist?list=PL123", "playlist", "PL123"),
|
||||
("https://www.youtube.com/results?search_query=web+scraping", "search", "web scraping"),
|
||||
("https://www.youtube.com/hashtag/tech", "hashtag", "tech"),
|
||||
],
|
||||
)
|
||||
def test_resolve_url(url, kind, value):
|
||||
resolved = resolve_url(url)
|
||||
assert resolved is not None
|
||||
assert resolved.kind == kind
|
||||
assert resolved.value == value
|
||||
|
||||
|
||||
def test_resolve_url_unrecognized():
|
||||
assert resolve_url("https://example.com/foo") is None
|
||||
|
||||
|
||||
# --- optional: exercise captured real fixtures if present --------------------
|
||||
|
||||
|
||||
def test_parse_captured_video_fixture_if_present():
|
||||
player_path = _FIXTURE_DIR / "video_player_response.json"
|
||||
initial_path = _FIXTURE_DIR / "video_initial_data.json"
|
||||
if not player_path.exists():
|
||||
pytest.skip("no captured fixture (run scripts/e2e_youtube_scraper.py first)")
|
||||
html = (
|
||||
"<script>var ytInitialPlayerResponse = "
|
||||
+ player_path.read_text(encoding="utf-8")
|
||||
+ ";</script>"
|
||||
)
|
||||
if initial_path.exists():
|
||||
html += (
|
||||
"<script>var ytInitialData = "
|
||||
+ initial_path.read_text(encoding="utf-8")
|
||||
+ ";</script>"
|
||||
)
|
||||
result = parse_video_page(html)
|
||||
assert result is not None
|
||||
assert result["id"]
|
||||
assert result["title"]
|
||||
Loading…
Add table
Add a link
Reference in a new issue