mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-08 22:22:17 +02:00
feat(native-connector): added youtube scrapers
This commit is contained in:
DESKTOP-RTLN3BA\$punk
parent
f964e59686
commit
99378d7b63
19 changed files with 3959 additions and 0 deletions
|
|
@ -9,6 +9,12 @@ from fastapi import APIRouter, Depends, HTTPException, Query
|
|||
from scrapling.fetchers import AsyncFetcher
|
||||
|
||||
from app.auth.context import AuthContext
|
||||
from app.scrapers.youtube import (
|
||||
YouTubeCommentsInput,
|
||||
YouTubeScrapeInput,
|
||||
scrape_comments,
|
||||
scrape_youtube,
|
||||
)
|
||||
from app.users import require_session_context
|
||||
from app.utils.proxy import get_proxy_url
|
||||
|
||||
|
|
@ -26,6 +32,45 @@ _INNERTUBE_CLIENT = {
|
|||
}
|
||||
|
||||
|
||||
@router.post("/youtube/scrape")
|
||||
async def scrape_youtube_route(
|
||||
payload: YouTubeScrapeInput,
|
||||
_auth: AuthContext = Depends(require_session_context),
|
||||
) -> list[dict]:
|
||||
"""Scrape public YouTube data (search / video / channel / playlist / shorts).
|
||||
|
||||
Apify YouTube Scraper-compatible input/output. The scrape runs inline and is
|
||||
bounded only by the request's own ``maxResults`` / ``maxResultsShorts`` /
|
||||
``maxResultStreams`` — no separate per-request ceiling.
|
||||
"""
|
||||
try:
|
||||
return await scrape_youtube(payload)
|
||||
except Exception as e:
|
||||
logger.error("YouTube scrape failed: %s", e)
|
||||
raise HTTPException(
|
||||
status_code=502, detail=f"YouTube scrape failed: {e!s}"
|
||||
) from e
|
||||
|
||||
|
||||
@router.post("/youtube/comments")
|
||||
async def scrape_comments_route(
|
||||
payload: YouTubeCommentsInput,
|
||||
_auth: AuthContext = Depends(require_session_context),
|
||||
) -> list[dict]:
|
||||
"""Scrape YouTube comments (+ replies) for the given video URLs.
|
||||
|
||||
Apify "YouTube Comments Scraper"-compatible. Runs inline and is bounded only
|
||||
by the request's own ``maxComments`` — no separate per-request ceiling.
|
||||
"""
|
||||
try:
|
||||
return await scrape_comments(payload)
|
||||
except Exception as e:
|
||||
logger.error("YouTube comments scrape failed: %s", e)
|
||||
raise HTTPException(
|
||||
status_code=502, detail=f"YouTube comments scrape failed: {e!s}"
|
||||
) from e
|
||||
|
||||
|
||||
@router.get("/youtube/playlist-videos")
|
||||
async def get_playlist_videos(
|
||||
url: str = Query(..., description="YouTube playlist URL"),
|
||||
|
|
|
|||
1
surfsense_backend/app/scrapers/__init__.py
Normal file
1
surfsense_backend/app/scrapers/__init__.py
Normal file
|
|
@ -0,0 +1 @@
|
|||
"""Platform-native scrapers (non-connector data extraction tools)."""
|
||||
287
surfsense_backend/app/scrapers/youtube/README.md
Normal file
287
surfsense_backend/app/scrapers/youtube/README.md
Normal file
|
|
@ -0,0 +1,287 @@
|
|||
# YouTube Scraper
|
||||
|
||||
A platform-native YouTube scraper that is a **drop-in clone of the Apify
|
||||
"YouTube Scraper" and "YouTube Comments Scraper" actors** — same input surface,
|
||||
same output item shape. It talks to YouTube's internal **InnerTube** API plus
|
||||
the public watch/channel HTML, egresses through a residential proxy, and streams
|
||||
Apify-shaped dicts.
|
||||
|
||||
No API keys, no Apify account, no headless browser on the happy path.
|
||||
|
||||
---
|
||||
|
||||
## Quick start
|
||||
|
||||
```python
|
||||
from app.scrapers.youtube import (
|
||||
YouTubeScrapeInput, scrape_youtube,
|
||||
YouTubeCommentsInput, scrape_comments,
|
||||
)
|
||||
|
||||
# Videos — by search query and/or direct URLs (video/channel/playlist/hashtag/search)
|
||||
videos = await scrape_youtube(
|
||||
YouTubeScrapeInput(searchQueries=["surfsense"], maxResults=50)
|
||||
)
|
||||
videos = await scrape_youtube(
|
||||
YouTubeScrapeInput(startUrls=[{"url": "https://www.youtube.com/@SomeChannel"}],
|
||||
maxResults=20, downloadSubtitles=True)
|
||||
)
|
||||
|
||||
# Comments — one output item per top-level comment AND per reply
|
||||
comments = await scrape_comments(
|
||||
YouTubeCommentsInput(
|
||||
startUrls=[{"url": "https://www.youtube.com/watch?v=VIDEO_ID"}],
|
||||
maxComments=200, sortCommentsBy="TOP_COMMENTS",
|
||||
)
|
||||
)
|
||||
```
|
||||
|
||||
Both have a streaming twin — `iter_youtube()` / `iter_comments()` — that yields
|
||||
items as they arrive (unbounded, continuation-paged). `scrape_*` is just a
|
||||
collector with an optional `limit` guard.
|
||||
|
||||
The HTTP surface lives in `app/routes/youtube_routes.py`.
|
||||
|
||||
---
|
||||
|
||||
## Module map
|
||||
|
||||
| File | Responsibility |
|
||||
|------|----------------|
|
||||
| `__init__.py` | Public exports (entry points + schemas). |
|
||||
| `schemas.py` | Pydantic input/output models mirroring the Apify camelCase spec. `extra="allow"` on outputs keeps the contract open. |
|
||||
| `scraper.py` | Video orchestrator. Resolves URLs → per-flow async generators (`_video_flow`, `_search_flow`, `_channel_flow`, `_playlist_flow`), runs them through the `fan_out` worker pool. |
|
||||
| `comments.py` | Comments orchestrator. Watch page → comments-section continuation → `/next` paging, with concurrent per-thread reply fetching. |
|
||||
| `innertube.py` | **The network seam.** Proxy-only fetch (`fetch_html`, `post_innertube`), reusable sticky-IP sessions, reactive IP rotation, `StealthyFetcher` fallback, and the InnerTube payload builder. |
|
||||
| `parsers.py` | Pure, I/O-free JSON/HTML traversal + normalization (`find_all`/`find_first`/`dig`, `parse_video_page`, `parse_search_response`, comment/continuation token extractors, `parse_count`, …). |
|
||||
| `url_resolver.py` | Classify a URL into `video` / `channel` / `playlist` / `hashtag` / `search` and extract its id. |
|
||||
| `search_filters.py` | Encode Apify search filters into YouTube's `sp=` base64 protobuf (sort/date/type/length/feature flags), composable. |
|
||||
| `subtitles.py` | Subtitle download via `youtube-transcript-api`, shaped to Apify `subtitles[]`. |
|
||||
|
||||
Everything in `parsers.py` is deterministic and unit-tested offline; everything
|
||||
that touches the network is funneled through `innertube.py`.
|
||||
|
||||
---
|
||||
|
||||
## How it fetches (the important part)
|
||||
|
||||
All network I/O goes through **`fetch_html`** (GET watch/channel pages) and
|
||||
**`post_innertube`** (POST InnerTube `browse`/`search`/`next`). Design rules:
|
||||
|
||||
1. **Proxy-only egress.** Every request goes through the residential proxy
|
||||
(`app/utils/proxy.get_proxy_url`). We never connect directly — a direct hit
|
||||
would expose and risk-block the server IP.
|
||||
2. **Session reuse = sticky IP.** Within one flow (a continuation chain, or the
|
||||
jobs a worker pulls), a single keep-alive `FetcherSession` is reused. This
|
||||
roughly **halves warm latency** (~2.1s → ~1.0s) because only the first
|
||||
request pays the TCP+TLS handshake, and it pins one sticky exit IP instead of
|
||||
drawing a new (often slow) residential node per request.
|
||||
3. **Reactive IP rotation.** A sticky IP is kept until it's actually blocked. On
|
||||
`403`/`429` or a connection error, the session rotates to a fresh IP and
|
||||
retries, up to `_MAX_ROTATIONS` (3). A probe of 120 sequential requests on
|
||||
one IP saw zero blocks, so rotation is reactive, not proactive.
|
||||
4. **Browser fallback.** If all proxy attempts fail on an HTML page, `fetch_html`
|
||||
falls back to `StealthyFetcher` (headless, `solve_cloudflare=True`) in a
|
||||
worker thread. Optional — needs patchright browsers installed. Age-gated
|
||||
content requires login and is **not** bypassable.
|
||||
|
||||
The active session is bound to the current async task via a `ContextVar`
|
||||
(`_current_session`), so parsers and orchestrators never thread a session
|
||||
argument through every call — each concurrent flow transparently uses its own
|
||||
session/IP.
|
||||
|
||||
### InnerTube payloads
|
||||
|
||||
`build_innertube_payload(...)` builds the `WEB` client `context` payload
|
||||
(I/O-free, unit-testable). Some endpoints reject a keyless POST; `scraper._post`
|
||||
retries once with the public web key (`INNERTUBE_PUBLIC_API_KEY`) when the
|
||||
keyless call returns nothing. `hl=<lang>` on a `/next` call returns the
|
||||
creator-localized title/description (the translation flow).
|
||||
|
||||
---
|
||||
|
||||
## Concurrency model
|
||||
|
||||
Independent jobs — each `startUrl`, each `searchQuery`, each comment video — run
|
||||
concurrently through **`fan_out`**, a warm **worker pool** (`_FANOUT_CONCURRENCY
|
||||
= 16`):
|
||||
|
||||
- Each worker opens **one** proxy session and reuses it across the sequential
|
||||
jobs it pulls, so only the first job per worker pays the handshake.
|
||||
- **A bad job yields nothing rather than aborting the batch** (per-job
|
||||
try/except). One dead URL / comments-disabled video never kills the run.
|
||||
- Results stream out as each job finishes; **within** a flow, continuation
|
||||
paging stays sequential.
|
||||
- If the consumer stops early (collector hits its `limit`), workers are
|
||||
cancelled and **awaited** so every session's `finally` closes — no leaked
|
||||
keep-alive connections.
|
||||
|
||||
Comment reply threads for a page are fetched **concurrently** on the same
|
||||
multiplexed session (`asyncio.gather`), capped at the remaining budget.
|
||||
|
||||
---
|
||||
|
||||
## Data flow
|
||||
|
||||
- **Video by URL** → fetch watch HTML → `parse_video_page` (reads
|
||||
`ytInitialData` + `ytInitialPlayerResponse`) → optional subtitles + translation.
|
||||
- **Search** → InnerTube `/search` (+ `sp=` filter protobuf) → paginate via
|
||||
continuation tokens up to `maxResults`.
|
||||
- **Channel** → fetch the videos-tab seed once (reused for channel-wide metadata
|
||||
+ the About panel via `/browse`), then page `videos` / `shorts` / `streams`
|
||||
tabs, each capped independently (`maxResults` / `maxResultsShorts` /
|
||||
`maxResultStreams`). `sortVideosBy` uses the sort chips; `oldestPostDate` cuts
|
||||
off newest-first.
|
||||
- **Playlist** → `/browse` `VL<id>` → resolve each video via the video flow.
|
||||
- **Comments** → watch HTML seeds the comments-section token → `/next` returns
|
||||
comment entities + per-thread reply tokens + the page token. `maxComments`
|
||||
counts **every** emitted item (comments + replies).
|
||||
|
||||
### `commentsCount`
|
||||
|
||||
For the **comments** scraper, the authoritative total is read from the
|
||||
comments-section header (`commentsHeaderRenderer.countText`), not the watch-page
|
||||
HTML where it's lazy-loaded/absent. **Known gap:** the **video** scraper's
|
||||
`VideoItem.commentsCount` still comes from search/watch HTML and is often `null`
|
||||
— it would need an extra `/next` call to backfill (intentionally not done to
|
||||
keep the video path cheap).
|
||||
|
||||
---
|
||||
|
||||
## API spec
|
||||
|
||||
Mirrors the Apify "YouTube Scraper" and "YouTube Comments Scraper" actors
|
||||
(camelCase, `extra="allow"`). Inputs use Pydantic defaults; **every field is
|
||||
additive** — unknown inputs are accepted, unsourced outputs come back as
|
||||
`None`/`[]` — so parity grows without breaking consumers. `schemas.py` is the
|
||||
source of truth.
|
||||
|
||||
### Video scraper — input (`YouTubeScrapeInput`)
|
||||
|
||||
| Field | Type / values | Default | Notes |
|
||||
|-------|---------------|---------|-------|
|
||||
| `searchQueries` | `string[]` | `[]` | Discovery by query. Ignored when `startUrls` is set. |
|
||||
| `startUrls` | `[{ "url": string }]` | `[]` | Direct URLs: video, channel, playlist, hashtag, search. Overrides `searchQueries`. |
|
||||
| `maxResults` | `int ≥ 0` | `0` | Cap of regular videos **per query and per channel**. `0` = fetch none. |
|
||||
| `maxResultsShorts` | `int ≥ 0` | `0` | Cap of Shorts per channel. |
|
||||
| `maxResultStreams` | `int ≥ 0` | `0` | Cap of live/streams per channel. |
|
||||
| `downloadSubtitles` | `bool` | `false` | Populate `subtitles[]`. |
|
||||
| `subtitlesLanguage` | `string` | `"en"` | Also drives the translation flow when non-`en` (see `translatedTitle`). |
|
||||
| `subtitlesFormat` | `srt` \| `vtt` \| `xml` \| `plaintext` | `"srt"` | |
|
||||
| `preferAutoGeneratedSubtitles` | `bool` | `false` | |
|
||||
| `saveSubsToKVS` | `bool` | `false` | Accepted for parity; no-op (Apify key-value-store concept). |
|
||||
| `sortingOrder` | `relevance` \| `rating` \| `date` \| `views` | `null` | Search only. |
|
||||
| `dateFilter` | `hour` \| `today` \| `week` \| `month` \| `year` | `null` | Search only. |
|
||||
| `videoType` | `video` \| `movie` | `null` | Search only. |
|
||||
| `lengthFilter` | `under4` \| `between420` \| `plus20` | `null` | Search only (<4min / 4–20min / >20min). |
|
||||
| `isHD` `hasSubtitles` `hasCC` `is3D` `isLive` `isBought` `is4K` `is360` `hasLocation` `isHDR` `isVR180` | `bool` | `null` | Search feature filters (encoded into `sp=`). |
|
||||
| `oldestPostDate` | `string` (date) | `null` | Channel cutoff; day-accurate (relative times). |
|
||||
| `sortVideosBy` | `NEWEST` \| `POPULAR` \| `OLDEST` | `null` | Channel videos tab sort chip. |
|
||||
|
||||
### Video scraper — output (`VideoItem`)
|
||||
|
||||
| Field | Type | Populated? |
|
||||
|-------|------|-----------|
|
||||
| `title` `id` `url` `viewCount` `date` `duration` | str/int | yes |
|
||||
| `type` | `video` \| `shorts` \| `stream` | yes |
|
||||
| `thumbnailUrl` | str | yes |
|
||||
| `input` `fromYTUrl` `order` | str/int | yes (provenance: source query/URL, origin URL, index) |
|
||||
| `text` | str | yes (description) |
|
||||
| `descriptionLinks` | `[{ url, text }]` | yes |
|
||||
| `hashtags` | `string[]` | yes |
|
||||
| `likes` `commentsCount` `commentsTurnedOff` | int/bool | partial (often `null` on the video path — see `commentsCount` note) |
|
||||
| `location` | str | when present |
|
||||
| `collaborators` | `[{ name, username, url }]` | when present |
|
||||
| `translatedTitle` `translatedText` | str | when `subtitlesLanguage != "en"` |
|
||||
| `subtitles` | `[{ srtUrl, type, language, srt }]` | when `downloadSubtitles` |
|
||||
| `isMembersOnly` `isPaidContent` | bool | yes (default `false`) |
|
||||
| `isMonetized` `isAgeRestricted` | bool | best-effort (`null` when unknown) |
|
||||
| `channelName` `channelUrl` `channelUsername` `channelId` | str | yes |
|
||||
| `numberOfSubscribers` `channelTotalVideos` `channelTotalViews` | int | channel/deep fields |
|
||||
| `channelDescription` `channelLocation` `channelJoinedDate` | str | channel About panel |
|
||||
| `isChannelVerified` `channelBannerUrl` `channelAvatarUrl` | bool/str | channel fields |
|
||||
|
||||
### Comments scraper — input (`YouTubeCommentsInput`)
|
||||
|
||||
| Field | Type / values | Default | Notes |
|
||||
|-------|---------------|---------|-------|
|
||||
| `startUrls` | `[{ "url": string }]` | `[]` | Video URLs only (non-video URLs skipped). |
|
||||
| `maxComments` | `int ≥ 1` | `1` | Counts **every** emitted item (top-level comments **and** replies). |
|
||||
| `sortCommentsBy` | `TOP_COMMENTS` \| `NEWEST_FIRST` | `"NEWEST_FIRST"` | |
|
||||
| `oldestCommentDate` | `string` (date) | `null` | Forces newest-first and stops at the cutoff. |
|
||||
|
||||
### Comments scraper — output (`CommentItem`)
|
||||
|
||||
| Field | Type | Notes |
|
||||
|-------|------|-------|
|
||||
| `cid` | str | Comment id. |
|
||||
| `comment` | str | Text. |
|
||||
| `author` | str | |
|
||||
| `type` | `comment` \| `reply` | |
|
||||
| `replyToCid` | str | Parent `cid` (replies only). |
|
||||
| `replyCount` | int | Replies under a top-level comment. |
|
||||
| `voteCount` | int | Likes. |
|
||||
| `authorIsChannelOwner` `hasCreatorHeart` | bool | |
|
||||
| `publishedTimeText` | str | Relative time ("2 days ago"). |
|
||||
| `videoId` `pageUrl` `title` | str | Source video. |
|
||||
| `commentsCount` | int | Authoritative total from the comments header. |
|
||||
|
||||
---
|
||||
|
||||
## Configuration
|
||||
|
||||
- **Proxy** — required for real runs; configured via `app/utils/proxy.py`
|
||||
(residential rotating gateway env vars). With no proxy configured the fetchers
|
||||
fall back to one-shot direct `AsyncFetcher` calls (fine for local tests, not
|
||||
for production).
|
||||
- **Concurrency** — `scraper._FANOUT_CONCURRENCY` (16). The gateway handled 64
|
||||
parallel flows with zero failures in a ramp probe, so this leaves headroom.
|
||||
- **Rotation** — `innertube._BLOCK_STATUSES` (`403`, `429`) and
|
||||
`_MAX_ROTATIONS` (3).
|
||||
|
||||
---
|
||||
|
||||
## Testing
|
||||
|
||||
- **Offline unit tests** (no network) — run these on every change:
|
||||
```bash
|
||||
cd surfsense_backend
|
||||
.venv/Scripts/python.exe -m pytest tests/unit/scrapers/youtube/
|
||||
```
|
||||
- `test_parsers.py` — parser/normalization + filter-protobuf + URL-resolver
|
||||
cases against hand-built and (if present) captured real fixtures.
|
||||
- `test_fetch_resilience.py` — deterministic rotate-on-block (`429`/error →
|
||||
rotate → `200`, exhaustion, no-rotate on `404`, stealthy fallback) and the
|
||||
`fan_out` no-session-leak-on-early-stop guarantee, all with stubbed sessions.
|
||||
- **Live functional harness** — `scripts/e2e_youtube_scraper.py` (needs live
|
||||
network + optional proxy creds). Exercises video/search/channel/comments/
|
||||
location/collaborators/translation end to end, and **regenerates the offline
|
||||
fixtures** into `tests/unit/scrapers/youtube/fixtures/`:
|
||||
```bash
|
||||
.venv/Scripts/python.exe scripts/e2e_youtube_scraper.py
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Extending it
|
||||
|
||||
- **Add an output field** → populate it in the relevant `parsers.py` function
|
||||
and add it to `schemas.py`. Because outputs are `extra="allow"`, forgetting the
|
||||
schema line won't drop the value, but declaring it documents the contract.
|
||||
- **Add a URL kind** → extend `url_resolver.resolve_url` + add a `_*_flow` in
|
||||
`scraper.py` and a branch in `_dispatch`.
|
||||
- **Add a search filter** → add the field to `YouTubeScrapeInput` and encode it
|
||||
in `search_filters.build_search_params` (verify byte-for-byte against a real
|
||||
YouTube `sp=` token in the unit test).
|
||||
|
||||
### Known ceilings (grep `ponytail:` in the source for the live list)
|
||||
|
||||
- Hashtag pages are routed through search (`#tag` query), not a dedicated
|
||||
hashtag browse.
|
||||
- `oldestPostDate` / `oldestCommentDate` cutoffs are day-accurate at best
|
||||
(channel/list pages only expose coarse relative times like "2 years ago").
|
||||
- Keyless-vs-keyed InnerTube retry does one extra request on the keyed path
|
||||
instead of remembering which worked.
|
||||
- Video-path `commentsCount` (see above).
|
||||
- Playlist scraping reads only the first `/browse` page (~100 videos); paging
|
||||
the continuation token for longer playlists is not yet wired.
|
||||
16
surfsense_backend/app/scrapers/youtube/__init__.py
Normal file
16
surfsense_backend/app/scrapers/youtube/__init__.py
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
"""Platform-native YouTube scraper (Apify YouTube Scraper-compatible)."""
|
||||
|
||||
from .comments import iter_comments, scrape_comments
|
||||
from .schemas import CommentItem, VideoItem, YouTubeCommentsInput, YouTubeScrapeInput
|
||||
from .scraper import iter_youtube, scrape_youtube
|
||||
|
||||
__all__ = [
|
||||
"CommentItem",
|
||||
"VideoItem",
|
||||
"YouTubeCommentsInput",
|
||||
"YouTubeScrapeInput",
|
||||
"iter_comments",
|
||||
"iter_youtube",
|
||||
"scrape_comments",
|
||||
"scrape_youtube",
|
||||
]
|
||||
184
surfsense_backend/app/scrapers/youtube/comments.py
Normal file
184
surfsense_backend/app/scrapers/youtube/comments.py
Normal file
|
|
@ -0,0 +1,184 @@
|
|||
"""Orchestrator for the YouTube Comments scraper (Apify-compatible).
|
||||
|
||||
Distinct from the video scraper: one output item per top-level comment *and*
|
||||
per reply, sourced from the InnerTube ``/next`` endpoint. The watch page seeds a
|
||||
comments-section continuation; ``/next`` then returns comment entity payloads,
|
||||
per-thread reply tokens, and the paging token. ``maxComments`` counts every
|
||||
emitted item (comments + replies), matching Apify.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any
|
||||
|
||||
from .innertube import INNERTUBE_NEXT_URL, build_innertube_payload, fetch_html
|
||||
from .parsers import (
|
||||
comment_next_token,
|
||||
comment_reply_tokens,
|
||||
comment_section_token,
|
||||
comment_sort_tokens,
|
||||
dig,
|
||||
extract_yt_initial_data,
|
||||
find_first,
|
||||
parse_comment_entities,
|
||||
parse_count,
|
||||
parse_video_page,
|
||||
)
|
||||
from .scraper import _post, _published_date, fan_out
|
||||
from .schemas import CommentItem, YouTubeCommentsInput
|
||||
from .url_resolver import resolve_url
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Apify sort value -> YouTube sort-menu label. The section token loads "Top" by
|
||||
# default, so only "Newest" needs an explicit switch request.
|
||||
_SORT_LABELS = {"TOP_COMMENTS": "Top", "NEWEST_FIRST": "Newest"}
|
||||
|
||||
|
||||
async def _post_next(token: str) -> dict[str, Any] | None:
|
||||
return await _post(INNERTUBE_NEXT_URL, build_innertube_payload(continuation_token=token))
|
||||
|
||||
|
||||
def _finalize(partial: dict[str, Any], base: dict[str, Any], reply_to: str | None) -> dict:
|
||||
return CommentItem(**{**base, **partial, "replyToCid": reply_to}).to_output()
|
||||
|
||||
|
||||
def _older(entity: dict[str, Any], cutoff) -> bool:
|
||||
when = _published_date(entity.get("publishedTimeText"))
|
||||
return when is not None and when < cutoff
|
||||
|
||||
|
||||
async def _collect_replies(
|
||||
token: str, *, parent_cid: str, base: dict[str, Any], cutoff
|
||||
) -> list[dict]:
|
||||
"""Collect one thread's replies (``replyLevel`` 1) until exhausted."""
|
||||
replies: list[dict] = []
|
||||
while token:
|
||||
data = await _post_next(token)
|
||||
if not data:
|
||||
break
|
||||
for entity in parse_comment_entities(data):
|
||||
if cutoff and _older(entity, cutoff):
|
||||
continue
|
||||
replies.append(_finalize(entity, base, parent_cid))
|
||||
token = comment_next_token(data)
|
||||
return replies
|
||||
|
||||
|
||||
async def _comments_for_video(
|
||||
video_id: str, input_model: YouTubeCommentsInput
|
||||
) -> AsyncIterator[dict]:
|
||||
limit = input_model.maxComments
|
||||
page_url = f"https://www.youtube.com/watch?v={video_id}"
|
||||
html = await fetch_html(page_url)
|
||||
if not html:
|
||||
return
|
||||
initial = extract_yt_initial_data(html)
|
||||
if not initial:
|
||||
return
|
||||
section = comment_section_token(initial)
|
||||
if not section: # comments disabled or absent
|
||||
return
|
||||
|
||||
meta = parse_video_page(html) or {}
|
||||
base = {
|
||||
"videoId": video_id,
|
||||
"pageUrl": page_url,
|
||||
"title": meta.get("title"),
|
||||
"commentsCount": meta.get("commentsCount"),
|
||||
}
|
||||
|
||||
data = await _post_next(section)
|
||||
if not data:
|
||||
return
|
||||
|
||||
# Authoritative total lives in the comments-section header (exact integer),
|
||||
# not the watch-page HTML where the count is lazy-loaded / absent.
|
||||
header = find_first(data, "commentsHeaderRenderer")
|
||||
if header:
|
||||
base["commentsCount"] = parse_count(dig(header, "countText", "runs", 0, "text"))
|
||||
|
||||
# oldestCommentDate forces newest-first (Apify behavior).
|
||||
cutoff = _published_date(input_model.oldestCommentDate)
|
||||
sort = "NEWEST_FIRST" if cutoff else input_model.sortCommentsBy
|
||||
label = _SORT_LABELS[sort]
|
||||
if label == "Newest":
|
||||
sort_tokens = comment_sort_tokens(data)
|
||||
if label in sort_tokens:
|
||||
switched = await _post_next(sort_tokens[label])
|
||||
if switched:
|
||||
data = switched
|
||||
|
||||
count = 0
|
||||
while data:
|
||||
reply_tokens = comment_reply_tokens(data)
|
||||
entities = parse_comment_entities(data)
|
||||
|
||||
# Fetch this page's reply threads concurrently (the reused session
|
||||
# multiplexes requests), then emit in Apify order: comment, its replies.
|
||||
# ponytail: prefetch is capped at the remaining budget — each thread
|
||||
# yields >=1 item, so more threads than budget can't all be needed.
|
||||
pending = [
|
||||
(e["cid"], reply_tokens[e["cid"]])
|
||||
for e in entities
|
||||
if e["cid"] in reply_tokens
|
||||
][: max(limit - count, 0)]
|
||||
fetched = await asyncio.gather(
|
||||
*(
|
||||
_collect_replies(tok, parent_cid=cid, base=base, cutoff=cutoff)
|
||||
for cid, tok in pending
|
||||
)
|
||||
)
|
||||
replies_by_cid = {cid: r for (cid, _), r in zip(pending, fetched)}
|
||||
|
||||
for entity in entities:
|
||||
if count >= limit:
|
||||
return
|
||||
# Newest-first: the first too-old comment means the rest are older.
|
||||
if cutoff and _older(entity, cutoff):
|
||||
return
|
||||
yield _finalize(entity, base, None)
|
||||
count += 1
|
||||
|
||||
for reply in replies_by_cid.get(entity["cid"], []):
|
||||
if count >= limit:
|
||||
return
|
||||
yield reply
|
||||
count += 1
|
||||
|
||||
token = comment_next_token(data)
|
||||
if not token or count >= limit:
|
||||
return
|
||||
data = await _post_next(token)
|
||||
|
||||
|
||||
async def iter_comments(input_model: YouTubeCommentsInput) -> AsyncIterator[dict]:
|
||||
"""Yield Apify-shaped comment items for every startUrl video.
|
||||
|
||||
Videos are scraped concurrently (each video's own comment paging stays
|
||||
sequential), which is the dominant speedup when reviewing many videos.
|
||||
"""
|
||||
jobs = []
|
||||
for entry in input_model.startUrls:
|
||||
resolved = resolve_url(entry.url)
|
||||
if not resolved or resolved.kind != "video":
|
||||
logger.warning("Comments: not a video URL, skipping: %s", entry.url)
|
||||
continue
|
||||
jobs.append(_comments_for_video(resolved.value, input_model))
|
||||
async for comment in fan_out(jobs):
|
||||
yield comment
|
||||
|
||||
|
||||
async def scrape_comments(
|
||||
input_model: YouTubeCommentsInput, *, limit: int | None = None
|
||||
) -> list[dict]:
|
||||
"""Collect :func:`iter_comments` into a list, honoring an optional guard."""
|
||||
results: list[dict] = []
|
||||
async for comment in iter_comments(input_model):
|
||||
results.append(comment)
|
||||
if limit is not None and len(results) >= limit:
|
||||
break
|
||||
return results
|
||||
315
surfsense_backend/app/scrapers/youtube/innertube.py
Normal file
315
surfsense_backend/app/scrapers/youtube/innertube.py
Normal file
|
|
@ -0,0 +1,315 @@
|
|||
"""Proxy-aware fetch seam + InnerTube payload builder for the YouTube scraper.
|
||||
|
||||
All network I/O for the scraper flows through :func:`fetch_html` and
|
||||
:func:`post_innertube`, and always egresses through the residential proxy (never
|
||||
a direct connection — that would expose and risk-block the server IP). Within a
|
||||
continuation chain, :func:`proxy_session` opens ONE reused keep-alive session so
|
||||
sequential pages share a connection and a sticky exit IP (roughly halving warm
|
||||
latency vs a fresh rotating IP per request). HTML pages fall back to a
|
||||
``StealthyFetcher`` browser tier for anti-bot walls. Parsers and the orchestrator
|
||||
never see which tier served the bytes.
|
||||
|
||||
The ``AsyncFetcher.post(..., json=...)`` + ``page.json()`` / ``page.html_content``
|
||||
pattern is already proven in ``app/routes/youtube_routes.py``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from contextlib import asynccontextmanager
|
||||
from contextvars import ContextVar
|
||||
from typing import Any
|
||||
|
||||
from scrapling.fetchers import AsyncFetcher, FetcherSession
|
||||
|
||||
from app.utils.proxy import get_proxy_url
|
||||
|
||||
try: # browser tier is optional (needs patchright browsers installed)
|
||||
from scrapling.fetchers import StealthyFetcher
|
||||
except Exception: # pragma: no cover - import guard
|
||||
StealthyFetcher = None # type: ignore[assignment]
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Per-flow proxy session (set by ``proxy_session`` around one continuation
|
||||
# chain). Reusing one keep-alive connection halves warm latency AND pins a
|
||||
# single sticky exit IP on the rotating gateway, instead of paying a fresh
|
||||
# TCP+TLS handshake and drawing a new (often slow) residential node per request.
|
||||
# A ContextVar keeps each concurrent fan-out flow on its own session/IP without
|
||||
# threading a param through every parser call.
|
||||
_current_session: ContextVar[_RotatingSession | None] = ContextVar(
|
||||
"yt_proxy_session", default=None
|
||||
)
|
||||
|
||||
# Statuses that mean "this IP is throttled/blocked" -> rotate to a fresh one.
|
||||
# A probe of 120 sequential requests on one sticky IP saw zero blocks, so we
|
||||
# stay sticky for speed and rotate only *reactively*, up to this many times per
|
||||
# request. ponytail upgrade path: also rotate proactively every N requests if a
|
||||
# future run ever trips these.
|
||||
_BLOCK_STATUSES = frozenset({403, 429})
|
||||
_MAX_ROTATIONS = 3
|
||||
|
||||
|
||||
class _RotatingSession:
|
||||
"""Owns one live ``FetcherSession`` (sticky IP) and can swap it for a fresh one.
|
||||
|
||||
``rotate()`` closes the current keep-alive connection and opens a new one, so
|
||||
the rotating gateway hands out a different residential exit IP. Used
|
||||
sequentially within a single flow (never shared across concurrent tasks), so
|
||||
no locking is needed. ``session`` is ``None`` only when no proxy is configured.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._cm: Any | None = None
|
||||
self.session: Any | None = None
|
||||
self.rotations = 0
|
||||
|
||||
async def _open(self) -> None:
|
||||
proxy = get_proxy_url()
|
||||
if proxy is None:
|
||||
self._cm = self.session = None
|
||||
return
|
||||
self._cm = FetcherSession(
|
||||
proxy=proxy, stealthy_headers=True, impersonate="chrome"
|
||||
)
|
||||
self.session = await self._cm.__aenter__()
|
||||
|
||||
async def close(self) -> None:
|
||||
if self._cm is not None:
|
||||
try:
|
||||
await self._cm.__aexit__(None, None, None)
|
||||
except Exception: # best-effort teardown
|
||||
pass
|
||||
self._cm = self.session = None
|
||||
|
||||
async def rotate(self) -> Any | None:
|
||||
"""Drop the current IP and connect through a fresh one. Returns new session."""
|
||||
await self.close()
|
||||
self.rotations += 1
|
||||
await self._open()
|
||||
logger.info("[youtube] rotated proxy session (rotation #%d)", self.rotations)
|
||||
return self.session
|
||||
|
||||
|
||||
async def open_proxy_holder() -> _RotatingSession:
|
||||
"""Open a warm rotate-on-block session holder (caller owns ``close()``)."""
|
||||
holder = _RotatingSession()
|
||||
await holder._open()
|
||||
return holder
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def bind_proxy_holder(holder: _RotatingSession):
|
||||
"""Route this task's fetches through ``holder`` for the enclosed block.
|
||||
|
||||
Does NOT close the holder — enables pooling warm sessions across sequential
|
||||
jobs so each job skips the ~2s proxy TCP+TLS handshake.
|
||||
"""
|
||||
token = _current_session.set(holder)
|
||||
try:
|
||||
yield holder
|
||||
finally:
|
||||
_current_session.reset(token)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def proxy_session():
|
||||
"""Open one reused, rotate-on-block proxy session for a continuation chain."""
|
||||
holder = await open_proxy_holder()
|
||||
try:
|
||||
async with bind_proxy_holder(holder):
|
||||
yield holder
|
||||
finally:
|
||||
await holder.close()
|
||||
|
||||
# Consent cookies to dodge the EU consent interstitial that otherwise returns a
|
||||
# page with no ``ytInitialData``. Mirrors app/routes/youtube_routes.py.
|
||||
CONSENT_COOKIES = {
|
||||
"CONSENT": "PENDING+999",
|
||||
"SOCS": "CAISNQgDEitib3FfaWRlbnRpdHlmcm9udGVuZHVpc2VydmVyXzIwMjMwODI5LjA3X3AxGgJlbiADGgYIgOa_pgY",
|
||||
}
|
||||
|
||||
# Public InnerTube "WEB" client block. The web API key below is YouTube's
|
||||
# long-standing public key; ``search``/``next`` may reject a keyless POST where
|
||||
# ``browse`` accepts it, so the builder can attach both key and visitorData.
|
||||
INNERTUBE_PUBLIC_API_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"
|
||||
|
||||
INNERTUBE_BROWSE_URL = "https://www.youtube.com/youtubei/v1/browse"
|
||||
INNERTUBE_SEARCH_URL = "https://www.youtube.com/youtubei/v1/search"
|
||||
INNERTUBE_NEXT_URL = "https://www.youtube.com/youtubei/v1/next"
|
||||
|
||||
|
||||
def build_innertube_payload(
|
||||
*,
|
||||
continuation_token: str | None = None,
|
||||
browse_id: str | None = None,
|
||||
search_query: str | None = None,
|
||||
search_params: str | None = None,
|
||||
visitor_data: str | None = None,
|
||||
video_id: str | None = None,
|
||||
hl: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Build the standard InnerTube ``context.client`` payload.
|
||||
|
||||
Ported from the Scrapfly reference ``call_youtube_api`` but I/O-free so it is
|
||||
unit-testable and reusable across browse/search/next/player endpoints.
|
||||
``hl`` selects the response language: a ``/next`` call with a non-default
|
||||
``hl`` returns the video's creator-localized title/description.
|
||||
"""
|
||||
client: dict[str, Any] = {
|
||||
"hl": hl or "en",
|
||||
"gl": "US",
|
||||
"clientName": "WEB",
|
||||
"clientVersion": "2.20241111.07.00",
|
||||
"platform": "DESKTOP",
|
||||
"userInterfaceTheme": "USER_INTERFACE_THEME_DARK",
|
||||
}
|
||||
if visitor_data:
|
||||
client["visitorData"] = visitor_data
|
||||
|
||||
payload: dict[str, Any] = {
|
||||
"context": {
|
||||
"client": client,
|
||||
"user": {"lockedSafetyMode": False},
|
||||
"request": {"useSsl": True},
|
||||
}
|
||||
}
|
||||
if browse_id is not None:
|
||||
payload["browseId"] = browse_id
|
||||
if video_id is not None:
|
||||
payload["videoId"] = video_id
|
||||
if search_query is not None:
|
||||
payload["query"] = search_query
|
||||
if search_params is not None:
|
||||
payload["params"] = search_params
|
||||
if continuation_token is not None:
|
||||
payload["continuation"] = continuation_token
|
||||
return payload
|
||||
|
||||
|
||||
async def post_innertube(
|
||||
base_url: str,
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
api_key: str | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
"""POST an InnerTube payload and return parsed JSON (or ``None`` on failure).
|
||||
|
||||
Always egresses through the proxy (a reused per-flow session when one is
|
||||
open, else a one-shot ``AsyncFetcher``). Never connects directly — a direct
|
||||
hit would expose and risk-block the server IP.
|
||||
"""
|
||||
url = f"{base_url}?key={api_key}" if api_key else base_url
|
||||
holder = _current_session.get()
|
||||
for attempt in range(_MAX_ROTATIONS + 1):
|
||||
session = holder.session if holder else None
|
||||
try:
|
||||
started = time.perf_counter()
|
||||
if session is not None:
|
||||
page = await session.post(url, json=payload)
|
||||
else:
|
||||
page = await AsyncFetcher.post(
|
||||
url, json=payload, proxy=get_proxy_url(), stealthy_headers=True
|
||||
)
|
||||
fetch_ms = (time.perf_counter() - started) * 1000
|
||||
logger.info(
|
||||
"[youtube][perf] source=innertube reused=%s url=%s status=%s fetch_ms=%.1f",
|
||||
session is not None,
|
||||
base_url,
|
||||
page.status,
|
||||
fetch_ms,
|
||||
)
|
||||
if page.status == 200:
|
||||
return page.json()
|
||||
logger.warning("InnerTube POST %s returned %s", base_url, page.status)
|
||||
if not (holder and page.status in _BLOCK_STATUSES and attempt < _MAX_ROTATIONS):
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.warning("InnerTube POST %s failed: %s", base_url, e)
|
||||
if not (holder and attempt < _MAX_ROTATIONS):
|
||||
return None
|
||||
await holder.rotate() # blocked/errored on a proxy session: try a fresh IP
|
||||
return None
|
||||
|
||||
|
||||
async def fetch_html(url: str, *, cookies: dict[str, str] | None = None) -> str | None:
|
||||
"""GET a YouTube page and return raw HTML (or ``None`` on failure).
|
||||
|
||||
Fetches through the proxy (reused per-flow session when open, else a one-shot
|
||||
``AsyncFetcher``); falls back to the ``StealthyFetcher`` browser tier for
|
||||
anti-bot walls. Sets consent cookies by default so EU-egress fetches don't
|
||||
hit the consent wall and return a page without ``ytInitialData``.
|
||||
"""
|
||||
merged = {**CONSENT_COOKIES, **(cookies or {})}
|
||||
headers = {"Accept-Language": "en-US,en;q=0.9"}
|
||||
holder = _current_session.get()
|
||||
for attempt in range(_MAX_ROTATIONS + 1):
|
||||
session = holder.session if holder else None
|
||||
try:
|
||||
started = time.perf_counter()
|
||||
if session is not None:
|
||||
page = await session.get(url, headers=headers, cookies=merged)
|
||||
else:
|
||||
page = await AsyncFetcher.get(
|
||||
url,
|
||||
headers=headers,
|
||||
cookies=merged,
|
||||
proxy=get_proxy_url(),
|
||||
stealthy_headers=True,
|
||||
)
|
||||
fetch_ms = (time.perf_counter() - started) * 1000
|
||||
logger.info(
|
||||
"[youtube][perf] source=html reused=%s url=%s status=%s fetch_ms=%.1f",
|
||||
session is not None,
|
||||
url,
|
||||
page.status,
|
||||
fetch_ms,
|
||||
)
|
||||
if page.status == 200:
|
||||
return page.html_content
|
||||
logger.warning("HTML GET %s returned %s", url, page.status)
|
||||
if not (holder and page.status in _BLOCK_STATUSES and attempt < _MAX_ROTATIONS):
|
||||
break
|
||||
except Exception as e:
|
||||
logger.warning("HTML GET %s failed: %s", url, e)
|
||||
if not (holder and attempt < _MAX_ROTATIONS):
|
||||
break
|
||||
await holder.rotate() # blocked/errored on a proxy session: try a fresh IP
|
||||
# All proxy attempts blocked/failed -> last-resort browser tier.
|
||||
return await _fetch_html_stealthy(url)
|
||||
|
||||
|
||||
async def _fetch_html_stealthy(url: str) -> str | None:
|
||||
"""Last-resort browser fetch for anti-bot walls (mirrors the crawler tier).
|
||||
|
||||
``StealthyFetcher.fetch`` is synchronous, so it runs in a worker thread to
|
||||
keep the event loop free. Returns ``None`` when the browser tier is
|
||||
unavailable or still blocked.
|
||||
"""
|
||||
if StealthyFetcher is None:
|
||||
return None
|
||||
try:
|
||||
started = time.perf_counter()
|
||||
page = await asyncio.to_thread(
|
||||
StealthyFetcher.fetch,
|
||||
url,
|
||||
headless=True,
|
||||
network_idle=True,
|
||||
solve_cloudflare=True,
|
||||
proxy=get_proxy_url(),
|
||||
)
|
||||
fetch_ms = (time.perf_counter() - started) * 1000
|
||||
logger.info(
|
||||
"[youtube][perf] source=html tier=stealthy url=%s status=%s fetch_ms=%.1f",
|
||||
url,
|
||||
page.status,
|
||||
fetch_ms,
|
||||
)
|
||||
if page.status == 200:
|
||||
return page.html_content
|
||||
logger.warning("HTML GET %s tier=stealthy returned %s", url, page.status)
|
||||
except Exception as e:
|
||||
logger.warning("HTML GET %s tier=stealthy failed: %s", url, e)
|
||||
return None
|
||||
812
surfsense_backend/app/scrapers/youtube/parsers.py
Normal file
812
surfsense_backend/app/scrapers/youtube/parsers.py
Normal file
|
|
@ -0,0 +1,812 @@
|
|||
"""Pure parsing/normalization for YouTube page + InnerTube JSON.
|
||||
|
||||
I/O-free and dependency-free (no ``jmespath``/``jsonpath-ng``): all traversal
|
||||
uses two tiny helpers — :func:`find_all` (the ``$..key`` equivalent, same style
|
||||
as ``_extract_playlist_video_ids`` in ``app/routes/youtube_routes.py``) and
|
||||
:func:`dig` (null-safe positional get). Ported from the Scrapfly reference
|
||||
``references/scrapfly-scrapers/youtube-scraper/youtube.py`` with all brittle deep
|
||||
jmespath paths rewritten and the throwing ``convert_to_number`` replaced by a
|
||||
robust :func:`parse_count`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Traversal helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def find_all(obj: Any, key: str) -> list[Any]:
|
||||
"""Collect every value stored under ``key`` anywhere in a nested structure."""
|
||||
out: list[Any] = []
|
||||
|
||||
def _walk(node: Any) -> None:
|
||||
if isinstance(node, dict):
|
||||
for k, v in node.items():
|
||||
if k == key:
|
||||
out.append(v)
|
||||
_walk(v)
|
||||
elif isinstance(node, list):
|
||||
for item in node:
|
||||
_walk(item)
|
||||
|
||||
_walk(obj)
|
||||
return out
|
||||
|
||||
|
||||
def find_first(obj: Any, key: str) -> Any:
|
||||
"""First value stored under ``key`` anywhere in the structure, else ``None``."""
|
||||
matches = find_all(obj, key)
|
||||
return matches[0] if matches else None
|
||||
|
||||
|
||||
def find_last(obj: Any, key: str) -> Any:
|
||||
"""Last value stored under ``key`` (continuation tokens want the newest)."""
|
||||
matches = find_all(obj, key)
|
||||
return matches[-1] if matches else None
|
||||
|
||||
|
||||
def dig(obj: Any, *path: Any) -> Any:
|
||||
"""Null-safe positional get through dict keys / list indices."""
|
||||
cur = obj
|
||||
for step in path:
|
||||
if isinstance(step, int) and isinstance(cur, list):
|
||||
if -len(cur) <= step < len(cur):
|
||||
cur = cur[step]
|
||||
else:
|
||||
return None
|
||||
elif isinstance(cur, dict):
|
||||
cur = cur.get(step)
|
||||
else:
|
||||
return None
|
||||
if cur is None:
|
||||
return None
|
||||
return cur
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Value normalization
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_COUNT_RE = re.compile(r"([\d,.]+)\s*([KMB]?)", re.IGNORECASE)
|
||||
_MULTIPLIER = {"": 1, "K": 1_000, "M": 1_000_000, "B": 1_000_000_000}
|
||||
|
||||
|
||||
def parse_count(value: Any) -> int | None:
|
||||
"""Normalize a YouTube count string to an int, else ``None``.
|
||||
|
||||
Handles ``"451K views"``, ``"1.2M"``, ``"1,234"``, plain ints, and returns
|
||||
``None`` for ``"No views"`` / missing (the reference's ``convert_to_number``
|
||||
throws on all of these).
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, (int, float)):
|
||||
return int(value)
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
match = _COUNT_RE.search(value.strip())
|
||||
if not match:
|
||||
return None
|
||||
number, suffix = match.group(1), match.group(2).upper()
|
||||
number = number.replace(",", "")
|
||||
if number in ("", "."):
|
||||
return None
|
||||
try:
|
||||
return int(float(number) * _MULTIPLIER[suffix])
|
||||
except (ValueError, KeyError):
|
||||
return None
|
||||
|
||||
|
||||
def parse_date(microformat: dict | None) -> str | None:
|
||||
"""Prefer the real publish date from a video's microformat renderer.
|
||||
|
||||
``playerMicroformatRenderer.publishDate`` is an actual date (``2024-08-27``),
|
||||
unlike the relative ``"7 days ago"`` strings on list pages.
|
||||
"""
|
||||
if not microformat:
|
||||
return None
|
||||
return microformat.get("publishDate") or microformat.get("uploadDate")
|
||||
|
||||
|
||||
def seconds_to_duration(seconds: Any) -> str | None:
|
||||
"""Format a length in seconds as ``HH:MM:SS`` (Apify ``duration`` shape)."""
|
||||
total = parse_count(seconds)
|
||||
if total is None:
|
||||
return None
|
||||
hours, rem = divmod(total, 3600)
|
||||
minutes, secs = divmod(rem, 60)
|
||||
return f"{hours:02d}:{minutes:02d}:{secs:02d}"
|
||||
|
||||
|
||||
def _best_thumbnail(thumbnails: Any) -> str | None:
|
||||
"""Return the highest-resolution thumbnail URL from a thumbnails list."""
|
||||
if not isinstance(thumbnails, list) or not thumbnails:
|
||||
return None
|
||||
best = None
|
||||
best_area = -1
|
||||
for t in thumbnails:
|
||||
if not isinstance(t, dict) or "url" not in t:
|
||||
continue
|
||||
area = (t.get("width") or 0) * (t.get("height") or 0)
|
||||
if area >= best_area:
|
||||
best_area = area
|
||||
best = t["url"]
|
||||
return best
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Embedded page-JSON extraction (brace matcher, from youtube_routes.py)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _extract_json_object(html: str, patterns: list[re.Pattern[str]]) -> dict | None:
|
||||
start = -1
|
||||
for pattern in patterns:
|
||||
match = pattern.search(html)
|
||||
if match:
|
||||
start = match.end()
|
||||
break
|
||||
if start == -1:
|
||||
return None
|
||||
|
||||
depth = 0
|
||||
i = start
|
||||
while i < len(html):
|
||||
ch = html[i]
|
||||
if ch == "{":
|
||||
depth += 1
|
||||
elif ch == "}":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
break
|
||||
elif ch == '"':
|
||||
i += 1
|
||||
while i < len(html) and html[i] != '"':
|
||||
if html[i] == "\\":
|
||||
i += 1
|
||||
i += 1
|
||||
i += 1
|
||||
|
||||
try:
|
||||
return json.loads(html[start : i + 1])
|
||||
except (json.JSONDecodeError, IndexError):
|
||||
return None
|
||||
|
||||
|
||||
def extract_yt_initial_data(html: str) -> dict | None:
|
||||
"""Extract the ``ytInitialData`` object embedded in a YouTube page."""
|
||||
return _extract_json_object(
|
||||
html,
|
||||
[
|
||||
re.compile(r"var\s+ytInitialData\s*=\s*"),
|
||||
re.compile(r'window\["ytInitialData"\]\s*=\s*'),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def extract_yt_initial_player_response(html: str) -> dict | None:
|
||||
"""Extract the ``ytInitialPlayerResponse`` object from a video page."""
|
||||
return _extract_json_object(
|
||||
html,
|
||||
[
|
||||
re.compile(r"var\s+ytInitialPlayerResponse\s*=\s*"),
|
||||
re.compile(r'window\["ytInitialPlayerResponse"\]\s*=\s*'),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Page / API parsers -> partial VideoItem dicts
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _unwrap_redirect(url: str) -> str:
|
||||
"""Resolve a ``youtube.com/redirect?...&q=<target>`` link to its target."""
|
||||
if "/redirect" in url and "q=" in url:
|
||||
q = parse_qs(urlparse(url).query).get("q")
|
||||
if q:
|
||||
return q[0]
|
||||
if url.startswith("/"):
|
||||
return f"https://www.youtube.com{url}"
|
||||
return url
|
||||
|
||||
|
||||
def parse_description_links(initial: dict) -> list[dict[str, str]] | None:
|
||||
"""Extract ``{url, text}`` links from the video's attributed description.
|
||||
|
||||
Each ``commandRun`` marks a linked span (``startIndex``/``length``) into the
|
||||
description ``content``; external links are wrapped in a YouTube redirect
|
||||
whose real target is the ``q`` query param.
|
||||
"""
|
||||
ad = find_first(initial, "attributedDescription")
|
||||
if not isinstance(ad, dict):
|
||||
return None
|
||||
content = ad.get("content") or ""
|
||||
# YouTube's startIndex/length are UTF-16 code-unit offsets (JS string
|
||||
# semantics); emoji in the description are surrogate pairs, so slice on the
|
||||
# UTF-16-LE bytes to keep spans aligned instead of Python code points.
|
||||
units = content.encode("utf-16-le")
|
||||
out: list[dict[str, str]] = []
|
||||
for run in ad.get("commandRuns") or []:
|
||||
start, length = run.get("startIndex"), run.get("length")
|
||||
if not isinstance(start, int) or not isinstance(length, int):
|
||||
continue
|
||||
cmd = dig(run, "onTap", "innertubeCommand")
|
||||
url = dig(cmd, "urlEndpoint", "url") or dig(
|
||||
cmd, "commandMetadata", "webCommandMetadata", "url"
|
||||
)
|
||||
if not url:
|
||||
continue
|
||||
text = units[start * 2 : (start + length) * 2].decode("utf-16-le", "ignore")
|
||||
out.append({"url": _unwrap_redirect(url), "text": text})
|
||||
return out or None
|
||||
|
||||
|
||||
def _comments_turned_off(initial: dict) -> bool | None:
|
||||
"""Detect a disabled comments section, else ``None`` if no section is present.
|
||||
|
||||
An enabled section carries a ``continuationItemRenderer`` (the token that
|
||||
loads the comments); a disabled one keeps the section but drops the token.
|
||||
"""
|
||||
sections = [
|
||||
s
|
||||
for s in find_all(initial, "itemSectionRenderer")
|
||||
if isinstance(s, dict) and s.get("sectionIdentifier") == "comment-item-section"
|
||||
]
|
||||
if not sections:
|
||||
return None
|
||||
return not any(find_all(s, "continuationItemRenderer") for s in sections)
|
||||
|
||||
|
||||
def _is_members_only(player: dict, initial: dict) -> bool:
|
||||
"""True for members-only videos (a members badge or a members-only offer)."""
|
||||
for badge in find_all(initial, "metadataBadgeRenderer"):
|
||||
if isinstance(badge, dict) and badge.get("style") == "BADGE_STYLE_TYPE_MEMBERS_ONLY":
|
||||
return True
|
||||
status = player.get("playabilityStatus") or {}
|
||||
return find_first(status, "offerId") == "sponsors_only_video"
|
||||
|
||||
|
||||
def _is_age_restricted(player: dict, microformat: dict) -> bool | None:
|
||||
"""True when the player response signals an age gate, else ``None``.
|
||||
|
||||
``playabilityStatus`` is authoritative (age-gated videos report a legacy age
|
||||
gate reason or an "age"-mentioning reason); ``isFamilySafe`` is the fallback.
|
||||
"""
|
||||
status = player.get("playabilityStatus")
|
||||
if isinstance(status, dict):
|
||||
reason = (status.get("reason") or "").lower()
|
||||
if status.get("desktopLegacyAgeGateReason") or "age" in reason:
|
||||
return True
|
||||
if microformat.get("isFamilySafe") is False:
|
||||
return True
|
||||
return None
|
||||
|
||||
|
||||
def _channel_from_byline(renderer: dict) -> dict[str, Any]:
|
||||
"""Pull channel name/id/url from a list item's owner byline runs.
|
||||
|
||||
Search ``videoRenderer``s carry the uploader in ``ownerText`` /
|
||||
``longBylineText``; the run's ``browseEndpoint`` holds the channel id and the
|
||||
canonical ``/@handle`` (or ``/channel/UC...``) url.
|
||||
"""
|
||||
run = dig(renderer, "ownerText", "runs", 0) or dig(
|
||||
renderer, "longBylineText", "runs", 0
|
||||
)
|
||||
if not isinstance(run, dict):
|
||||
return {}
|
||||
browse = dig(run, "navigationEndpoint", "browseEndpoint") or {}
|
||||
cid = browse.get("browseId")
|
||||
base = browse.get("canonicalBaseUrl")
|
||||
out: dict[str, Any] = {}
|
||||
if run.get("text"):
|
||||
out["channelName"] = run["text"]
|
||||
if cid:
|
||||
out["channelId"] = cid
|
||||
if isinstance(base, str) and base:
|
||||
out["channelUrl"] = f"https://www.youtube.com{base}"
|
||||
if base.startswith("/@"):
|
||||
out["channelUsername"] = base[2:] or None
|
||||
elif cid:
|
||||
out["channelUrl"] = f"https://www.youtube.com/channel/{cid}"
|
||||
return out
|
||||
|
||||
|
||||
def parse_channel_metadata(initial_data: dict) -> dict[str, Any]:
|
||||
"""Channel identity from ``channelMetadataRenderer`` (in the channel HTML).
|
||||
|
||||
Zero extra fetch: name/id/url/avatar/description are already in the channel
|
||||
page's ``ytInitialData`` and apply to every video the channel flow yields.
|
||||
"""
|
||||
meta = find_first(initial_data, "channelMetadataRenderer")
|
||||
if not isinstance(meta, dict):
|
||||
return {}
|
||||
out: dict[str, Any] = {}
|
||||
if meta.get("title"):
|
||||
out["channelName"] = meta["title"]
|
||||
if meta.get("externalId"):
|
||||
out["channelId"] = meta["externalId"]
|
||||
if meta.get("description"):
|
||||
out["channelDescription"] = meta["description"]
|
||||
url = meta.get("vanityChannelUrl") or meta.get("channelUrl")
|
||||
if url:
|
||||
out["channelUrl"] = url
|
||||
avatar = _best_thumbnail(dig(meta, "avatar", "thumbnails"))
|
||||
if avatar:
|
||||
out["channelAvatarUrl"] = avatar
|
||||
banner = _best_thumbnail(
|
||||
dig(
|
||||
find_first(initial_data, "pageHeaderViewModel"),
|
||||
"banner",
|
||||
"imageBannerViewModel",
|
||||
"image",
|
||||
"sources",
|
||||
)
|
||||
)
|
||||
if banner:
|
||||
out["channelBannerUrl"] = banner
|
||||
return out
|
||||
|
||||
|
||||
def channel_about_tokens(initial_data: dict) -> list[str]:
|
||||
"""Continuation tokens for the channel's engagement panels (one is About).
|
||||
|
||||
The About panel isn't embedded in the channel HTML; it's fetched via a
|
||||
``/browse`` continuation. Panels aren't labeled, so callers try each token
|
||||
and keep the response that yields an ``aboutChannelViewModel``.
|
||||
"""
|
||||
tokens: list[str] = []
|
||||
for ep in find_all(initial_data, "showEngagementPanelEndpoint"):
|
||||
token = dig(find_first(ep, "continuationCommand"), "token")
|
||||
if token:
|
||||
tokens.append(token)
|
||||
return tokens
|
||||
|
||||
|
||||
def parse_channel_about(about: dict) -> dict[str, Any]:
|
||||
"""Deep channel fields from an ``aboutChannelViewModel``."""
|
||||
out: dict[str, Any] = {}
|
||||
if about.get("description"):
|
||||
out["channelDescription"] = about["description"]
|
||||
if about.get("country"):
|
||||
out["channelLocation"] = about["country"]
|
||||
subs = parse_count(about.get("subscriberCountText"))
|
||||
if subs is not None:
|
||||
out["numberOfSubscribers"] = subs
|
||||
views = parse_count(about.get("viewCountText"))
|
||||
if views is not None:
|
||||
out["channelTotalViews"] = views
|
||||
videos = parse_count(about.get("videoCountText"))
|
||||
if videos is not None:
|
||||
out["channelTotalVideos"] = videos
|
||||
joined = dig(about, "joinedDateText", "content") or about.get("joinedDateText")
|
||||
if isinstance(joined, str) and joined:
|
||||
# "Joined Feb 8, 2005" -> keep the date portion.
|
||||
out["channelJoinedDate"] = joined.replace("Joined", "").strip()
|
||||
return out
|
||||
|
||||
|
||||
# The primary-info super-title links to a geo-restricted search for tagged
|
||||
# videos; its a11y label is the reliable "is this a location?" signal (the same
|
||||
# slot holds hashtags otherwise). The label is English (hl=en client).
|
||||
_GEO_TAG_RE = re.compile(r"geo tagged with (.+)", re.IGNORECASE)
|
||||
|
||||
|
||||
def parse_location(initial: dict) -> str | None:
|
||||
"""Video geo-tag (place name) from ``videoPrimaryInfoRenderer.superTitleLink``.
|
||||
|
||||
Returns ``None`` when the super-title is hashtags (or absent) rather than a
|
||||
location. The place name is read from the a11y label ("...geo tagged with
|
||||
Rome"), which is proper-cased where the visible runs are upper-cased.
|
||||
"""
|
||||
st = dig(find_first(initial, "videoPrimaryInfoRenderer"), "superTitleLink")
|
||||
if not isinstance(st, dict):
|
||||
return None
|
||||
label = dig(st, "accessibility", "accessibilityData", "label") or ""
|
||||
match = _GEO_TAG_RE.search(label)
|
||||
return match.group(1).strip() if match else None
|
||||
|
||||
|
||||
def parse_translation(next_data: dict) -> tuple[str | None, str | None]:
|
||||
"""(title, description) from a ``/next`` response fetched with a target ``hl``.
|
||||
|
||||
YouTube renders the creator-localized title/description into
|
||||
``videoPrimaryInfoRenderer`` / ``videoSecondaryInfoRenderer`` for the request
|
||||
language. Videos without a localization return their original text.
|
||||
"""
|
||||
vpir = find_first(next_data, "videoPrimaryInfoRenderer") or {}
|
||||
title = "".join(r.get("text", "") for r in (dig(vpir, "title", "runs") or [])) or None
|
||||
|
||||
vsir = find_first(next_data, "videoSecondaryInfoRenderer") or {}
|
||||
description = dig(vsir, "attributedDescription", "content")
|
||||
if description is None:
|
||||
description = (
|
||||
"".join(r.get("text", "") for r in (dig(vsir, "description", "runs") or []))
|
||||
or None
|
||||
)
|
||||
return title, description
|
||||
|
||||
|
||||
def parse_collaborators(initial: dict) -> list[dict[str, str | None]] | None:
|
||||
"""Collaborator channels from the multi-owner "Collaborators" dialog.
|
||||
|
||||
Collaboration videos replace the owner's plain ``title`` with an
|
||||
``attributedTitle`` whose tap opens a dialog listing each credited channel
|
||||
(``listItemViewModel``). Returns ``None`` for ordinary single-owner videos.
|
||||
"""
|
||||
owner = dig(find_first(initial, "videoSecondaryInfoRenderer"), "owner", "videoOwnerRenderer")
|
||||
attributed = owner.get("attributedTitle") if isinstance(owner, dict) else None
|
||||
if not isinstance(attributed, dict):
|
||||
return None
|
||||
|
||||
# The dialog's own list holds one row per channel. Read those rows directly
|
||||
# (not find_all) — each row's subscribe button nests its own menu items.
|
||||
dialog = find_first(attributed, "dialogViewModel")
|
||||
rows = dig(find_first(dialog, "listViewModel"), "listItems") or []
|
||||
|
||||
collaborators: list[dict[str, str | None]] = []
|
||||
for row in rows:
|
||||
item = row.get("listItemViewModel") if isinstance(row, dict) else None
|
||||
name = dig(item, "title", "content")
|
||||
if not name:
|
||||
continue
|
||||
base = dig(find_first(item, "browseEndpoint"), "canonicalBaseUrl")
|
||||
collaborators.append(
|
||||
{
|
||||
"name": name,
|
||||
"username": base[2:] if isinstance(base, str) and base.startswith("/@") else None,
|
||||
"url": f"https://www.youtube.com{base}" if isinstance(base, str) and base else None,
|
||||
}
|
||||
)
|
||||
return collaborators or None
|
||||
|
||||
|
||||
def parse_video_page(html: str) -> dict[str, Any] | None:
|
||||
"""Parse a video/shorts watch page into a partial VideoItem dict.
|
||||
|
||||
Merges ``ytInitialPlayerResponse.videoDetails`` (title, views, length,
|
||||
description, thumbnails) with ``ytInitialData`` (likes, comment count,
|
||||
channel info) and the microformat renderer (real publish date).
|
||||
"""
|
||||
player = extract_yt_initial_player_response(html)
|
||||
initial = extract_yt_initial_data(html)
|
||||
if not player:
|
||||
return None
|
||||
|
||||
details = player.get("videoDetails") or {}
|
||||
microformat = dig(player, "microformat", "playerMicroformatRenderer") or {}
|
||||
video_id = details.get("videoId")
|
||||
|
||||
result: dict[str, Any] = {
|
||||
"id": video_id,
|
||||
"url": f"https://www.youtube.com/watch?v={video_id}" if video_id else None,
|
||||
"title": details.get("title"),
|
||||
"text": details.get("shortDescription"),
|
||||
"viewCount": parse_count(details.get("viewCount")),
|
||||
"duration": seconds_to_duration(details.get("lengthSeconds")),
|
||||
"date": parse_date(microformat),
|
||||
"thumbnailUrl": _best_thumbnail(dig(details, "thumbnail", "thumbnails")),
|
||||
"hashtags": details.get("keywords") or [],
|
||||
"channelName": details.get("author"),
|
||||
"channelId": details.get("channelId"),
|
||||
"isAgeRestricted": _is_age_restricted(player, microformat),
|
||||
# Monetized ⇔ the player response carries ad slots. ponytail: absence
|
||||
# isn't proof (a logged-out fetch may omit ads), so False→None, not False.
|
||||
"isMonetized": bool(player.get("adPlacements") or player.get("playerAds"))
|
||||
or None,
|
||||
"isMembersOnly": _is_members_only(player, initial or {}),
|
||||
# "Includes paid promotion" disclosure overlay.
|
||||
"isPaidContent": find_first(player, "paidContentOverlayRenderer") is not None,
|
||||
}
|
||||
|
||||
if initial:
|
||||
# Likes: the LIKE button's label carries the count string.
|
||||
likes = [
|
||||
b.get("title")
|
||||
for b in find_all(initial, "buttonViewModel")
|
||||
if isinstance(b, dict) and b.get("iconName") == "LIKE"
|
||||
]
|
||||
result["likes"] = parse_count(likes[0]) if likes else None
|
||||
|
||||
# Comment count lives in the engagement panel contextual info.
|
||||
contextual = find_first(initial, "contextualInfo")
|
||||
result["commentsCount"] = parse_count(dig(contextual, "runs", 0, "text"))
|
||||
result["commentsTurnedOff"] = _comments_turned_off(initial)
|
||||
result["descriptionLinks"] = parse_description_links(initial)
|
||||
result["location"] = parse_location(initial)
|
||||
result["collaborators"] = parse_collaborators(initial)
|
||||
|
||||
# Channel handle / URL from the canonical base url ("/@Handle").
|
||||
base = find_first(initial, "canonicalBaseUrl")
|
||||
if isinstance(base, str) and base:
|
||||
result["channelUrl"] = f"https://www.youtube.com{base}"
|
||||
# Only "/@handle" yields a username; "/channel/UC..." is an id, not a handle.
|
||||
if base.startswith("/@"):
|
||||
result["channelUsername"] = base[2:] or None
|
||||
|
||||
subs = find_first(initial, "subscriberCountText")
|
||||
result["numberOfSubscribers"] = parse_count(
|
||||
subs.get("simpleText") if isinstance(subs, dict) else subs
|
||||
)
|
||||
|
||||
# Verified badge on the channel owner.
|
||||
badges = find_all(initial, "metadataBadgeRenderer")
|
||||
result["isChannelVerified"] = any(
|
||||
isinstance(b, dict) and b.get("tooltip") == "Verified" for b in badges
|
||||
) or None
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _view_count_from_renderer(renderer: dict) -> int | None:
|
||||
for key in ("shortViewCountText", "viewCountText"):
|
||||
node = renderer.get(key)
|
||||
if isinstance(node, dict):
|
||||
text = node.get("simpleText") or dig(node, "runs", 0, "text")
|
||||
count = parse_count(text)
|
||||
if count is not None:
|
||||
return count
|
||||
return None
|
||||
|
||||
|
||||
def parse_search_response(data: dict) -> tuple[list[dict[str, Any]], str | None]:
|
||||
"""Parse an InnerTube search response into partial VideoItem dicts + token."""
|
||||
results: list[dict[str, Any]] = []
|
||||
for r in find_all(data, "videoRenderer"):
|
||||
if not isinstance(r, dict) or "videoId" not in r:
|
||||
continue
|
||||
vid = r["videoId"]
|
||||
results.append(
|
||||
{
|
||||
"id": vid,
|
||||
"url": f"https://www.youtube.com/watch?v={vid}",
|
||||
"title": dig(r, "title", "runs", 0, "text"),
|
||||
"text": dig(
|
||||
r, "detailedMetadataSnippets", 0, "snippetText", "runs", 0, "text"
|
||||
),
|
||||
"date": None, # list pages only expose relative time
|
||||
"publishedTimeText": dig(r, "publishedTimeText", "simpleText"),
|
||||
"duration": dig(r, "lengthText", "simpleText"),
|
||||
"viewCount": _view_count_from_renderer(r),
|
||||
"thumbnailUrl": _best_thumbnail(dig(r, "thumbnail", "thumbnails")),
|
||||
**_channel_from_byline(r),
|
||||
}
|
||||
)
|
||||
return results, _continuation_token(data)
|
||||
|
||||
|
||||
def _continuation_token(data: dict) -> str | None:
|
||||
tokens = [
|
||||
t.get("token")
|
||||
for t in find_all(data, "continuationCommand")
|
||||
if isinstance(t, dict) and t.get("token")
|
||||
]
|
||||
return tokens[-1] if tokens else None
|
||||
|
||||
|
||||
def parse_playlist_video_ids(data: dict) -> list[str]:
|
||||
"""Ordered, de-duped video ids from a playlist ``/browse`` response.
|
||||
|
||||
Playlist entries are ``lockupViewModel`` (the old ``playlistVideoRenderer``
|
||||
is retired); ``contentId`` holds the id. The 11-char guard keeps only videos
|
||||
and drops any playlist/channel lockup (e.g. the sidebar self-lockup).
|
||||
"""
|
||||
seen: set[str] = set()
|
||||
ids: list[str] = []
|
||||
for lockup in find_all(data, "lockupViewModel"):
|
||||
vid = lockup.get("contentId") if isinstance(lockup, dict) else None
|
||||
if vid and len(vid) == 11 and vid not in seen:
|
||||
seen.add(vid)
|
||||
ids.append(vid)
|
||||
return ids
|
||||
|
||||
|
||||
def parse_channel_videos(data: dict) -> tuple[list[dict[str, Any]], str | None]:
|
||||
"""Parse an InnerTube channel-videos browse response (richItem lockups)."""
|
||||
results: list[dict[str, Any]] = []
|
||||
for item in find_all(data, "richItemRenderer"):
|
||||
lockup = dig(item, "content", "lockupViewModel")
|
||||
if not isinstance(lockup, dict):
|
||||
continue
|
||||
vid = lockup.get("contentId")
|
||||
if not vid:
|
||||
continue
|
||||
meta = dig(lockup, "metadata", "lockupMetadataViewModel")
|
||||
rows = dig(meta, "metadata", "contentMetadataViewModel", "metadataRows") or []
|
||||
parts = dig(rows, 0, "metadataParts") or []
|
||||
results.append(
|
||||
{
|
||||
"id": vid,
|
||||
"url": f"https://www.youtube.com/watch?v={vid}",
|
||||
"title": dig(meta, "title", "content"),
|
||||
"viewCount": parse_count(dig(parts, 0, "text", "content")),
|
||||
"date": None,
|
||||
"publishedTimeText": dig(parts, 1, "text", "content"),
|
||||
"duration": _lockup_duration(lockup),
|
||||
"thumbnailUrl": _best_thumbnail(
|
||||
dig(lockup, "contentImage", "thumbnailViewModel", "image", "sources")
|
||||
),
|
||||
}
|
||||
)
|
||||
return results, _continuation_token(data)
|
||||
|
||||
|
||||
def parse_channel_shorts(data: dict) -> tuple[list[dict[str, Any]], str | None]:
|
||||
"""Parse a channel Shorts browse/seed response (``shortsLockupViewModel``).
|
||||
|
||||
Shorts use a different lockup than videos/streams: the id is on the reel
|
||||
watch endpoint, and title/views live in ``overlayMetadata``.
|
||||
"""
|
||||
results: list[dict[str, Any]] = []
|
||||
for s in find_all(data, "shortsLockupViewModel"):
|
||||
if not isinstance(s, dict):
|
||||
continue
|
||||
vid = dig(s, "onTap", "innertubeCommand", "reelWatchEndpoint", "videoId")
|
||||
if not vid:
|
||||
entity = s.get("entityId") or ""
|
||||
vid = entity.rsplit("-", 1)[-1] if entity else None
|
||||
if not vid:
|
||||
continue
|
||||
results.append(
|
||||
{
|
||||
"id": vid,
|
||||
"url": f"https://www.youtube.com/shorts/{vid}",
|
||||
"title": dig(s, "overlayMetadata", "primaryText", "content"),
|
||||
"viewCount": parse_count(
|
||||
dig(s, "overlayMetadata", "secondaryText", "content")
|
||||
),
|
||||
"date": None,
|
||||
"thumbnailUrl": _best_thumbnail(
|
||||
dig(s, "thumbnailViewModel", "image", "sources")
|
||||
),
|
||||
}
|
||||
)
|
||||
return results, _continuation_token(data)
|
||||
|
||||
|
||||
def _lockup_duration(lockup: dict) -> str | None:
|
||||
overlays = dig(
|
||||
lockup, "contentImage", "thumbnailViewModel", "overlays"
|
||||
) or []
|
||||
for overlay in overlays:
|
||||
text = dig(
|
||||
overlay,
|
||||
"thumbnailBottomOverlayViewModel",
|
||||
"badges",
|
||||
0,
|
||||
"thumbnailBadgeViewModel",
|
||||
"text",
|
||||
)
|
||||
if text:
|
||||
return text
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Comments (/next endpoint)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def comment_section_token(initial_data: dict) -> str | None:
|
||||
"""The watch page's comments-section continuation (seeds the /next call)."""
|
||||
for section in find_all(initial_data, "itemSectionRenderer"):
|
||||
if (
|
||||
isinstance(section, dict)
|
||||
and section.get("sectionIdentifier") == "comment-item-section"
|
||||
):
|
||||
token = dig(
|
||||
find_first(section, "continuationItemRenderer"),
|
||||
"continuationEndpoint",
|
||||
"continuationCommand",
|
||||
"token",
|
||||
)
|
||||
if token:
|
||||
return token
|
||||
return None
|
||||
|
||||
|
||||
def comment_sort_tokens(data: dict) -> dict[str, str]:
|
||||
"""Map the comment sort labels ("Top"/"Newest") to their continuations."""
|
||||
tokens: dict[str, str] = {}
|
||||
sub = find_first(data, "sortFilterSubMenuRenderer")
|
||||
for item in dig(sub, "subMenuItems") or []:
|
||||
title = item.get("title") if isinstance(item, dict) else None
|
||||
token = dig(item, "serviceEndpoint", "continuationCommand", "token")
|
||||
if title and token:
|
||||
tokens[title] = token
|
||||
return tokens
|
||||
|
||||
|
||||
def _comment_continuation_items(data: dict) -> list[Any]:
|
||||
items: list[Any] = []
|
||||
for key in ("reloadContinuationItemsCommand", "appendContinuationItemsAction"):
|
||||
for action in find_all(data, key):
|
||||
if isinstance(action, dict):
|
||||
items += action.get("continuationItems") or []
|
||||
return items
|
||||
|
||||
|
||||
def comment_next_token(data: dict) -> str | None:
|
||||
"""Pagination token: the trailing bare ``continuationItemRenderer``.
|
||||
|
||||
Reply loaders are nested inside ``commentThreadRenderer``s; the page's own
|
||||
"load more" token is the last top-level continuation item, so scan the
|
||||
action's ``continuationItems`` from the end.
|
||||
"""
|
||||
for item in reversed(_comment_continuation_items(data)):
|
||||
if isinstance(item, dict) and "continuationItemRenderer" in item:
|
||||
token = dig(
|
||||
item,
|
||||
"continuationItemRenderer",
|
||||
"continuationEndpoint",
|
||||
"continuationCommand",
|
||||
"token",
|
||||
)
|
||||
if token:
|
||||
return token
|
||||
return None
|
||||
|
||||
|
||||
def comment_reply_tokens(data: dict) -> dict[str, str]:
|
||||
"""Map each top-level comment id to its replies continuation token."""
|
||||
tokens: dict[str, str] = {}
|
||||
for thread in find_all(data, "commentThreadRenderer"):
|
||||
cid = dig(thread, "commentViewModel", "commentViewModel", "commentId")
|
||||
token = dig(
|
||||
find_first(thread, "continuationItemRenderer"),
|
||||
"continuationEndpoint",
|
||||
"continuationCommand",
|
||||
"token",
|
||||
)
|
||||
if cid and token:
|
||||
tokens[cid] = token
|
||||
return tokens
|
||||
|
||||
|
||||
def parse_comment_entity(cep: dict) -> dict[str, Any]:
|
||||
"""Map one ``commentEntityPayload`` to a partial CommentItem dict."""
|
||||
props = cep.get("properties") or {}
|
||||
author = cep.get("author") or {}
|
||||
toolbar = cep.get("toolbar") or {}
|
||||
return {
|
||||
"cid": props.get("commentId"),
|
||||
"comment": dig(props, "content", "content"),
|
||||
"author": author.get("displayName"),
|
||||
"publishedTimeText": props.get("publishedTime"),
|
||||
"voteCount": parse_count(toolbar.get("likeCountNotliked")),
|
||||
"replyCount": parse_count(toolbar.get("replyCount")),
|
||||
"authorIsChannelOwner": bool(author.get("isCreator")),
|
||||
# A creator heart attaches the channel owner's avatar to the toolbar.
|
||||
"hasCreatorHeart": bool(toolbar.get("creatorThumbnailUrl")),
|
||||
"type": "comment" if (props.get("replyLevel") or 0) == 0 else "reply",
|
||||
}
|
||||
|
||||
|
||||
def parse_comment_entities(data: dict) -> list[dict[str, Any]]:
|
||||
"""All comment payloads in a /next response, in display order."""
|
||||
return [
|
||||
parse_comment_entity(cep)
|
||||
for cep in find_all(data, "commentEntityPayload")
|
||||
if isinstance(cep, dict)
|
||||
]
|
||||
|
||||
|
||||
def parse_channel_sort_tokens(initial_data: dict) -> dict[str, str]:
|
||||
"""Map channel-videos sort labels ("Latest"/"Popular"/"Oldest") to tokens."""
|
||||
tokens: dict[str, str] = {}
|
||||
for chip in find_all(initial_data, "chipViewModel"):
|
||||
if not isinstance(chip, dict):
|
||||
continue
|
||||
label = chip.get("text")
|
||||
token = dig(
|
||||
chip, "tapCommand", "innertubeCommand", "continuationCommand", "token"
|
||||
)
|
||||
if label and token:
|
||||
tokens[label] = token
|
||||
return tokens
|
||||
210
surfsense_backend/app/scrapers/youtube/schemas.py
Normal file
210
surfsense_backend/app/scrapers/youtube/schemas.py
Normal file
|
|
@ -0,0 +1,210 @@
|
|||
# ruff: noqa: N815 - field names intentionally mirror the Apify camelCase spec
|
||||
"""Apify-compatible input/output models for the YouTube scraper.
|
||||
|
||||
The models mirror the public Apify "YouTube Scraper" actor spec so the endpoint
|
||||
can be a drop-in. The MVP populates a reliably-sourced subset; every other field
|
||||
is accepted (input) or emitted as ``None``/``[]`` (output) so parity is additive.
|
||||
|
||||
``VideoItem`` uses ``extra="allow"`` on purpose: the Apify spec explicitly says
|
||||
"Other properties may be included", and it lets us grow the output shape without
|
||||
breaking existing consumers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
SubtitlesFormat = Literal["srt", "vtt", "xml", "plaintext"]
|
||||
SortingOrder = Literal["relevance", "rating", "date", "views"]
|
||||
DateFilter = Literal["hour", "today", "week", "month", "year"]
|
||||
VideoTypeFilter = Literal["video", "movie"]
|
||||
LengthFilter = Literal["under4", "between420", "plus20"]
|
||||
SortVideosBy = Literal["NEWEST", "POPULAR", "OLDEST"]
|
||||
VideoContentType = Literal["video", "shorts", "stream"]
|
||||
SortCommentsBy = Literal["TOP_COMMENTS", "NEWEST_FIRST"]
|
||||
CommentType = Literal["comment", "reply"]
|
||||
|
||||
|
||||
class StartUrl(BaseModel):
|
||||
"""A single direct URL entry (Apify passes ``{"url": ...}`` objects)."""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
url: str
|
||||
|
||||
|
||||
class YouTubeScrapeInput(BaseModel):
|
||||
"""Full Apify input surface.
|
||||
|
||||
``maxResults*`` semantics follow Apify: ``0`` means "fetch none of that type"
|
||||
(see the ``startUrls`` note in the spec), NOT unlimited. A caller wanting an
|
||||
unbounded run leaves the value high; the request-time guard lives in the
|
||||
route, not here.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
# Discovery
|
||||
searchQueries: list[str] = Field(default_factory=list)
|
||||
startUrls: list[StartUrl] = Field(default_factory=list)
|
||||
|
||||
# Per-type caps (independent, applied per search term and per channel)
|
||||
maxResults: int = Field(default=0, ge=0, le=999999)
|
||||
maxResultsShorts: int = Field(default=0, ge=0, le=999999)
|
||||
maxResultStreams: int = Field(default=0, ge=0, le=999999)
|
||||
|
||||
# Subtitles
|
||||
downloadSubtitles: bool = False
|
||||
subtitlesLanguage: str = "en"
|
||||
subtitlesFormat: SubtitlesFormat = "srt"
|
||||
saveSubsToKVS: bool = False
|
||||
preferAutoGeneratedSubtitles: bool = False
|
||||
|
||||
# Search filters
|
||||
sortingOrder: SortingOrder | None = None
|
||||
dateFilter: DateFilter | None = None
|
||||
videoType: VideoTypeFilter | None = None
|
||||
lengthFilter: LengthFilter | None = None
|
||||
isHD: bool | None = None
|
||||
hasSubtitles: bool | None = None
|
||||
hasCC: bool | None = None
|
||||
is3D: bool | None = None
|
||||
isLive: bool | None = None
|
||||
isBought: bool | None = None
|
||||
is4K: bool | None = None
|
||||
is360: bool | None = None
|
||||
hasLocation: bool | None = None
|
||||
isHDR: bool | None = None
|
||||
isVR180: bool | None = None
|
||||
|
||||
# Channel/date scoping
|
||||
oldestPostDate: str | None = None
|
||||
sortVideosBy: SortVideosBy | None = None
|
||||
|
||||
|
||||
class DescriptionLink(BaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
url: str | None = None
|
||||
text: str | None = None
|
||||
|
||||
|
||||
class SubtitleTrack(BaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
srtUrl: str | None = None
|
||||
type: str | None = None
|
||||
language: str | None = None
|
||||
srt: str | None = None
|
||||
|
||||
|
||||
class Collaborator(BaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
name: str | None = None
|
||||
username: str | None = None
|
||||
url: str | None = None
|
||||
|
||||
|
||||
class VideoItem(BaseModel):
|
||||
"""Apify output item. Unsourced fields default to ``None``/``[]``.
|
||||
|
||||
``extra="allow"`` keeps the contract open (Apify: "Other properties may be
|
||||
included") so gradually-added fields never break existing consumers.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
# Core video
|
||||
title: str | None = None
|
||||
id: str | None = None
|
||||
url: str | None = None
|
||||
viewCount: int | None = None
|
||||
date: str | None = None
|
||||
duration: str | None = None
|
||||
type: VideoContentType = "video"
|
||||
thumbnailUrl: str | None = None
|
||||
input: str | None = None
|
||||
fromYTUrl: str | None = None
|
||||
order: int | None = None
|
||||
|
||||
# Engagement / content
|
||||
likes: int | None = None
|
||||
commentsCount: int | None = None
|
||||
commentsTurnedOff: bool | None = None
|
||||
text: str | None = None
|
||||
descriptionLinks: list[DescriptionLink] | None = None
|
||||
subtitles: list[SubtitleTrack] | None = None
|
||||
hashtags: list[str] = Field(default_factory=list)
|
||||
location: str | None = None
|
||||
|
||||
# Translations
|
||||
translatedTitle: str | None = None
|
||||
translatedText: str | None = None
|
||||
|
||||
# Video flags
|
||||
isMonetized: bool | None = None
|
||||
isMembersOnly: bool = False
|
||||
isPaidContent: bool = False
|
||||
isAgeRestricted: bool | None = None
|
||||
collaborators: list[Collaborator] | None = None
|
||||
|
||||
# Channel
|
||||
channelName: str | None = None
|
||||
channelUrl: str | None = None
|
||||
channelUsername: str | None = None
|
||||
channelId: str | None = None
|
||||
numberOfSubscribers: int | None = None
|
||||
channelTotalVideos: int | None = None
|
||||
channelDescription: str | None = None
|
||||
channelLocation: str | None = None
|
||||
channelJoinedDate: str | None = None
|
||||
channelTotalViews: int | None = None
|
||||
isChannelVerified: bool | None = None
|
||||
channelBannerUrl: str | None = None
|
||||
channelAvatarUrl: str | None = None
|
||||
|
||||
def to_output(self) -> dict[str, Any]:
|
||||
"""Serialize to the flat dict shape Apify emits (keeps extras)."""
|
||||
return self.model_dump(exclude_none=False)
|
||||
|
||||
|
||||
class YouTubeCommentsInput(BaseModel):
|
||||
"""Apify "YouTube Comments Scraper" input surface.
|
||||
|
||||
``maxComments`` counts every emitted item (top-level comments and replies).
|
||||
Setting ``oldestCommentDate`` forces newest-first sort, matching Apify.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
startUrls: list[StartUrl] = Field(default_factory=list)
|
||||
maxComments: int = Field(default=1, ge=1, le=999999)
|
||||
sortCommentsBy: SortCommentsBy = "NEWEST_FIRST"
|
||||
oldestCommentDate: str | None = None
|
||||
|
||||
|
||||
class CommentItem(BaseModel):
|
||||
"""Apify comment output item (one per top-level comment or reply)."""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
author: str | None = None
|
||||
comment: str | None = None
|
||||
cid: str | None = None
|
||||
replyCount: int | None = None
|
||||
replyToCid: str | None = None
|
||||
voteCount: int | None = None
|
||||
authorIsChannelOwner: bool | None = None
|
||||
publishedTimeText: str | None = None
|
||||
type: CommentType | None = None
|
||||
hasCreatorHeart: bool | None = None
|
||||
videoId: str | None = None
|
||||
pageUrl: str | None = None
|
||||
commentsCount: int | None = None
|
||||
title: str | None = None
|
||||
|
||||
def to_output(self) -> dict[str, Any]:
|
||||
return self.model_dump(exclude_none=False)
|
||||
493
surfsense_backend/app/scrapers/youtube/scraper.py
Normal file
493
surfsense_backend/app/scrapers/youtube/scraper.py
Normal file
|
|
@ -0,0 +1,493 @@
|
|||
"""Orchestrator for the YouTube scraper.
|
||||
|
||||
The core is the async generator :func:`iter_youtube` (unbounded / continuation
|
||||
paged); :func:`scrape_youtube` is a thin collector with a caller-supplied
|
||||
``limit`` guard. Per-type counters (regular / shorts / streams) are applied
|
||||
independently per search term and per channel, matching Apify semantics. Any cap
|
||||
is caller policy, never baked into flow logic.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
from .innertube import (
|
||||
INNERTUBE_BROWSE_URL,
|
||||
INNERTUBE_NEXT_URL,
|
||||
INNERTUBE_PUBLIC_API_KEY,
|
||||
INNERTUBE_SEARCH_URL,
|
||||
bind_proxy_holder,
|
||||
build_innertube_payload,
|
||||
fetch_html,
|
||||
open_proxy_holder,
|
||||
post_innertube,
|
||||
)
|
||||
from .parsers import (
|
||||
channel_about_tokens,
|
||||
extract_yt_initial_data,
|
||||
find_first,
|
||||
parse_channel_about,
|
||||
parse_channel_metadata,
|
||||
parse_channel_shorts,
|
||||
parse_channel_sort_tokens,
|
||||
parse_channel_videos,
|
||||
parse_playlist_video_ids,
|
||||
parse_search_response,
|
||||
parse_translation,
|
||||
parse_video_page,
|
||||
)
|
||||
from .schemas import VideoItem, YouTubeScrapeInput
|
||||
from .search_filters import build_search_params
|
||||
from .subtitles import fetch_subtitles
|
||||
from .url_resolver import ResolvedUrl, resolve_url
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SORT_LABELS = {"NEWEST": "Latest", "POPULAR": "Popular", "OLDEST": "Oldest"}
|
||||
|
||||
# Independent jobs (one per startUrl / search query / video) run concurrently on
|
||||
# a pool of warm proxy sessions (sticky IPs). A ramp probe on the gateway ran 64
|
||||
# parallel flows with zero failures, so the proxy is not the ceiling; 16 workers
|
||||
# saturate typical job counts while leaving gateway headroom for other callers.
|
||||
_FANOUT_CONCURRENCY = 16
|
||||
|
||||
|
||||
async def fan_out(
|
||||
jobs: list[AsyncIterator[dict[str, Any]]], *, concurrency: int = _FANOUT_CONCURRENCY
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
"""Stream items from independent async-iterator jobs via a warm worker pool.
|
||||
|
||||
Each worker opens ONE proxy session and reuses it across the sequential jobs
|
||||
it pulls, so only the first job per worker pays the ~2s proxy TCP+TLS
|
||||
handshake. A bad job yields nothing rather than aborting the batch; results
|
||||
stream out as each job finishes. Workers are cancelled if the consumer stops
|
||||
early (e.g. the collector hits its limit).
|
||||
"""
|
||||
if not jobs:
|
||||
return
|
||||
job_queue: asyncio.Queue[AsyncIterator[dict[str, Any]]] = asyncio.Queue()
|
||||
for job in jobs:
|
||||
job_queue.put_nowait(job)
|
||||
results: asyncio.Queue[list[dict[str, Any]]] = asyncio.Queue()
|
||||
|
||||
async def worker() -> None:
|
||||
holder = None
|
||||
try:
|
||||
holder = await open_proxy_holder()
|
||||
except Exception as e: # no session: jobs still run via one-shot fetches
|
||||
logger.warning("[youtube] proxy session open failed: %s", e)
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
job = job_queue.get_nowait()
|
||||
except asyncio.QueueEmpty:
|
||||
return
|
||||
items: list[dict[str, Any]] = []
|
||||
try:
|
||||
if holder is not None:
|
||||
async with bind_proxy_holder(holder):
|
||||
items = [item async for item in job]
|
||||
else:
|
||||
items = [item async for item in job]
|
||||
except Exception as e: # one bad video/URL must not kill the run
|
||||
logger.warning("[youtube] fan-out job failed: %s", e)
|
||||
await results.put(items)
|
||||
finally:
|
||||
if holder is not None:
|
||||
await holder.close()
|
||||
|
||||
tasks = [
|
||||
asyncio.create_task(worker()) for _ in range(min(concurrency, len(jobs)))
|
||||
]
|
||||
try:
|
||||
for _ in range(len(jobs)):
|
||||
for item in await results.get():
|
||||
yield item
|
||||
finally:
|
||||
for task in tasks:
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
# Await cancellation so each worker's finally closes its session before
|
||||
# we return — no leaked keep-alive connections when the consumer stops
|
||||
# early (e.g. the collector hit its limit).
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
|
||||
async def _post(url: str, payload: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""POST to InnerTube, retrying with the public web key if keyless fails.
|
||||
|
||||
ponytail: retries with the key only when the keyless call returns nothing;
|
||||
could remember which one worked to avoid the extra request.
|
||||
"""
|
||||
data = await post_innertube(url, payload)
|
||||
if data is None:
|
||||
data = await post_innertube(url, payload, api_key=INNERTUBE_PUBLIC_API_KEY)
|
||||
return data
|
||||
|
||||
|
||||
async def _finalize(
|
||||
partial: dict[str, Any],
|
||||
*,
|
||||
input_model: YouTubeScrapeInput,
|
||||
source_input: str | None,
|
||||
from_url: str | None,
|
||||
order: int,
|
||||
content_type: str,
|
||||
) -> dict[str, Any]:
|
||||
item = VideoItem(**partial)
|
||||
item.type = content_type # type: ignore[assignment]
|
||||
item.input = source_input
|
||||
item.fromYTUrl = from_url
|
||||
item.order = order
|
||||
if input_model.downloadSubtitles and item.id:
|
||||
item.subtitles = await fetch_subtitles(
|
||||
item.id,
|
||||
language=input_model.subtitlesLanguage,
|
||||
fmt=input_model.subtitlesFormat,
|
||||
prefer_generated=input_model.preferAutoGeneratedSubtitles,
|
||||
)
|
||||
# translatedTitle/Text: one extra /next in the requested language. ponytail:
|
||||
# gated on a non-English subtitlesLanguage so default runs pay nothing; costs
|
||||
# one request per item when a translation language is set.
|
||||
lang = input_model.subtitlesLanguage
|
||||
if item.id and lang and lang != "en":
|
||||
data = await _post(
|
||||
INNERTUBE_NEXT_URL, build_innertube_payload(video_id=item.id, hl=lang)
|
||||
)
|
||||
if data:
|
||||
item.translatedTitle, item.translatedText = parse_translation(data)
|
||||
return item.to_output()
|
||||
|
||||
|
||||
async def _video_flow(
|
||||
video_id: str,
|
||||
*,
|
||||
input_model: YouTubeScrapeInput,
|
||||
source_input: str | None,
|
||||
from_url: str | None,
|
||||
order: int,
|
||||
content_type: str,
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
url = f"https://www.youtube.com/watch?v={video_id}"
|
||||
html = await fetch_html(url)
|
||||
if not html:
|
||||
return
|
||||
partial = parse_video_page(html)
|
||||
if not partial:
|
||||
return
|
||||
yield await _finalize(
|
||||
partial,
|
||||
input_model=input_model,
|
||||
source_input=source_input,
|
||||
from_url=from_url,
|
||||
order=order,
|
||||
content_type=content_type,
|
||||
)
|
||||
|
||||
|
||||
async def _search_flow(
|
||||
query: str,
|
||||
*,
|
||||
input_model: YouTubeScrapeInput,
|
||||
source_input: str,
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
limit = input_model.maxResults
|
||||
if limit <= 0:
|
||||
return
|
||||
from_url = f"https://www.youtube.com/results?search_query={quote(query)}"
|
||||
params = build_search_params(input_model)
|
||||
payload = build_innertube_payload(search_query=query, search_params=params)
|
||||
data = await _post(INNERTUBE_SEARCH_URL, payload)
|
||||
|
||||
order = 0
|
||||
while data:
|
||||
items, token = parse_search_response(data)
|
||||
for it in items:
|
||||
if order >= limit:
|
||||
return
|
||||
yield await _finalize(
|
||||
it,
|
||||
input_model=input_model,
|
||||
source_input=source_input,
|
||||
from_url=from_url,
|
||||
order=order,
|
||||
content_type="video",
|
||||
)
|
||||
order += 1
|
||||
if not token or order >= limit:
|
||||
return
|
||||
data = await _post(
|
||||
INNERTUBE_SEARCH_URL, build_innertube_payload(continuation_token=token)
|
||||
)
|
||||
|
||||
|
||||
def _channel_tab_url(handle: str, tab: str) -> str:
|
||||
if handle.startswith("UC") and len(handle) > 10:
|
||||
return f"https://www.youtube.com/channel/{handle}/{tab}"
|
||||
return f"https://www.youtube.com/@{handle}/{tab}"
|
||||
|
||||
|
||||
# tab path -> (content type, list parser). Streams share the video lockup shape.
|
||||
_CHANNEL_TABS = {
|
||||
"videos": ("video", parse_channel_videos),
|
||||
"shorts": ("shorts", parse_channel_shorts),
|
||||
"streams": ("stream", parse_channel_videos),
|
||||
}
|
||||
|
||||
|
||||
async def _fetch_channel_about(initial: dict) -> dict[str, Any]:
|
||||
"""One ``/browse`` call for the About panel (deep channel fields).
|
||||
|
||||
Panels are unlabeled, so try each engagement token and keep the first that
|
||||
returns an ``aboutChannelViewModel``. ponytail: worst case is one extra
|
||||
no-op browse before the hit; a labeled-panel signal would remove it.
|
||||
"""
|
||||
for token in channel_about_tokens(initial):
|
||||
data = await _post(
|
||||
INNERTUBE_BROWSE_URL, build_innertube_payload(continuation_token=token)
|
||||
)
|
||||
about = find_first(data, "aboutChannelViewModel") if data else None
|
||||
if about:
|
||||
return parse_channel_about(about)
|
||||
return {}
|
||||
|
||||
|
||||
def _published_date(text: str | None):
|
||||
"""Parse a relative/absolute time string to a ``date`` (best-effort).
|
||||
|
||||
ponytail: channel list pages only expose coarse relative times ("2 years
|
||||
ago"), so the ``oldestPostDate`` cutoff is day-accurate at best.
|
||||
"""
|
||||
if not text:
|
||||
return None
|
||||
import dateparser
|
||||
|
||||
dt = dateparser.parse(text)
|
||||
return dt.date() if dt else None
|
||||
|
||||
|
||||
async def _channel_tab_flow(
|
||||
handle: str,
|
||||
tab: str,
|
||||
*,
|
||||
limit: int,
|
||||
input_model: YouTubeScrapeInput,
|
||||
source_input: str,
|
||||
channel_meta: dict[str, Any],
|
||||
initial: dict | None = None,
|
||||
cutoff=None,
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
"""Page one channel tab (videos/shorts/streams) up to ``limit`` items.
|
||||
|
||||
Videos honor ``sortVideosBy`` via the sort chips (re-fetch sorted from the
|
||||
start); shorts/streams page straight from the seed's first page + its
|
||||
continuation, since those tabs don't expose the same sort chips. ``initial``
|
||||
may be prefetched (videos tab) to avoid re-downloading the seed.
|
||||
"""
|
||||
if limit <= 0:
|
||||
return
|
||||
content_type, parse_fn = _CHANNEL_TABS[tab]
|
||||
from_url = _channel_tab_url(handle, tab)
|
||||
if initial is None:
|
||||
seed_html = await fetch_html(from_url)
|
||||
if not seed_html:
|
||||
return
|
||||
initial = extract_yt_initial_data(seed_html)
|
||||
if not initial:
|
||||
return
|
||||
|
||||
# Videos: prefer a sort chip token (fetches page 1 sorted). Otherwise parse
|
||||
# the seed's first page directly and follow its continuation.
|
||||
items: list[dict[str, Any]] = []
|
||||
token: str | None = None
|
||||
if tab == "videos":
|
||||
tokens = parse_channel_sort_tokens(initial)
|
||||
label = _SORT_LABELS.get(input_model.sortVideosBy or "NEWEST", "Latest")
|
||||
token = tokens.get(label) or next(iter(tokens.values()), None)
|
||||
if token is None:
|
||||
items, token = parse_fn(initial)
|
||||
|
||||
order = 0
|
||||
while order < limit:
|
||||
for it in items:
|
||||
if order >= limit:
|
||||
return
|
||||
# Newest-first ordering: once we pass the cutoff, the rest are older.
|
||||
if cutoff is not None:
|
||||
item_date = _published_date(it.get("publishedTimeText"))
|
||||
if item_date is not None and item_date < cutoff:
|
||||
return
|
||||
it.setdefault("channelUsername", handle)
|
||||
for key, value in channel_meta.items():
|
||||
it.setdefault(key, value)
|
||||
yield await _finalize(
|
||||
it,
|
||||
input_model=input_model,
|
||||
source_input=source_input,
|
||||
from_url=from_url,
|
||||
order=order,
|
||||
content_type=content_type,
|
||||
)
|
||||
order += 1
|
||||
if not token:
|
||||
return
|
||||
data = await _post(
|
||||
INNERTUBE_BROWSE_URL, build_innertube_payload(continuation_token=token)
|
||||
)
|
||||
if not data:
|
||||
return
|
||||
items, token = parse_fn(data)
|
||||
if not items:
|
||||
return
|
||||
|
||||
|
||||
async def _channel_flow(
|
||||
handle: str,
|
||||
*,
|
||||
input_model: YouTubeScrapeInput,
|
||||
source_input: str,
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
"""Scrape a channel's videos, shorts, and streams — each capped independently.
|
||||
|
||||
The videos seed is fetched once and reused to derive channel-wide metadata
|
||||
(identity, banner, and the About panel's deep fields) stamped on every item.
|
||||
"""
|
||||
videos_seed = await fetch_html(_channel_tab_url(handle, "videos"))
|
||||
initial = extract_yt_initial_data(videos_seed) if videos_seed else None
|
||||
channel_meta: dict[str, Any] = {}
|
||||
if initial:
|
||||
channel_meta = parse_channel_metadata(initial)
|
||||
channel_meta.update(await _fetch_channel_about(initial))
|
||||
|
||||
cutoff = _published_date(input_model.oldestPostDate)
|
||||
|
||||
for tab, limit in (
|
||||
("videos", input_model.maxResults),
|
||||
("shorts", input_model.maxResultsShorts),
|
||||
("streams", input_model.maxResultStreams),
|
||||
):
|
||||
async for item in _channel_tab_flow(
|
||||
handle,
|
||||
tab,
|
||||
limit=limit,
|
||||
input_model=input_model,
|
||||
source_input=source_input,
|
||||
channel_meta=channel_meta,
|
||||
initial=initial if tab == "videos" else None,
|
||||
cutoff=cutoff,
|
||||
):
|
||||
yield item
|
||||
|
||||
|
||||
async def _playlist_flow(
|
||||
playlist_id: str,
|
||||
*,
|
||||
input_model: YouTubeScrapeInput,
|
||||
source_input: str,
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
limit = input_model.maxResults
|
||||
if limit <= 0:
|
||||
return
|
||||
data = await _post(
|
||||
INNERTUBE_BROWSE_URL, build_innertube_payload(browse_id=f"VL{playlist_id}")
|
||||
)
|
||||
if not data:
|
||||
return
|
||||
# ponytail: reads the first browse page only (~100 items). Upgrade path:
|
||||
# follow the continuation token for playlists longer than one page.
|
||||
video_ids = parse_playlist_video_ids(data)
|
||||
|
||||
for order, vid in enumerate(video_ids):
|
||||
if order >= limit:
|
||||
return
|
||||
async for item in _video_flow(
|
||||
vid,
|
||||
input_model=input_model,
|
||||
source_input=source_input,
|
||||
from_url=source_input,
|
||||
order=order,
|
||||
content_type="video",
|
||||
):
|
||||
yield item
|
||||
|
||||
|
||||
async def _dispatch(
|
||||
resolved: ResolvedUrl, input_model: YouTubeScrapeInput
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
if resolved.kind == "video":
|
||||
content_type = "shorts" if "/shorts/" in resolved.url else "video"
|
||||
async for item in _video_flow(
|
||||
resolved.value,
|
||||
input_model=input_model,
|
||||
source_input=resolved.url,
|
||||
from_url=resolved.url,
|
||||
order=0,
|
||||
content_type=content_type,
|
||||
):
|
||||
yield item
|
||||
elif resolved.kind == "channel":
|
||||
async for item in _channel_flow(
|
||||
resolved.value, input_model=input_model, source_input=resolved.url
|
||||
):
|
||||
yield item
|
||||
elif resolved.kind == "playlist":
|
||||
async for item in _playlist_flow(
|
||||
resolved.value, input_model=input_model, source_input=resolved.url
|
||||
):
|
||||
yield item
|
||||
elif resolved.kind in ("search", "hashtag"):
|
||||
# ponytail: hashtag pages have their own layout; MVP routes the tag
|
||||
# through search (query "#tag"). Upgrade path: dedicated hashtag browse.
|
||||
query = resolved.value if resolved.kind == "search" else f"#{resolved.value}"
|
||||
async for item in _search_flow(
|
||||
query, input_model=input_model, source_input=resolved.url
|
||||
):
|
||||
yield item
|
||||
|
||||
|
||||
async def iter_youtube(
|
||||
input_model: YouTubeScrapeInput,
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
"""Yield Apify-shaped video items. startUrls override searchQueries.
|
||||
|
||||
Independent startUrls / queries fan out concurrently; each flow's own
|
||||
continuation paging stays sequential.
|
||||
"""
|
||||
if input_model.startUrls:
|
||||
jobs = []
|
||||
for entry in input_model.startUrls:
|
||||
resolved = resolve_url(entry.url)
|
||||
if not resolved:
|
||||
logger.warning("Unrecognized YouTube URL: %s", entry.url)
|
||||
continue
|
||||
jobs.append(_dispatch(resolved, input_model))
|
||||
async for item in fan_out(jobs):
|
||||
yield item
|
||||
return
|
||||
|
||||
jobs = [
|
||||
_search_flow(query, input_model=input_model, source_input=query)
|
||||
for query in input_model.searchQueries
|
||||
]
|
||||
async for item in fan_out(jobs):
|
||||
yield item
|
||||
|
||||
|
||||
async def scrape_youtube(
|
||||
input_model: YouTubeScrapeInput, *, limit: int | None = None
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Collect :func:`iter_youtube` into a list, honoring an optional ``limit``.
|
||||
|
||||
``limit`` is a request-time policy guard (used by the route), NOT a ceiling
|
||||
in the streaming core.
|
||||
"""
|
||||
results: list[dict[str, Any]] = []
|
||||
async for item in iter_youtube(input_model):
|
||||
results.append(item)
|
||||
if limit is not None and len(results) >= limit:
|
||||
break
|
||||
return results
|
||||
92
surfsense_backend/app/scrapers/youtube/search_filters.py
Normal file
92
surfsense_backend/app/scrapers/youtube/search_filters.py
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
"""Encode Apify search filters into YouTube's ``sp=`` (InnerTube ``params``).
|
||||
|
||||
YouTube encodes search filters as a base64 protobuf. The message is::
|
||||
|
||||
SearchParams {
|
||||
1: sortOrder (varint enum)
|
||||
2: Filters {
|
||||
1: uploadDate (varint enum)
|
||||
2: type (varint enum)
|
||||
3: duration (varint enum)
|
||||
<feature fields>: bool (varint 1)
|
||||
}
|
||||
}
|
||||
|
||||
Rather than a lookup table of single tokens, we build the protobuf directly so
|
||||
arbitrary combinations (sort + date + type + length + any boolean features)
|
||||
compose correctly. The field numbers/enums below are verified to reproduce
|
||||
YouTube's own standalone tokens byte-for-byte (see the unit test).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
|
||||
_SORT_ORDER = {"relevance": 0, "rating": 1, "date": 2, "views": 3}
|
||||
_UPLOAD_DATE = {"hour": 1, "today": 2, "week": 3, "month": 4, "year": 5}
|
||||
_TYPE = {"video": 1, "channel": 2, "playlist": 3, "movie": 4}
|
||||
_DURATION = {"under4": 1, "plus20": 2, "between420": 3}
|
||||
|
||||
# Apify boolean flag -> Filters feature field number.
|
||||
_FEATURES = {
|
||||
"isHD": 4,
|
||||
"hasSubtitles": 5,
|
||||
"hasCC": 6,
|
||||
"is3D": 7,
|
||||
"isLive": 8,
|
||||
"isBought": 9,
|
||||
"is4K": 14,
|
||||
"is360": 15,
|
||||
"hasLocation": 23,
|
||||
"isHDR": 25,
|
||||
"isVR180": 26,
|
||||
}
|
||||
|
||||
|
||||
def _varint(value: int) -> bytes:
|
||||
out = bytearray()
|
||||
while True:
|
||||
byte = value & 0x7F
|
||||
value >>= 7
|
||||
out.append(byte | (0x80 if value else 0))
|
||||
if not value:
|
||||
return bytes(out)
|
||||
|
||||
|
||||
def _tag_varint(field: int, value: int) -> bytes:
|
||||
"""A protobuf ``field: varint`` entry (wire type 0)."""
|
||||
return _varint(field << 3) + _varint(value)
|
||||
|
||||
|
||||
def _tag_message(field: int, data: bytes) -> bytes:
|
||||
"""A protobuf ``field: message`` entry (wire type 2, length-delimited)."""
|
||||
return _varint((field << 3) | 2) + _varint(len(data)) + data
|
||||
|
||||
|
||||
def build_search_params(input_model) -> str | None:
|
||||
"""Encode the input's filters into a base64 ``sp=`` token, or ``None``.
|
||||
|
||||
All set filters compose into a single protobuf; ``None`` when nothing is set
|
||||
(so the caller omits ``params`` entirely).
|
||||
"""
|
||||
filters = bytearray()
|
||||
if input_model.dateFilter in _UPLOAD_DATE:
|
||||
filters += _tag_varint(1, _UPLOAD_DATE[input_model.dateFilter])
|
||||
if input_model.videoType in _TYPE:
|
||||
filters += _tag_varint(2, _TYPE[input_model.videoType])
|
||||
if input_model.lengthFilter in _DURATION:
|
||||
filters += _tag_varint(3, _DURATION[input_model.lengthFilter])
|
||||
for flag, field in _FEATURES.items():
|
||||
if getattr(input_model, flag, None):
|
||||
filters += _tag_varint(field, 1)
|
||||
|
||||
message = bytearray()
|
||||
sort = _SORT_ORDER.get(input_model.sortingOrder or "")
|
||||
if sort: # relevance (0) is the default, so it needs no bytes
|
||||
message += _tag_varint(1, sort)
|
||||
if filters:
|
||||
message += _tag_message(2, bytes(filters))
|
||||
|
||||
if not message:
|
||||
return None
|
||||
return base64.b64encode(bytes(message)).decode("ascii")
|
||||
98
surfsense_backend/app/scrapers/youtube/subtitles.py
Normal file
98
surfsense_backend/app/scrapers/youtube/subtitles.py
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
"""Subtitle download via ``youtube-transcript-api``, shaped to Apify ``subtitles[]``.
|
||||
|
||||
Uses the library's built-in formatters (no hand-rolled srt/vtt conversion). The
|
||||
transcript API has no XML formatter, so ``xml`` falls back to the raw snippet
|
||||
data. Fetch/proxy wiring mirrors ``youtube_processor.py``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from fake_useragent import UserAgent
|
||||
from requests import Session
|
||||
from youtube_transcript_api import YouTubeTranscriptApi
|
||||
from youtube_transcript_api.formatters import (
|
||||
SRTFormatter,
|
||||
TextFormatter,
|
||||
WebVTTFormatter,
|
||||
)
|
||||
|
||||
from app.utils.proxy import get_requests_proxies
|
||||
|
||||
from .schemas import SubtitleTrack
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_FORMATTERS = {
|
||||
"srt": SRTFormatter,
|
||||
"vtt": WebVTTFormatter,
|
||||
"plaintext": TextFormatter,
|
||||
}
|
||||
|
||||
|
||||
def _build_client() -> YouTubeTranscriptApi:
|
||||
http_client = Session()
|
||||
http_client.headers.update({"User-Agent": UserAgent().random})
|
||||
proxies = get_requests_proxies()
|
||||
if proxies:
|
||||
http_client.proxies.update(proxies)
|
||||
return YouTubeTranscriptApi(http_client=http_client)
|
||||
|
||||
|
||||
def _select_transcript(transcript_list, language: str, prefer_generated: bool):
|
||||
"""Pick a transcript honoring language + generated/manual preference."""
|
||||
# ``any`` means take whatever the video offers first.
|
||||
if language == "any":
|
||||
return next(iter(transcript_list))
|
||||
|
||||
codes = [language]
|
||||
if prefer_generated:
|
||||
try:
|
||||
return transcript_list.find_generated_transcript(codes)
|
||||
except Exception:
|
||||
return transcript_list.find_transcript(codes)
|
||||
return transcript_list.find_transcript(codes)
|
||||
|
||||
|
||||
def _fetch_subtitles_sync(video_id: str, language: str, fmt: str, prefer_generated: bool):
|
||||
api = _build_client()
|
||||
transcript_list = api.list(video_id)
|
||||
transcript = _select_transcript(transcript_list, language, prefer_generated)
|
||||
fetched = transcript.fetch()
|
||||
|
||||
if fmt == "xml":
|
||||
# No XML formatter in the library; emit the raw snippet data as text.
|
||||
body = str(fetched.to_raw_data())
|
||||
else:
|
||||
formatter_cls = _FORMATTERS.get(fmt, SRTFormatter)
|
||||
body = formatter_cls().format_transcript(fetched)
|
||||
|
||||
return SubtitleTrack(
|
||||
srtUrl=None,
|
||||
type="auto_generated" if transcript.is_generated else "user_generated",
|
||||
language=transcript.language_code,
|
||||
srt=body,
|
||||
)
|
||||
|
||||
|
||||
async def fetch_subtitles(
|
||||
video_id: str,
|
||||
*,
|
||||
language: str = "en",
|
||||
fmt: str = "srt",
|
||||
prefer_generated: bool = False,
|
||||
) -> list[SubtitleTrack] | None:
|
||||
"""Return the Apify ``subtitles[]`` list for a video, or ``None`` if none.
|
||||
|
||||
Runs the blocking transcript API in a worker thread to stay async-friendly.
|
||||
"""
|
||||
try:
|
||||
track = await asyncio.to_thread(
|
||||
_fetch_subtitles_sync, video_id, language, fmt, prefer_generated
|
||||
)
|
||||
return [track]
|
||||
except Exception as e:
|
||||
logger.info("No subtitles for video %s (%s): %s", video_id, language, e)
|
||||
return None
|
||||
85
surfsense_backend/app/scrapers/youtube/url_resolver.py
Normal file
85
surfsense_backend/app/scrapers/youtube/url_resolver.py
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
"""Classify a YouTube URL and extract its identifier.
|
||||
|
||||
Covers the ``startUrls`` shapes the Apify spec accepts: video, channel,
|
||||
playlist, hashtag, and search-results pages. Video-ID extraction reuses the
|
||||
logic already in ``app/tasks/document_processors/youtube_processor.py``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from typing import Literal
|
||||
from urllib.parse import parse_qs, unquote, urlparse
|
||||
|
||||
ResolvedKind = Literal["video", "channel", "playlist", "hashtag", "search"]
|
||||
|
||||
_PLAYLIST_ID_RE = re.compile(r"[?&]list=([\w-]+)")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResolvedUrl:
|
||||
kind: ResolvedKind
|
||||
value: str # video id, channel handle/id, playlist id, hashtag, or query
|
||||
url: str
|
||||
|
||||
|
||||
def get_youtube_video_id(url: str) -> str | None:
|
||||
"""Extract a video ID from watch/youtu.be/embed/shorts URL formats."""
|
||||
parsed = urlparse(url)
|
||||
hostname = parsed.hostname or ""
|
||||
if hostname == "youtu.be":
|
||||
return parsed.path[1:] or None
|
||||
if hostname in ("www.youtube.com", "youtube.com", "m.youtube.com"):
|
||||
if parsed.path == "/watch":
|
||||
return parse_qs(parsed.query).get("v", [None])[0]
|
||||
for prefix in ("/embed/", "/v/", "/shorts/"):
|
||||
if parsed.path.startswith(prefix):
|
||||
return parsed.path[len(prefix) :].split("/")[0] or None
|
||||
return None
|
||||
|
||||
|
||||
def resolve_url(url: str) -> ResolvedUrl | None:
|
||||
"""Classify a YouTube URL into a scrape job, or ``None`` if unrecognized."""
|
||||
parsed = urlparse(url)
|
||||
path = parsed.path or ""
|
||||
|
||||
# Shorts are videos with their own path.
|
||||
if "/shorts/" in path:
|
||||
vid = path.split("/shorts/")[1].split("/")[0]
|
||||
return ResolvedUrl("video", vid, url) if vid else None
|
||||
|
||||
video_id = get_youtube_video_id(url)
|
||||
if video_id:
|
||||
return ResolvedUrl("video", video_id, url)
|
||||
|
||||
# Playlist (either a /playlist page or any URL carrying ?list=).
|
||||
playlist_match = _PLAYLIST_ID_RE.search(url)
|
||||
if path.startswith("/playlist") and playlist_match:
|
||||
return ResolvedUrl("playlist", playlist_match.group(1), url)
|
||||
|
||||
# Search results page.
|
||||
if path == "/results":
|
||||
query = parse_qs(parsed.query).get("search_query", [None])[0]
|
||||
if query:
|
||||
return ResolvedUrl("search", unquote(query), url)
|
||||
return None
|
||||
|
||||
# Hashtag page (/hashtag/<tag>).
|
||||
if path.startswith("/hashtag/"):
|
||||
tag = path[len("/hashtag/") :].split("/")[0]
|
||||
return ResolvedUrl("hashtag", unquote(tag), url) if tag else None
|
||||
|
||||
# Channel: /@handle, /channel/UC..., /c/Name, /user/Name.
|
||||
if path.startswith("/@"):
|
||||
return ResolvedUrl("channel", path[2:].split("/")[0], url)
|
||||
for prefix in ("/channel/", "/c/", "/user/"):
|
||||
if path.startswith(prefix):
|
||||
handle = path[len(prefix) :].split("/")[0]
|
||||
return ResolvedUrl("channel", handle, url) if handle else None
|
||||
|
||||
# A trailing ?list= without an explicit /playlist path.
|
||||
if playlist_match:
|
||||
return ResolvedUrl("playlist", playlist_match.group(1), url)
|
||||
|
||||
return None
|
||||
Loading…
Add table
Add a link
Reference in a new issue