apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-14 22:52:15 +02:00
Code Issues Projects Releases Packages Wiki Activity

refactor(native-connector): relocate youtube scraper under app/proprietary + parallelize playlists

Browse source 
Move app/scrapers -> app/proprietary/scrapers/youtube to sit alongside the existing proprietary web_crawler/platforms namespace, updating all external imports (routes, tests, e2e script, README). Internal imports were relative so are unchanged.

Also parallelize playlist per-video resolution: page video ids sequentially, then resolve the heavy watch-page fetches concurrently via fan_out (~150 videos ~70s, down from a few minutes). Items stream in completion order; sort by the order field for playlist order.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
DESKTOP-RTLN3BA\$punk 2026-07-02 02:34:04 -07:00
parent 99378d7b63
commit 0445c646c6
16 changed files with 412 additions and 320 deletions
Download patch file Download diff file Expand all files Collapse all files

4
surfsense_backend/tests/unit/scrapers/youtube/test_fetch_resilience.py
View file

@ -11,8 +11,8 @@ from collections.abc import AsyncIterator
import pytest
from app.scrapers.youtube import innertube, scraper
from app.scrapers.youtube.innertube import (
from app.proprietary.scrapers.youtube import innertube, scraper
from app.proprietary.scrapers.youtube.innertube import (
INNERTUBE_SEARCH_URL,
_current_session,
fetch_html,

15
surfsense_backend/tests/unit/scrapers/youtube/test_parsers.py
View file

@ -12,7 +12,7 @@ from pathlib import Path
import pytest
from app.scrapers.youtube.parsers import (
from app.proprietary.scrapers.youtube.parsers import (
channel_about_tokens,
comment_next_token,
comment_reply_tokens,
@ -35,9 +35,9 @@ from app.scrapers.youtube.parsers import (
parse_video_page,
seconds_to_duration,
)
from app.scrapers.youtube.schemas import YouTubeScrapeInput
from app.scrapers.youtube.search_filters import build_search_params
from app.scrapers.youtube.url_resolver import resolve_url
from app.proprietary.scrapers.youtube.schemas import YouTubeScrapeInput
from app.proprietary.scrapers.youtube.search_filters import build_search_params
from app.proprietary.scrapers.youtube.url_resolver import resolve_url
pytestmark = pytest.mark.unit
@ -429,8 +429,13 @@ def test_parse_playlist_video_ids():
},
# a playlist self-lockup (non-video, longer id) that must be ignored
"sidebar": {"lockupViewModel": {"contentId": "PLZHQObOWTQDPD3MizzM2xVFitgF8hE_ab"}},
"continuationItemRenderer": {
"continuationEndpoint": {"continuationCommand": {"token": "PAGE2"}}
},
}
assert parse_playlist_video_ids(data) == ["fNk_zzaMoSs", "k7RM-ot2NWY"]
ids, token = parse_playlist_video_ids(data)
assert ids == ["fNk_zzaMoSs", "k7RM-ot2NWY"]
assert token == "PAGE2"
# --- search filter (sp=) encoder --------------------------------------------