mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-26 23:51:14 +02:00
fix: reject spoofed non-YouTube URLs
This commit is contained in:
parent
4dddd55512
commit
6330d5a789
2 changed files with 29 additions and 8 deletions
|
|
@ -15,6 +15,9 @@ ResolvedKind = Literal["video", "channel", "playlist", "hashtag", "search"]
|
|||
|
||||
_PLAYLIST_ID_RE = re.compile(r"[?&]list=([\w-]+)")
|
||||
|
||||
_YOUTUBE_HOSTS = frozenset({"www.youtube.com", "youtube.com", "m.youtube.com"})
|
||||
"""Hosts whose page paths (/@, /channel, /playlist, ...) are trusted as YouTube."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResolvedUrl:
|
||||
|
|
@ -29,7 +32,7 @@ def get_youtube_video_id(url: str) -> str | None:
|
|||
hostname = parsed.hostname or ""
|
||||
if hostname == "youtu.be":
|
||||
return parsed.path[1:] or None
|
||||
if hostname in ("www.youtube.com", "youtube.com", "m.youtube.com"):
|
||||
if hostname in _YOUTUBE_HOSTS:
|
||||
if parsed.path == "/watch":
|
||||
return parse_qs(parsed.query).get("v", [None])[0]
|
||||
for prefix in ("/embed/", "/v/", "/shorts/"):
|
||||
|
|
@ -41,17 +44,21 @@ def get_youtube_video_id(url: str) -> str | None:
|
|||
def resolve_url(url: str) -> ResolvedUrl | None:
|
||||
"""Classify a YouTube URL into a scrape job, or ``None`` if unrecognized."""
|
||||
parsed = urlparse(url)
|
||||
hostname = (parsed.hostname or "").lower()
|
||||
path = parsed.path or ""
|
||||
|
||||
# Shorts are videos with their own path.
|
||||
if "/shorts/" in path:
|
||||
vid = path.split("/shorts/")[1].split("/")[0]
|
||||
return ResolvedUrl("video", vid, url) if vid else None
|
||||
|
||||
# Videos: watch / youtu.be / embed / shorts. get_youtube_video_id validates
|
||||
# the host itself, so this also covers youtu.be short links.
|
||||
video_id = get_youtube_video_id(url)
|
||||
if video_id:
|
||||
return ResolvedUrl("video", video_id, url)
|
||||
|
||||
# Every remaining shape is a youtube.com page keyed off its path. Require a
|
||||
# YouTube host so a well-formed non-YouTube URL (e.g. https://evil.com/@x)
|
||||
# is rejected instead of misclassified as a channel by its path alone.
|
||||
if hostname not in _YOUTUBE_HOSTS:
|
||||
return None
|
||||
|
||||
# Playlist (either a /playlist page or any URL carrying ?list=).
|
||||
playlist_match = _PLAYLIST_ID_RE.search(url)
|
||||
if path.startswith("/playlist") and playlist_match:
|
||||
|
|
|
|||
|
|
@ -946,8 +946,22 @@ def test_resolve_url(url, kind, value):
|
|||
assert resolved.value == value
|
||||
|
||||
|
||||
def test_resolve_url_unrecognized():
|
||||
assert resolve_url("https://example.com/foo") is None
|
||||
@pytest.mark.parametrize(
|
||||
"url",
|
||||
[
|
||||
"https://example.com/foo",
|
||||
# Well-formed non-YouTube URLs whose path mimics a YouTube page must be
|
||||
# rejected, not misclassified by path alone (host-spoof guard).
|
||||
"https://evil.com/@Apify",
|
||||
"https://evil.com/channel/UC123456789abc",
|
||||
"https://evil.com/shorts/abc123",
|
||||
"https://evil.com/playlist?list=PL123",
|
||||
"https://evil.com/hashtag/tech",
|
||||
"https://evil.com/results?search_query=web+scraping",
|
||||
],
|
||||
)
|
||||
def test_resolve_url_unrecognized(url):
|
||||
assert resolve_url(url) is None
|
||||
|
||||
|
||||
# --- optional: exercise captured real fixtures if present --------------------
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue