apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-18 23:11:12 +02:00
Code Issues Projects Releases Packages Wiki Activity

refactor: streamline TikTok and Instagram scraping logic by removing search_queries and enhancing documentation for clarity

Browse source
This commit is contained in:
DESKTOP-RTLN3BA\$punk 2026-07-13 17:11:25 -07:00
parent e8b3692b54
commit 2b018c4474
111 changed files with 1800 additions and 1580 deletions
Download patch file Download diff file Expand all files Collapse all files

4
surfsense_backend/tests/unit/capabilities/instagram/test_schemas.py
View file

@ -66,6 +66,4 @@ def test_details_wraps_profile_items():
def test_details_rejects_both_sources():
with pytest.raises(ValidationError):
DetailsInput(
urls=["https://www.instagram.com/natgeo/"], search_queries=["x"]
)
DetailsInput(urls=["https://www.instagram.com/natgeo/"], search_queries=["x"])

2
surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_executor.py
View file

@ -54,7 +54,6 @@ async def test_forwards_typed_sources_and_limit():
ScrapeInput(
profiles=["nasa"],
hashtags=["food"],
search_queries=["cats"],
results_per_page=7,
max_items=25,
)
@ -63,7 +62,6 @@ async def test_forwards_typed_sources_and_limit():
(actor_input, limit) = scraper.calls[0]
assert actor_input.profiles == ["nasa"]
assert actor_input.hashtags == ["food"]
assert actor_input.searchQueries == ["cats"]
assert actor_input.resultsPerPage == 7
# The outer collection limit is the caller's total-item cap.
assert limit == 25

3
surfsense_backend/tests/unit/platforms/instagram/test_budget.py
View file

@ -42,8 +42,7 @@ def _profile_payload(n: int) -> dict:
"edge_owner_to_timeline_media": {
"count": n,
"edges": [
{"node": {"id": str(i), "shortcode": f"S{i}"}}
for i in range(n)
{"node": {"id": str(i), "shortcode": f"S{i}"}} for i in range(n)
],
},
}

8
surfsense_backend/tests/unit/platforms/instagram/test_discovery.py
View file

@ -33,9 +33,7 @@ async def test_google_discovery_keeps_only_profiles(monkeypatch):
"https://example.com/not-instagram",
),
)
targets = await scraper._discover(
"nat geo photos", search_type="profile", limit=10
)
targets = await scraper._discover("nat geo photos", search_type="profile", limit=10)
assert [(t.kind, t.value) for t in targets] == [("profile", "natgeo")]
@ -48,9 +46,7 @@ async def test_google_discovery_dedupes(monkeypatch):
"https://www.instagram.com/natgeo/",
),
)
targets = await scraper._discover(
"nat geo photos", search_type="profile", limit=10
)
targets = await scraper._discover("nat geo photos", search_type="profile", limit=10)
assert len(targets) == 1

4
surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py
View file

@ -102,7 +102,9 @@ async def test_warms_then_returns_json():
holder = _FakeHolder([_FakeSession(200, csrftoken=True)])
token = _current_session.set(holder)
try:
result = await fetch_json("api/v1/users/web_profile_info/", {"username": "natgeo"})
result = await fetch_json(
"api/v1/users/web_profile_info/", {"username": "natgeo"}
)
finally:
_current_session.reset(token)
assert result == _PAYLOAD

10
surfsense_backend/tests/unit/platforms/instagram/test_parsers.py
View file

@ -175,8 +175,14 @@ def test_parse_post_prefers_relay_json():
"image_versions2": {"candidates": [{"url": "https://cdn/c2.jpg"}]},
},
],
"usertags": {"in": [{"position": [0.5, 0.5], "user": {"username": "tagged1", "id": "77"}}]},
"coauthor_producers": [{"username": "coauthor1", "id": "88", "is_verified": True}],
"usertags": {
"in": [
{"position": [0.5, 0.5], "user": {"username": "tagged1", "id": "77"}}
]
},
"coauthor_producers": [
{"username": "coauthor1", "id": "88", "is_verified": True}
],
"location": {"id": "123", "name": "Bali"},
}
html = (

149
surfsense_backend/tests/unit/platforms/tiktok/test_discovery.py
View file

@ -1,149 +0,0 @@
"""Offline tests for Google-backed TikTok video discovery.
``searchQueries`` are login-walled on TikTok's native search, so they route
through the ``google_search`` platform (``site:tiktok.com``): each organic URL
is classified with ``resolve_target`` and only video hits (``/@user/video/<id>``)
are kept — profiles/hashtags/search/photo/non-tiktok are dropped (accounts
belong to the user-search verb). These tests inject a fake ``scrape_serps`` so
there is no network: they pin the classification, cross-query de-dup, the limit
cap, the barren-query ErrorItem, and that no ``/search?q=`` listing target is
ever built.
"""
from __future__ import annotations
import json
from app.proprietary.platforms.tiktok import (
TikTokScrapeInput,
orchestrator,
scrape_tiktok,
)
def _fake_serps(*organic_urls: str):
async def _scrape_serps(input_model, *, limit=None):
assert input_model.site == "tiktok.com"
assert input_model.maxPagesPerQuery == 1
return [{"organicResults": [{"url": u} for u in organic_urls]}]
return _scrape_serps
def _video_page(url: str) -> str:
"""Render a rehydration blob for a ``/@user/video/<id>`` URL."""
video_id = url.rsplit("/", 1)[1]
username = url.split("@")[1].split("/")[0]
blob = {
"__DEFAULT_SCOPE__": {
"webapp.video-detail": {
"itemInfo": {
"itemStruct": {
"id": video_id,
"desc": "hi",
"author": {"uniqueId": username},
"stats": {"diggCount": 1},
}
}
}
}
}
return (
'<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" '
f'type="application/json">{json.dumps(blob)}</script>'
)
async def _fetch_video(url: str) -> str:
return _video_page(url)
async def test_search_discovery_keeps_only_videos(monkeypatch):
# Only the video URL survives; profile / hashtag / search / photo /
# non-tiktok organic results are dropped.
monkeypatch.setattr(
orchestrator,
"scrape_serps",
_fake_serps(
"https://www.tiktok.com/@nasa/video/123",
"https://www.tiktok.com/@nasa",
"https://www.tiktok.com/tag/space",
"https://www.tiktok.com/search?q=space",
"https://www.tiktok.com/@nasa/photo/999",
"https://example.com/not-tiktok",
),
)
items = await scrape_tiktok(
TikTokScrapeInput(searchQueries=["space"], resultsPerPage=10),
fetch=_fetch_video,
)
assert [i["id"] for i in items] == ["123"]
async def test_search_discovery_dedupes_across_queries(monkeypatch):
# The same video surfacing under two queries is scraped once.
monkeypatch.setattr(
orchestrator,
"scrape_serps",
_fake_serps("https://www.tiktok.com/@nasa/video/123"),
)
items = await scrape_tiktok(
TikTokScrapeInput(searchQueries=["space", "rockets"], resultsPerPage=10),
fetch=_fetch_video,
)
assert [i["id"] for i in items] == ["123"]
async def test_search_discovery_respects_per_target_limit(monkeypatch):
monkeypatch.setattr(
orchestrator,
"scrape_serps",
_fake_serps(
"https://www.tiktok.com/@a/video/1",
"https://www.tiktok.com/@b/video/2",
"https://www.tiktok.com/@c/video/3",
),
)
items = await scrape_tiktok(
TikTokScrapeInput(searchQueries=["x"], resultsPerPage=2),
fetch=_fetch_video,
)
assert [i["id"] for i in items] == ["1", "2"]
async def test_search_barren_query_emits_error_item(monkeypatch):
# A query whose discovery finds no video URLs degrades to one ErrorItem.
monkeypatch.setattr(
orchestrator,
"scrape_serps",
_fake_serps(
"https://www.tiktok.com/@nasa",
"https://example.com/x",
),
)
items = await scrape_tiktok(
TikTokScrapeInput(searchQueries=["space"], resultsPerPage=10),
fetch=_fetch_video,
)
assert len(items) == 1
assert items[0]["errorCode"] == "no_items"
assert items[0]["input"] == "space"
async def test_search_never_builds_listing_target(monkeypatch):
# searchQueries must never hit the (login-walled) native search listing flow.
monkeypatch.setattr(
orchestrator,
"scrape_serps",
_fake_serps("https://www.tiktok.com/@nasa/video/123"),
)
async def _boom_listing(_url: str, _count: int) -> list[dict]:
raise AssertionError("searchQueries must not build a listing target")
items = await scrape_tiktok(
TikTokScrapeInput(searchQueries=["space"], resultsPerPage=10),
fetch=_fetch_video,
fetch_listing=_boom_listing,
)
assert [i["id"] for i in items] == ["123"]

8
surfsense_backend/tests/unit/platforms/tiktok/test_fetch_resilience.py
View file

@ -84,7 +84,9 @@ async def test_warms_then_returns_html():
async def test_rotates_when_warm_fails_then_succeeds():
holder = _FakeHolder([_FakeSession(200, warms=False), _FakeSession(200, warms=True)])
holder = _FakeHolder(
[_FakeSession(200, warms=False), _FakeSession(200, warms=True)]
)
token = _current_session.set(holder)
try:
result = await client.fetch_html("https://www.tiktok.com/@scout2015")
@ -119,9 +121,7 @@ async def test_rotates_and_rewarms_on_403():
async def test_persistent_403_raises_blocked(monkeypatch):
_no_sleep(monkeypatch)
holder = _FakeHolder(
[_FakeSession(403) for _ in range(client._MAX_ROTATIONS + 1)]
)
holder = _FakeHolder([_FakeSession(403) for _ in range(client._MAX_ROTATIONS + 1)])
token = _current_session.set(holder)
try:
raised = False

4
surfsense_backend/tests/unit/platforms/tiktok/test_target_resolver.py
View file

@ -6,7 +6,9 @@ from app.proprietary.platforms.tiktok.targets import resolve_target
def test_resolve_video_carries_username_and_id():
target = resolve_target("https://www.tiktok.com/@scout2015/video/6718335390845095173")
target = resolve_target(
"https://www.tiktok.com/@scout2015/video/6718335390845095173"
)
assert target is not None
assert target.kind == "video"
assert target.value == "6718335390845095173"

8
surfsense_backend/tests/unit/platforms/tiktok/test_user_search.py
View file

@ -28,9 +28,7 @@ async def test_user_search_parses_dedupes_and_caps():
async def fake_fetch(_url: str, _cap: int) -> list[dict]:
return [_user("1", "nasa"), _user("1", "nasa"), _user("2", "nasa2")]
items = await search_tiktok_users(
["nasa"], per_query=2, fetch_users=fake_fetch
)
items = await search_tiktok_users(["nasa"], per_query=2, fetch_users=fake_fetch)
assert [i["id"] for i in items] == ["1", "2"]
first = items[0]
@ -49,9 +47,7 @@ async def test_user_search_empty_query_emits_error_item():
async def fake_fetch(_url: str, _cap: int) -> list[dict]:
return []
items = await search_tiktok_users(
["ghost"], per_query=5, fetch_users=fake_fetch
)
items = await search_tiktok_users(["ghost"], per_query=5, fetch_users=fake_fetch)
assert len(items) == 1
assert items[0]["errorCode"] == "no_users"

8
surfsense_backend/tests/unit/services/test_requesty_model_normalizer.py
View file

@ -58,9 +58,7 @@ def test_chat_model_requires_slash_tools_and_context():
def test_excluded_provider_slug_is_filtered():
assert not is_requesty_chat_model(
_requesty_model(model_id="amazon/nova-pro-v1")
)
assert not is_requesty_chat_model(_requesty_model(model_id="amazon/nova-pro-v1"))
def test_image_generation_models_excluded_from_chat_and_flagged():
@ -89,9 +87,7 @@ def test_normalize_maps_context_window_and_capabilities():
name="GPT-4o mini",
),
_requesty_model(model_id="openai/gpt-4o-mini", tools=False),
_requesty_model(
model_id="black-forest-labs/flux", image_generation=True
),
_requesty_model(model_id="black-forest-labs/flux", image_generation=True),
]
)