mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-16 23:01:06 +02:00
refactor: streamline TikTok and Instagram scraping logic by removing search_queries and enhancing documentation for clarity
This commit is contained in:
DESKTOP-RTLN3BA\$punk
parent
e8b3692b54
commit
2b018c4474
111 changed files with 1800 additions and 1580 deletions
|
|
@ -14,11 +14,11 @@ Answer the delegated question from live TikTok data gathered with your verb, com
|
|||
</available_tools>
|
||||
|
||||
<playbook>
|
||||
- Finding videos on a topic: prefer `tiktok_scrape` with `hashtags` (no leading '#') or a direct TikTok URL in `urls` (fastest). `search_queries` also finds videos on a topic, but it is Google-backed and slow, so start with **at most 3** distinct queries and only add more if the first round returns nothing significant — never batch many phrasing variants of the same intent.
|
||||
- Finding videos on a topic: call `tiktok_scrape` with `hashtags` (no leading '#'), or pass a TikTok URL in `urls`. There is no keyword-video search — use hashtags or a video URL.
|
||||
- Scraping a specific video, profile, hashtag, or search page: pass its TikTok URL in `urls`.
|
||||
- Profiles: a creator's `profiles` feed returns the account's metadata (name, followers, bio, verification) reliably, but its video list is often withheld by TikTok — treat an empty video list as a known limit, not a failure to retry endlessly. Prefer `hashtags` or a direct video URL for videos.
|
||||
- Comments on a video: call `tiktok_comments` with the video URL(s) in `video_urls`.
|
||||
- Finding accounts by keyword: call `tiktok_user_search` with `queries` — that is the path for accounts. Use `search_queries` on `tiktok_scrape` only when you want videos, not accounts.
|
||||
- Finding accounts by keyword: call `tiktok_user_search` with `queries` — that is the path for accounts.
|
||||
- "What's trending now": call `tiktok_trending` (no query needed); set `max_items` for how many.
|
||||
- Controlling volume: use `max_items` for the total cap and `results_per_page` per target (per-verb equivalents: `comments_per_video`, `results_per_query`).
|
||||
- Requested counts: `max_items` defaults low — when the task asks for N items, set `max_items` (and the per-target count) above N. A call that caps below the target can never satisfy it.
|
||||
|
|
|
|||
|
|
@ -54,9 +54,7 @@ class DetailsInput(BaseModel):
|
|||
@model_validator(mode="after")
|
||||
def _exactly_one_source(self) -> DetailsInput:
|
||||
if not self.urls and not self.search_queries:
|
||||
raise ValueError(
|
||||
"Provide at least one of 'urls' or 'search_queries'."
|
||||
)
|
||||
raise ValueError("Provide at least one of 'urls' or 'search_queries'.")
|
||||
if self.urls and self.search_queries:
|
||||
raise ValueError(
|
||||
"Provide 'urls' OR 'search_queries', not both (they cannot be combined)."
|
||||
|
|
|
|||
|
|
@ -77,9 +77,7 @@ class ScrapeInput(BaseModel):
|
|||
@model_validator(mode="after")
|
||||
def _exactly_one_source(self) -> ScrapeInput:
|
||||
if not self.urls and not self.search_queries:
|
||||
raise ValueError(
|
||||
"Provide at least one of 'urls' or 'search_queries'."
|
||||
)
|
||||
raise ValueError("Provide at least one of 'urls' or 'search_queries'.")
|
||||
if self.urls and self.search_queries:
|
||||
raise ValueError(
|
||||
"Provide 'urls' OR 'search_queries', not both (they cannot be combined)."
|
||||
|
|
|
|||
|
|
@ -10,9 +10,8 @@ from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput
|
|||
TIKTOK_SCRAPE = Capability(
|
||||
name="tiktok.scrape",
|
||||
description=(
|
||||
"Scrape public TikTok videos. Use urls, profiles, hashtags, or "
|
||||
"search_queries (search_queries are resolved via Google to public "
|
||||
"videos; for accounts by keyword use tiktok.user_search)."
|
||||
"Scrape public TikTok videos. Use urls, profiles, or hashtags. To find "
|
||||
"accounts by keyword, use tiktok.user_search."
|
||||
),
|
||||
input_schema=ScrapeInput,
|
||||
output_schema=ScrapeOutput,
|
||||
|
|
|
|||
|
|
@ -26,7 +26,6 @@ def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor:
|
|||
startUrls=[{"url": url} for url in payload.urls],
|
||||
profiles=payload.profiles,
|
||||
hashtags=payload.hashtags,
|
||||
searchQueries=payload.search_queries,
|
||||
resultsPerPage=payload.results_per_page,
|
||||
)
|
||||
emit_progress(
|
||||
|
|
|
|||
|
|
@ -4,7 +4,8 @@ A lean, agent-friendly surface over ``TikTokScrapeInput``
|
|||
(``app/proprietary/platforms/tiktok``). The executor maps this to the full
|
||||
scraper input; the scraper's ``TikTokVideoItem`` is reused verbatim as the
|
||||
output element. Any TikTok URL kind (video, profile, hashtag, search) goes in
|
||||
``urls``; ``profiles``/``hashtags``/``search_queries`` are typed shortcuts.
|
||||
``urls``; ``profiles``/``hashtags`` are typed shortcuts. Keyword search is not a
|
||||
video source here — use ``tiktok.user_search`` to find accounts by keyword.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -26,8 +27,8 @@ class ScrapeInput(BaseModel):
|
|||
max_length=MAX_TIKTOK_SOURCES,
|
||||
description=(
|
||||
"TikTok URLs to scrape: a video, a profile (/@<user>), a hashtag "
|
||||
"(/tag/<name>), or a search URL. Provide these OR profiles/hashtags/"
|
||||
"search_queries (at least one source is required)."
|
||||
"(/tag/<name>), or a search URL. Provide these OR profiles/hashtags "
|
||||
"(at least one source is required)."
|
||||
),
|
||||
)
|
||||
profiles: list[str] = Field(
|
||||
|
|
@ -40,21 +41,11 @@ class ScrapeInput(BaseModel):
|
|||
max_length=MAX_TIKTOK_SOURCES,
|
||||
description="Hashtag names to scrape, without the leading '#'.",
|
||||
)
|
||||
search_queries: list[str] = Field(
|
||||
default_factory=list,
|
||||
max_length=MAX_TIKTOK_SOURCES,
|
||||
description=(
|
||||
"Search terms resolved via Google (site:tiktok.com) to public TikTok "
|
||||
"videos, since TikTok's own keyword search is login-walled. Slower "
|
||||
"than hashtags/urls. To find accounts by keyword, use "
|
||||
"tiktok.user_search instead."
|
||||
),
|
||||
)
|
||||
results_per_page: int = Field(
|
||||
default=10,
|
||||
ge=1,
|
||||
le=MAX_TIKTOK_ITEMS,
|
||||
description="Max videos to pull per profile/hashtag/search target.",
|
||||
description="Max videos to pull per profile/hashtag target.",
|
||||
)
|
||||
max_items: int = Field(
|
||||
default=10,
|
||||
|
|
@ -65,10 +56,9 @@ class ScrapeInput(BaseModel):
|
|||
|
||||
@model_validator(mode="after")
|
||||
def _require_a_source(self) -> ScrapeInput:
|
||||
if not any((self.urls, self.profiles, self.hashtags, self.search_queries)):
|
||||
if not any((self.urls, self.profiles, self.hashtags)):
|
||||
raise ValueError(
|
||||
"Provide at least one of 'urls', 'profiles', 'hashtags', or "
|
||||
"'search_queries'."
|
||||
"Provide at least one of 'urls', 'profiles', or 'hashtags'."
|
||||
)
|
||||
return self
|
||||
|
||||
|
|
|
|||
|
|
@ -222,9 +222,7 @@ class _RotatingSession:
|
|||
await self.close()
|
||||
self.rotations += 1
|
||||
await self._open()
|
||||
logger.info(
|
||||
"[instagram] rotated proxy session (rotation #%d)", self.rotations
|
||||
)
|
||||
logger.info("[instagram] rotated proxy session (rotation #%d)", self.rotations)
|
||||
return self.session
|
||||
|
||||
async def pace(self) -> None:
|
||||
|
|
@ -378,9 +376,7 @@ async def _fetch(
|
|||
if status == _BACKOFF_STATUS and backoffs < _MAX_BACKOFFS:
|
||||
backoffs += 1
|
||||
delay = _BACKOFF_BASE_S * (2 ** (backoffs - 1))
|
||||
logger.warning(
|
||||
"[instagram] 429 on %s; backing off %.1fs", path, delay
|
||||
)
|
||||
logger.warning("[instagram] 429 on %s; backing off %.1fs", path, delay)
|
||||
await asyncio.sleep(delay + random.uniform(0, 1))
|
||||
continue
|
||||
if status in _ROTATE_STATUSES:
|
||||
|
|
|
|||
|
|
@ -171,7 +171,9 @@ def _relay_child(node: dict[str, Any]) -> dict[str, Any]:
|
|||
mt = node.get("media_type")
|
||||
vv = node.get("video_versions")
|
||||
video_url = (
|
||||
vv[0].get("url") if isinstance(vv, list) and vv and isinstance(vv[0], dict) else None
|
||||
vv[0].get("url")
|
||||
if isinstance(vv, list) and vv and isinstance(vv[0], dict)
|
||||
else None
|
||||
)
|
||||
is_video = mt == 2 or bool(video_url)
|
||||
return {
|
||||
|
|
@ -290,9 +292,7 @@ def parse_profile(user: dict[str, Any]) -> dict[str, Any]:
|
|||
_APP_JSON_RE = re.compile(
|
||||
r'<script type="application/json"[^>]*>(.*?)</script>', re.DOTALL
|
||||
)
|
||||
_OG_RE = re.compile(
|
||||
r'<meta\s+property="og:([^"]+)"\s+content="([^"]*)"', re.IGNORECASE
|
||||
)
|
||||
_OG_RE = re.compile(r'<meta\s+property="og:([^"]+)"\s+content="([^"]*)"', re.IGNORECASE)
|
||||
# og tags are the fallback source (used only when the relay blob is absent). They
|
||||
# follow a fixed English shape because the fetch layer pins Accept-Language en-US:
|
||||
# og:description = "{likes} likes, {comments} comments - {username} on {Month D, YYYY}: "{caption}""
|
||||
|
|
@ -321,7 +321,9 @@ _MEDIA_ID_RE = re.compile(r"instagram://media\?id=(\d+)")
|
|||
def _og_date_to_iso(value: str) -> str | None:
|
||||
"""``"July 9, 2026"`` -> ``"2026-07-09"`` (date-only; og carries no time)."""
|
||||
try:
|
||||
return datetime.strptime(value, "%B %d, %Y").replace(tzinfo=UTC).date().isoformat()
|
||||
return (
|
||||
datetime.strptime(value, "%B %d, %Y").replace(tzinfo=UTC).date().isoformat()
|
||||
)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
|
@ -359,7 +361,7 @@ def _parse_og_meta(og: dict[str, str]) -> dict[str, Any]:
|
|||
elif owner_date:
|
||||
# No usable og:title: fall back to the caption after og:description's
|
||||
# date prefix — still clean (the counts/username/date are stripped).
|
||||
out["caption"] = _clean_caption(desc[owner_date.end():])
|
||||
out["caption"] = _clean_caption(desc[owner_date.end() :])
|
||||
return out
|
||||
|
||||
|
||||
|
|
@ -438,13 +440,21 @@ def _media_from_relay(
|
|||
mt = media.get("media_type")
|
||||
cap = media.get("caption")
|
||||
caption = (
|
||||
cap.get("text") if isinstance(cap, dict) else (cap if isinstance(cap, str) else None)
|
||||
cap.get("text")
|
||||
if isinstance(cap, dict)
|
||||
else (cap if isinstance(cap, str) else None)
|
||||
)
|
||||
carousel = media.get("carousel_media")
|
||||
carousel = [c for c in carousel if isinstance(c, dict)] if isinstance(carousel, list) else []
|
||||
carousel = (
|
||||
[c for c in carousel if isinstance(c, dict)]
|
||||
if isinstance(carousel, list)
|
||||
else []
|
||||
)
|
||||
vv = media.get("video_versions")
|
||||
video_url = (
|
||||
vv[0].get("url") if isinstance(vv, list) and vv and isinstance(vv[0], dict) else None
|
||||
vv[0].get("url")
|
||||
if isinstance(vv, list) and vv and isinstance(vv[0], dict)
|
||||
else None
|
||||
)
|
||||
is_video = mt == 2 or bool(video_url)
|
||||
owner = media.get("user") if isinstance(media.get("user"), dict) else {}
|
||||
|
|
@ -469,13 +479,18 @@ def _media_from_relay(
|
|||
"type": _MEDIA_TYPE.get(mt) or ("Video" if is_video else "Image"),
|
||||
"shortCode": media.get("code") or shortcode,
|
||||
"caption": caption,
|
||||
"hashtags": list(dict.fromkeys(_HASHTAG_RE.findall(caption))) if caption else [],
|
||||
"mentions": list(dict.fromkeys(_MENTION_RE.findall(caption))) if caption else [],
|
||||
"hashtags": list(dict.fromkeys(_HASHTAG_RE.findall(caption)))
|
||||
if caption
|
||||
else [],
|
||||
"mentions": list(dict.fromkeys(_MENTION_RE.findall(caption)))
|
||||
if caption
|
||||
else [],
|
||||
"url": url,
|
||||
"commentsCount": _int(media.get("comment_count")),
|
||||
"dimensionsHeight": _int(media.get("original_height")),
|
||||
"dimensionsWidth": _int(media.get("original_width")),
|
||||
"displayUrl": _iv2_url(media.get("image_versions2")) or media.get("display_uri"),
|
||||
"displayUrl": _iv2_url(media.get("image_versions2"))
|
||||
or media.get("display_uri"),
|
||||
"images": [
|
||||
u
|
||||
for c in carousel
|
||||
|
|
@ -535,8 +550,12 @@ def parse_post(
|
|||
"type": "Video" if is_video else "Image",
|
||||
"shortCode": shortcode,
|
||||
"caption": caption,
|
||||
"hashtags": list(dict.fromkeys(_HASHTAG_RE.findall(caption))) if caption else [],
|
||||
"mentions": list(dict.fromkeys(_MENTION_RE.findall(caption))) if caption else [],
|
||||
"hashtags": list(dict.fromkeys(_HASHTAG_RE.findall(caption)))
|
||||
if caption
|
||||
else [],
|
||||
"mentions": list(dict.fromkeys(_MENTION_RE.findall(caption)))
|
||||
if caption
|
||||
else [],
|
||||
"url": url,
|
||||
"commentsCount": og_meta.get("comments"),
|
||||
"displayUrl": og.get("image"),
|
||||
|
|
|
|||
|
|
@ -328,9 +328,7 @@ async def _discover_via_google(
|
|||
return resolved
|
||||
|
||||
|
||||
async def _discover(
|
||||
query: str, *, search_type: str, limit: int
|
||||
) -> list[ResolvedUrl]:
|
||||
async def _discover(query: str, *, search_type: str, limit: int) -> list[ResolvedUrl]:
|
||||
"""Resolve a discovery query into profile targets - anonymously.
|
||||
|
||||
A query that is a valid handle resolves directly against the anonymous
|
||||
|
|
@ -397,9 +395,7 @@ async def iter_instagram(
|
|||
|
||||
# posts / reels -> media feeds, de-duped by id across targets.
|
||||
jobs = [
|
||||
_media_flow(
|
||||
r, input_model=input_model, cutoff=cutoff, per_target=per_target
|
||||
)
|
||||
_media_flow(r, input_model=input_model, cutoff=cutoff, per_target=per_target)
|
||||
for r in targets
|
||||
]
|
||||
seen: set[str] = set()
|
||||
|
|
|
|||
|
|
@ -25,9 +25,7 @@ from urllib.parse import urlparse
|
|||
|
||||
ResolvedKind = Literal["profile", "post", "reel"]
|
||||
|
||||
_INSTAGRAM_HOSTS = frozenset(
|
||||
{"m.instagram.com", "www.instagram.com", "instagram.com"}
|
||||
)
|
||||
_INSTAGRAM_HOSTS = frozenset({"m.instagram.com", "www.instagram.com", "instagram.com"})
|
||||
_STRIP_SEGMENTS = frozenset({"_u", "profilecard"})
|
||||
_RESERVED = frozenset(
|
||||
{"p", "s", "tv", "reel", "reels", "share", "explore", "stories", "accounts"}
|
||||
|
|
@ -68,9 +66,7 @@ def resolve_url(url: str) -> ResolvedUrl | None:
|
|||
if "instagram.com" not in url.lower():
|
||||
token = url.strip().lstrip("@")
|
||||
if token and "/" not in token and "." not in token:
|
||||
return ResolvedUrl(
|
||||
"profile", token, f"https://www.instagram.com/{token}/"
|
||||
)
|
||||
return ResolvedUrl("profile", token, f"https://www.instagram.com/{token}/")
|
||||
segments = _segments(url)
|
||||
if not segments:
|
||||
return None
|
||||
|
|
@ -83,9 +79,7 @@ def resolve_url(url: str) -> ResolvedUrl | None:
|
|||
return ResolvedUrl("reel", code, url, numeric_post_id=code.isdigit())
|
||||
if head == "stories" and len(segments) >= 2:
|
||||
user = segments[1]
|
||||
return ResolvedUrl(
|
||||
"profile", user, f"https://www.instagram.com/{user}/"
|
||||
)
|
||||
return ResolvedUrl("profile", user, f"https://www.instagram.com/{user}/")
|
||||
if head not in _RESERVED:
|
||||
return ResolvedUrl("profile", head, url)
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -5,11 +5,11 @@ from __future__ import annotations
|
|||
from datetime import UTC, datetime
|
||||
|
||||
|
||||
def epoch_to_iso(seconds: int | None) -> str | None:
|
||||
def epoch_to_iso(seconds: int | str | None) -> str | None:
|
||||
"""Convert a Unix-seconds timestamp to ``YYYY-MM-DDTHH:MM:SS.000Z``."""
|
||||
if not seconds:
|
||||
return None
|
||||
stamp = datetime.fromtimestamp(seconds, tz=UTC)
|
||||
stamp = datetime.fromtimestamp(int(seconds), tz=UTC)
|
||||
return stamp.strftime("%Y-%m-%dT%H:%M:%S.000Z")
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -11,7 +11,9 @@ from ..targets.types import TikTokTarget
|
|||
from . import FetchFn
|
||||
|
||||
|
||||
async def iter_video(target: TikTokTarget, *, fetch: FetchFn) -> AsyncIterator[dict[str, Any]]:
|
||||
async def iter_video(
|
||||
target: TikTokTarget, *, fetch: FetchFn
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
html = await fetch(target.url)
|
||||
if not html:
|
||||
return
|
||||
|
|
|
|||
|
|
@ -10,10 +10,6 @@ from __future__ import annotations
|
|||
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
from app.proprietary.platforms.google_search.schemas import GoogleSearchScrapeInput
|
||||
from app.proprietary.platforms.google_search.scraper import scrape_serps
|
||||
|
||||
from .extraction.timestamps import now_iso
|
||||
from .flows import FetchCommentsFn, FetchFn, FetchListingFn, FetchUsersFn
|
||||
|
|
@ -35,26 +31,15 @@ from .targets.types import TikTokTarget
|
|||
|
||||
_PROFILE_URL = "https://www.tiktok.com/@{name}"
|
||||
_HASHTAG_URL = "https://www.tiktok.com/tag/{tag}"
|
||||
_SEARCH_URL = "https://www.tiktok.com/search?q={query}"
|
||||
_EXPLORE_URL = "https://www.tiktok.com/explore"
|
||||
|
||||
# A ``searchQueries`` term whose Google discovery surfaced no scrapable video
|
||||
# URLs degrades to one honest ErrorItem (mirrors the listing flow's contract:
|
||||
# never vanish silently).
|
||||
_EMPTY_DISCOVERY_MESSAGE = (
|
||||
"No public TikTok videos found for this query via Google discovery. Try a "
|
||||
"narrower phrasing, a hashtag, or a direct video URL."
|
||||
)
|
||||
|
||||
|
||||
def _resolve_targets(input_model: TikTokScrapeInput) -> list[TikTokTarget]:
|
||||
"""Build the target list from the URL/profile/hashtag sources.
|
||||
|
||||
``searchQueries`` is deliberately excluded: TikTok's own keyword search is
|
||||
login-walled for anonymous sessions, so it is routed through Google video
|
||||
discovery in :func:`iter_tiktok` instead. A raw ``tiktok.com/search?...``
|
||||
URL passed explicitly in ``startUrls``/``postURLs`` still resolves here and
|
||||
keeps its native listing routing.
|
||||
A raw ``tiktok.com/search?...`` URL passed explicitly in
|
||||
``startUrls``/``postURLs`` still resolves here and keeps its native listing
|
||||
routing; there is no keyword-search shortcut.
|
||||
"""
|
||||
targets: list[TikTokTarget] = []
|
||||
for entry in input_model.startUrls:
|
||||
|
|
@ -73,39 +58,6 @@ def _resolve_targets(input_model: TikTokScrapeInput) -> list[TikTokTarget]:
|
|||
return targets
|
||||
|
||||
|
||||
async def _discover_via_google(query: str, *, limit: int) -> list[TikTokTarget]:
|
||||
"""Discover public TikTok video targets via Google ``site:tiktok.com``.
|
||||
|
||||
TikTok's anonymous keyword search is login-walled, so we reuse the existing
|
||||
``google_search`` platform, classify each organic URL with ``resolve_target``,
|
||||
and keep only video hits (``/@user/video/<id>``) — the one kind that scrapes
|
||||
reliably over plain HTTP. Profile/hashtag/search/photo/non-tiktok results are
|
||||
dropped (accounts belong to the ``user_search`` verb). De-duped, capped at
|
||||
``limit``.
|
||||
"""
|
||||
serps = await scrape_serps(
|
||||
GoogleSearchScrapeInput(
|
||||
queries=query, site="tiktok.com", maxPagesPerQuery=1
|
||||
),
|
||||
limit=1,
|
||||
)
|
||||
resolved: list[TikTokTarget] = []
|
||||
seen: set[str] = set()
|
||||
for serp in serps:
|
||||
for org in serp.get("organicResults") or []:
|
||||
url = org.get("url", "") if isinstance(org, dict) else ""
|
||||
target = resolve_target(url)
|
||||
if target is None or target.kind != "video":
|
||||
continue
|
||||
if target.value in seen:
|
||||
continue
|
||||
seen.add(target.value)
|
||||
resolved.append(target)
|
||||
if len(resolved) >= limit:
|
||||
return resolved
|
||||
return resolved
|
||||
|
||||
|
||||
def _dispatch(
|
||||
target: TikTokTarget,
|
||||
*,
|
||||
|
|
@ -128,11 +80,9 @@ async def iter_tiktok(
|
|||
) -> AsyncIterator[dict[str, Any]]:
|
||||
"""Yield normalized items for every resolved target, in order.
|
||||
|
||||
Direct sources (URLs, profiles, hashtags) resolve up front; ``searchQueries``
|
||||
then run through Google video discovery. The video flow's ``fetch_html``
|
||||
opens its own warmed proxy session per call when none is bound; the listing
|
||||
flow drives its own browser. Neither binds a ContextVar across these
|
||||
``yield``s, so the generator stays context-safe.
|
||||
The video flow's ``fetch_html`` opens its own warmed proxy session per call
|
||||
when none is bound; the listing flow drives its own browser. Neither binds a
|
||||
ContextVar across these ``yield``s, so the generator stays context-safe.
|
||||
"""
|
||||
cap = input_model.resultsPerPage
|
||||
for target in _resolve_targets(input_model):
|
||||
|
|
@ -141,27 +91,6 @@ async def iter_tiktok(
|
|||
):
|
||||
yield item
|
||||
|
||||
# searchQueries -> Google-discovered public video URLs, de-duped across
|
||||
# queries so the same video surfacing under two terms is scraped once.
|
||||
seen_videos: set[str] = set()
|
||||
for query in input_model.searchQueries:
|
||||
discovered = await _discover_via_google(query, limit=cap)
|
||||
if not discovered:
|
||||
yield ErrorItem(
|
||||
url=_SEARCH_URL.format(query=quote(query)),
|
||||
input=query,
|
||||
error=_EMPTY_DISCOVERY_MESSAGE,
|
||||
errorCode="no_items",
|
||||
scrapedAt=now_iso(),
|
||||
).to_output()
|
||||
continue
|
||||
for target in discovered:
|
||||
if target.value in seen_videos:
|
||||
continue
|
||||
seen_videos.add(target.value)
|
||||
async for item in iter_video(target, fetch=fetch):
|
||||
yield item
|
||||
|
||||
|
||||
async def scrape_tiktok(
|
||||
input_model: TikTokScrapeInput,
|
||||
|
|
@ -174,7 +103,9 @@ async def scrape_tiktok(
|
|||
from app.capabilities.core.progress import emit_progress
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
async for item in iter_tiktok(input_model, fetch=fetch, fetch_listing=fetch_listing):
|
||||
async for item in iter_tiktok(
|
||||
input_model, fetch=fetch, fetch_listing=fetch_listing
|
||||
):
|
||||
results.append(item)
|
||||
emit_progress("scraping", current=len(results), total=limit, unit="item")
|
||||
if limit is not None and len(results) >= limit:
|
||||
|
|
|
|||
|
|
@ -54,9 +54,7 @@ from app.proprietary.platforms.instagram.url_resolver import resolve_url # noqa
|
|||
_PROFILE = "natgeo"
|
||||
_SEARCH_TERM = "national geographic"
|
||||
|
||||
_FIXTURE_DIR = (
|
||||
_BACKEND_ROOT / "tests" / "unit" / "platforms" / "instagram" / "fixtures"
|
||||
)
|
||||
_FIXTURE_DIR = _BACKEND_ROOT / "tests" / "unit" / "platforms" / "instagram" / "fixtures"
|
||||
|
||||
# Fields to strip from dumped fixtures so we never commit PII / volatile tokens.
|
||||
_PII_KEYS = frozenset(
|
||||
|
|
@ -98,7 +96,9 @@ async def step0_probe() -> bool:
|
|||
data = await fetch_json(
|
||||
"api/v1/users/web_profile_info/", {"username": _PROFILE}
|
||||
)
|
||||
user = (data or {}).get("data", {}).get("user") if isinstance(data, dict) else None
|
||||
user = (
|
||||
(data or {}).get("data", {}).get("user") if isinstance(data, dict) else None
|
||||
)
|
||||
print(f" web_profile_info({_PROFILE}) -> user={'yes' if user else 'no'}")
|
||||
return _check("sticky web_profile_info", minted and bool(user))
|
||||
|
||||
|
|
@ -179,9 +179,7 @@ async def step5_search() -> bool:
|
|||
|
||||
async def step6_dump_fixtures(post_url: str | None) -> bool:
|
||||
_hr("STEP 6 — dump trimmed, anonymized fixtures for offline tests")
|
||||
profile = await fetch_json(
|
||||
"api/v1/users/web_profile_info/", {"username": _PROFILE}
|
||||
)
|
||||
profile = await fetch_json("api/v1/users/web_profile_info/", {"username": _PROFILE})
|
||||
_FIXTURE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
wrote = []
|
||||
if isinstance(profile, dict) and profile.get("data", {}).get("user"):
|
||||
|
|
|
|||
|
|
@ -178,7 +178,9 @@ async def stage_pipeline() -> bool:
|
|||
f"{len(items)} item(s)",
|
||||
)
|
||||
if items:
|
||||
print(f" sample: {items[0].get('webVideoUrl')} — {items[0].get('text', '')[:60]!r}")
|
||||
print(
|
||||
f" sample: {items[0].get('webVideoUrl')} — {items[0].get('text', '')[:60]!r}"
|
||||
)
|
||||
return ok
|
||||
|
||||
|
||||
|
|
@ -210,9 +212,7 @@ async def stage_comments(video_url: str) -> tuple[bool, list[dict[str, Any]]]:
|
|||
# Comments load over a signed /api/comment/list XHR that TikTok serves to
|
||||
# anonymous sessions once the panel opens. Pass if real comments come back
|
||||
# OR a graceful ErrorItem (video has none / disabled / withheld).
|
||||
items = await scrape_tiktok_comments(
|
||||
[video_url], per_video=_COUNT, limit=_COUNT
|
||||
)
|
||||
items = await scrape_tiktok_comments([video_url], per_video=_COUNT, limit=_COUNT)
|
||||
has_comment = any(it.get("id") and not it.get("errorCode") for it in items)
|
||||
has_error = any(it.get("errorCode") == "no_comments" for it in items)
|
||||
ok = _check(
|
||||
|
|
@ -253,7 +253,9 @@ async def stage_trending() -> tuple[bool, list[dict[str, Any]]]:
|
|||
f"{len(items)} item(s); videos={len(real)}",
|
||||
)
|
||||
if real:
|
||||
print(f" sample: {real[0].get('webVideoUrl')} — {real[0].get('text', '')[:60]!r}")
|
||||
print(
|
||||
f" sample: {real[0].get('webVideoUrl')} — {real[0].get('text', '')[:60]!r}"
|
||||
)
|
||||
return ok, items
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -66,6 +66,4 @@ def test_details_wraps_profile_items():
|
|||
|
||||
def test_details_rejects_both_sources():
|
||||
with pytest.raises(ValidationError):
|
||||
DetailsInput(
|
||||
urls=["https://www.instagram.com/natgeo/"], search_queries=["x"]
|
||||
)
|
||||
DetailsInput(urls=["https://www.instagram.com/natgeo/"], search_queries=["x"])
|
||||
|
|
|
|||
|
|
@ -54,7 +54,6 @@ async def test_forwards_typed_sources_and_limit():
|
|||
ScrapeInput(
|
||||
profiles=["nasa"],
|
||||
hashtags=["food"],
|
||||
search_queries=["cats"],
|
||||
results_per_page=7,
|
||||
max_items=25,
|
||||
)
|
||||
|
|
@ -63,7 +62,6 @@ async def test_forwards_typed_sources_and_limit():
|
|||
(actor_input, limit) = scraper.calls[0]
|
||||
assert actor_input.profiles == ["nasa"]
|
||||
assert actor_input.hashtags == ["food"]
|
||||
assert actor_input.searchQueries == ["cats"]
|
||||
assert actor_input.resultsPerPage == 7
|
||||
# The outer collection limit is the caller's total-item cap.
|
||||
assert limit == 25
|
||||
|
|
|
|||
|
|
@ -42,8 +42,7 @@ def _profile_payload(n: int) -> dict:
|
|||
"edge_owner_to_timeline_media": {
|
||||
"count": n,
|
||||
"edges": [
|
||||
{"node": {"id": str(i), "shortcode": f"S{i}"}}
|
||||
for i in range(n)
|
||||
{"node": {"id": str(i), "shortcode": f"S{i}"}} for i in range(n)
|
||||
],
|
||||
},
|
||||
}
|
||||
|
|
|
|||
|
|
@ -33,9 +33,7 @@ async def test_google_discovery_keeps_only_profiles(monkeypatch):
|
|||
"https://example.com/not-instagram",
|
||||
),
|
||||
)
|
||||
targets = await scraper._discover(
|
||||
"nat geo photos", search_type="profile", limit=10
|
||||
)
|
||||
targets = await scraper._discover("nat geo photos", search_type="profile", limit=10)
|
||||
assert [(t.kind, t.value) for t in targets] == [("profile", "natgeo")]
|
||||
|
||||
|
||||
|
|
@ -48,9 +46,7 @@ async def test_google_discovery_dedupes(monkeypatch):
|
|||
"https://www.instagram.com/natgeo/",
|
||||
),
|
||||
)
|
||||
targets = await scraper._discover(
|
||||
"nat geo photos", search_type="profile", limit=10
|
||||
)
|
||||
targets = await scraper._discover("nat geo photos", search_type="profile", limit=10)
|
||||
assert len(targets) == 1
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -102,7 +102,9 @@ async def test_warms_then_returns_json():
|
|||
holder = _FakeHolder([_FakeSession(200, csrftoken=True)])
|
||||
token = _current_session.set(holder)
|
||||
try:
|
||||
result = await fetch_json("api/v1/users/web_profile_info/", {"username": "natgeo"})
|
||||
result = await fetch_json(
|
||||
"api/v1/users/web_profile_info/", {"username": "natgeo"}
|
||||
)
|
||||
finally:
|
||||
_current_session.reset(token)
|
||||
assert result == _PAYLOAD
|
||||
|
|
|
|||
|
|
@ -175,8 +175,14 @@ def test_parse_post_prefers_relay_json():
|
|||
"image_versions2": {"candidates": [{"url": "https://cdn/c2.jpg"}]},
|
||||
},
|
||||
],
|
||||
"usertags": {"in": [{"position": [0.5, 0.5], "user": {"username": "tagged1", "id": "77"}}]},
|
||||
"coauthor_producers": [{"username": "coauthor1", "id": "88", "is_verified": True}],
|
||||
"usertags": {
|
||||
"in": [
|
||||
{"position": [0.5, 0.5], "user": {"username": "tagged1", "id": "77"}}
|
||||
]
|
||||
},
|
||||
"coauthor_producers": [
|
||||
{"username": "coauthor1", "id": "88", "is_verified": True}
|
||||
],
|
||||
"location": {"id": "123", "name": "Bali"},
|
||||
}
|
||||
html = (
|
||||
|
|
|
|||
|
|
@ -1,149 +0,0 @@
|
|||
"""Offline tests for Google-backed TikTok video discovery.
|
||||
|
||||
``searchQueries`` are login-walled on TikTok's native search, so they route
|
||||
through the ``google_search`` platform (``site:tiktok.com``): each organic URL
|
||||
is classified with ``resolve_target`` and only video hits (``/@user/video/<id>``)
|
||||
are kept — profiles/hashtags/search/photo/non-tiktok are dropped (accounts
|
||||
belong to the user-search verb). These tests inject a fake ``scrape_serps`` so
|
||||
there is no network: they pin the classification, cross-query de-dup, the limit
|
||||
cap, the barren-query ErrorItem, and that no ``/search?q=`` listing target is
|
||||
ever built.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from app.proprietary.platforms.tiktok import (
|
||||
TikTokScrapeInput,
|
||||
orchestrator,
|
||||
scrape_tiktok,
|
||||
)
|
||||
|
||||
|
||||
def _fake_serps(*organic_urls: str):
|
||||
async def _scrape_serps(input_model, *, limit=None):
|
||||
assert input_model.site == "tiktok.com"
|
||||
assert input_model.maxPagesPerQuery == 1
|
||||
return [{"organicResults": [{"url": u} for u in organic_urls]}]
|
||||
|
||||
return _scrape_serps
|
||||
|
||||
|
||||
def _video_page(url: str) -> str:
|
||||
"""Render a rehydration blob for a ``/@user/video/<id>`` URL."""
|
||||
video_id = url.rsplit("/", 1)[1]
|
||||
username = url.split("@")[1].split("/")[0]
|
||||
blob = {
|
||||
"__DEFAULT_SCOPE__": {
|
||||
"webapp.video-detail": {
|
||||
"itemInfo": {
|
||||
"itemStruct": {
|
||||
"id": video_id,
|
||||
"desc": "hi",
|
||||
"author": {"uniqueId": username},
|
||||
"stats": {"diggCount": 1},
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return (
|
||||
'<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" '
|
||||
f'type="application/json">{json.dumps(blob)}</script>'
|
||||
)
|
||||
|
||||
|
||||
async def _fetch_video(url: str) -> str:
|
||||
return _video_page(url)
|
||||
|
||||
|
||||
async def test_search_discovery_keeps_only_videos(monkeypatch):
|
||||
# Only the video URL survives; profile / hashtag / search / photo /
|
||||
# non-tiktok organic results are dropped.
|
||||
monkeypatch.setattr(
|
||||
orchestrator,
|
||||
"scrape_serps",
|
||||
_fake_serps(
|
||||
"https://www.tiktok.com/@nasa/video/123",
|
||||
"https://www.tiktok.com/@nasa",
|
||||
"https://www.tiktok.com/tag/space",
|
||||
"https://www.tiktok.com/search?q=space",
|
||||
"https://www.tiktok.com/@nasa/photo/999",
|
||||
"https://example.com/not-tiktok",
|
||||
),
|
||||
)
|
||||
items = await scrape_tiktok(
|
||||
TikTokScrapeInput(searchQueries=["space"], resultsPerPage=10),
|
||||
fetch=_fetch_video,
|
||||
)
|
||||
assert [i["id"] for i in items] == ["123"]
|
||||
|
||||
|
||||
async def test_search_discovery_dedupes_across_queries(monkeypatch):
|
||||
# The same video surfacing under two queries is scraped once.
|
||||
monkeypatch.setattr(
|
||||
orchestrator,
|
||||
"scrape_serps",
|
||||
_fake_serps("https://www.tiktok.com/@nasa/video/123"),
|
||||
)
|
||||
items = await scrape_tiktok(
|
||||
TikTokScrapeInput(searchQueries=["space", "rockets"], resultsPerPage=10),
|
||||
fetch=_fetch_video,
|
||||
)
|
||||
assert [i["id"] for i in items] == ["123"]
|
||||
|
||||
|
||||
async def test_search_discovery_respects_per_target_limit(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
orchestrator,
|
||||
"scrape_serps",
|
||||
_fake_serps(
|
||||
"https://www.tiktok.com/@a/video/1",
|
||||
"https://www.tiktok.com/@b/video/2",
|
||||
"https://www.tiktok.com/@c/video/3",
|
||||
),
|
||||
)
|
||||
items = await scrape_tiktok(
|
||||
TikTokScrapeInput(searchQueries=["x"], resultsPerPage=2),
|
||||
fetch=_fetch_video,
|
||||
)
|
||||
assert [i["id"] for i in items] == ["1", "2"]
|
||||
|
||||
|
||||
async def test_search_barren_query_emits_error_item(monkeypatch):
|
||||
# A query whose discovery finds no video URLs degrades to one ErrorItem.
|
||||
monkeypatch.setattr(
|
||||
orchestrator,
|
||||
"scrape_serps",
|
||||
_fake_serps(
|
||||
"https://www.tiktok.com/@nasa",
|
||||
"https://example.com/x",
|
||||
),
|
||||
)
|
||||
items = await scrape_tiktok(
|
||||
TikTokScrapeInput(searchQueries=["space"], resultsPerPage=10),
|
||||
fetch=_fetch_video,
|
||||
)
|
||||
assert len(items) == 1
|
||||
assert items[0]["errorCode"] == "no_items"
|
||||
assert items[0]["input"] == "space"
|
||||
|
||||
|
||||
async def test_search_never_builds_listing_target(monkeypatch):
|
||||
# searchQueries must never hit the (login-walled) native search listing flow.
|
||||
monkeypatch.setattr(
|
||||
orchestrator,
|
||||
"scrape_serps",
|
||||
_fake_serps("https://www.tiktok.com/@nasa/video/123"),
|
||||
)
|
||||
|
||||
async def _boom_listing(_url: str, _count: int) -> list[dict]:
|
||||
raise AssertionError("searchQueries must not build a listing target")
|
||||
|
||||
items = await scrape_tiktok(
|
||||
TikTokScrapeInput(searchQueries=["space"], resultsPerPage=10),
|
||||
fetch=_fetch_video,
|
||||
fetch_listing=_boom_listing,
|
||||
)
|
||||
assert [i["id"] for i in items] == ["123"]
|
||||
|
|
@ -84,7 +84,9 @@ async def test_warms_then_returns_html():
|
|||
|
||||
|
||||
async def test_rotates_when_warm_fails_then_succeeds():
|
||||
holder = _FakeHolder([_FakeSession(200, warms=False), _FakeSession(200, warms=True)])
|
||||
holder = _FakeHolder(
|
||||
[_FakeSession(200, warms=False), _FakeSession(200, warms=True)]
|
||||
)
|
||||
token = _current_session.set(holder)
|
||||
try:
|
||||
result = await client.fetch_html("https://www.tiktok.com/@scout2015")
|
||||
|
|
@ -119,9 +121,7 @@ async def test_rotates_and_rewarms_on_403():
|
|||
|
||||
async def test_persistent_403_raises_blocked(monkeypatch):
|
||||
_no_sleep(monkeypatch)
|
||||
holder = _FakeHolder(
|
||||
[_FakeSession(403) for _ in range(client._MAX_ROTATIONS + 1)]
|
||||
)
|
||||
holder = _FakeHolder([_FakeSession(403) for _ in range(client._MAX_ROTATIONS + 1)])
|
||||
token = _current_session.set(holder)
|
||||
try:
|
||||
raised = False
|
||||
|
|
|
|||
|
|
@ -6,7 +6,9 @@ from app.proprietary.platforms.tiktok.targets import resolve_target
|
|||
|
||||
|
||||
def test_resolve_video_carries_username_and_id():
|
||||
target = resolve_target("https://www.tiktok.com/@scout2015/video/6718335390845095173")
|
||||
target = resolve_target(
|
||||
"https://www.tiktok.com/@scout2015/video/6718335390845095173"
|
||||
)
|
||||
assert target is not None
|
||||
assert target.kind == "video"
|
||||
assert target.value == "6718335390845095173"
|
||||
|
|
|
|||
|
|
@ -28,9 +28,7 @@ async def test_user_search_parses_dedupes_and_caps():
|
|||
async def fake_fetch(_url: str, _cap: int) -> list[dict]:
|
||||
return [_user("1", "nasa"), _user("1", "nasa"), _user("2", "nasa2")]
|
||||
|
||||
items = await search_tiktok_users(
|
||||
["nasa"], per_query=2, fetch_users=fake_fetch
|
||||
)
|
||||
items = await search_tiktok_users(["nasa"], per_query=2, fetch_users=fake_fetch)
|
||||
|
||||
assert [i["id"] for i in items] == ["1", "2"]
|
||||
first = items[0]
|
||||
|
|
@ -49,9 +47,7 @@ async def test_user_search_empty_query_emits_error_item():
|
|||
async def fake_fetch(_url: str, _cap: int) -> list[dict]:
|
||||
return []
|
||||
|
||||
items = await search_tiktok_users(
|
||||
["ghost"], per_query=5, fetch_users=fake_fetch
|
||||
)
|
||||
items = await search_tiktok_users(["ghost"], per_query=5, fetch_users=fake_fetch)
|
||||
|
||||
assert len(items) == 1
|
||||
assert items[0]["errorCode"] == "no_users"
|
||||
|
|
|
|||
|
|
@ -58,9 +58,7 @@ def test_chat_model_requires_slash_tools_and_context():
|
|||
|
||||
|
||||
def test_excluded_provider_slug_is_filtered():
|
||||
assert not is_requesty_chat_model(
|
||||
_requesty_model(model_id="amazon/nova-pro-v1")
|
||||
)
|
||||
assert not is_requesty_chat_model(_requesty_model(model_id="amazon/nova-pro-v1"))
|
||||
|
||||
|
||||
def test_image_generation_models_excluded_from_chat_and_flagged():
|
||||
|
|
@ -89,9 +87,7 @@ def test_normalize_maps_context_window_and_capabilities():
|
|||
name="GPT-4o mini",
|
||||
),
|
||||
_requesty_model(model_id="openai/gpt-4o-mini", tools=False),
|
||||
_requesty_model(
|
||||
model_id="black-forest-labs/flux", image_generation=True
|
||||
),
|
||||
_requesty_model(model_id="black-forest-labs/flux", image_generation=True),
|
||||
]
|
||||
)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue