mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-18 23:11:12 +02:00
refactor: streamline TikTok and Instagram scraping logic by removing search_queries and enhancing documentation for clarity
This commit is contained in:
DESKTOP-RTLN3BA\$punk
parent
e8b3692b54
commit
2b018c4474
111 changed files with 1800 additions and 1580 deletions
|
|
@ -14,11 +14,11 @@ Answer the delegated question from live TikTok data gathered with your verb, com
|
|||
</available_tools>
|
||||
|
||||
<playbook>
|
||||
- Finding videos on a topic: prefer `tiktok_scrape` with `hashtags` (no leading '#') or a direct TikTok URL in `urls` (fastest). `search_queries` also finds videos on a topic, but it is Google-backed and slow, so start with **at most 3** distinct queries and only add more if the first round returns nothing significant — never batch many phrasing variants of the same intent.
|
||||
- Finding videos on a topic: call `tiktok_scrape` with `hashtags` (no leading '#'), or pass a TikTok URL in `urls`. There is no keyword-video search — use hashtags or a video URL.
|
||||
- Scraping a specific video, profile, hashtag, or search page: pass its TikTok URL in `urls`.
|
||||
- Profiles: a creator's `profiles` feed returns the account's metadata (name, followers, bio, verification) reliably, but its video list is often withheld by TikTok — treat an empty video list as a known limit, not a failure to retry endlessly. Prefer `hashtags` or a direct video URL for videos.
|
||||
- Comments on a video: call `tiktok_comments` with the video URL(s) in `video_urls`.
|
||||
- Finding accounts by keyword: call `tiktok_user_search` with `queries` — that is the path for accounts. Use `search_queries` on `tiktok_scrape` only when you want videos, not accounts.
|
||||
- Finding accounts by keyword: call `tiktok_user_search` with `queries` — that is the path for accounts.
|
||||
- "What's trending now": call `tiktok_trending` (no query needed); set `max_items` for how many.
|
||||
- Controlling volume: use `max_items` for the total cap and `results_per_page` per target (per-verb equivalents: `comments_per_video`, `results_per_query`).
|
||||
- Requested counts: `max_items` defaults low — when the task asks for N items, set `max_items` (and the per-target count) above N. A call that caps below the target can never satisfy it.
|
||||
|
|
|
|||
|
|
@ -54,9 +54,7 @@ class DetailsInput(BaseModel):
|
|||
@model_validator(mode="after")
|
||||
def _exactly_one_source(self) -> DetailsInput:
|
||||
if not self.urls and not self.search_queries:
|
||||
raise ValueError(
|
||||
"Provide at least one of 'urls' or 'search_queries'."
|
||||
)
|
||||
raise ValueError("Provide at least one of 'urls' or 'search_queries'.")
|
||||
if self.urls and self.search_queries:
|
||||
raise ValueError(
|
||||
"Provide 'urls' OR 'search_queries', not both (they cannot be combined)."
|
||||
|
|
|
|||
|
|
@ -77,9 +77,7 @@ class ScrapeInput(BaseModel):
|
|||
@model_validator(mode="after")
|
||||
def _exactly_one_source(self) -> ScrapeInput:
|
||||
if not self.urls and not self.search_queries:
|
||||
raise ValueError(
|
||||
"Provide at least one of 'urls' or 'search_queries'."
|
||||
)
|
||||
raise ValueError("Provide at least one of 'urls' or 'search_queries'.")
|
||||
if self.urls and self.search_queries:
|
||||
raise ValueError(
|
||||
"Provide 'urls' OR 'search_queries', not both (they cannot be combined)."
|
||||
|
|
|
|||
|
|
@ -10,9 +10,8 @@ from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput
|
|||
TIKTOK_SCRAPE = Capability(
|
||||
name="tiktok.scrape",
|
||||
description=(
|
||||
"Scrape public TikTok videos. Use urls, profiles, hashtags, or "
|
||||
"search_queries (search_queries are resolved via Google to public "
|
||||
"videos; for accounts by keyword use tiktok.user_search)."
|
||||
"Scrape public TikTok videos. Use urls, profiles, or hashtags. To find "
|
||||
"accounts by keyword, use tiktok.user_search."
|
||||
),
|
||||
input_schema=ScrapeInput,
|
||||
output_schema=ScrapeOutput,
|
||||
|
|
|
|||
|
|
@ -26,7 +26,6 @@ def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor:
|
|||
startUrls=[{"url": url} for url in payload.urls],
|
||||
profiles=payload.profiles,
|
||||
hashtags=payload.hashtags,
|
||||
searchQueries=payload.search_queries,
|
||||
resultsPerPage=payload.results_per_page,
|
||||
)
|
||||
emit_progress(
|
||||
|
|
|
|||
|
|
@ -4,7 +4,8 @@ A lean, agent-friendly surface over ``TikTokScrapeInput``
|
|||
(``app/proprietary/platforms/tiktok``). The executor maps this to the full
|
||||
scraper input; the scraper's ``TikTokVideoItem`` is reused verbatim as the
|
||||
output element. Any TikTok URL kind (video, profile, hashtag, search) goes in
|
||||
``urls``; ``profiles``/``hashtags``/``search_queries`` are typed shortcuts.
|
||||
``urls``; ``profiles``/``hashtags`` are typed shortcuts. Keyword search is not a
|
||||
video source here — use ``tiktok.user_search`` to find accounts by keyword.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -26,8 +27,8 @@ class ScrapeInput(BaseModel):
|
|||
max_length=MAX_TIKTOK_SOURCES,
|
||||
description=(
|
||||
"TikTok URLs to scrape: a video, a profile (/@<user>), a hashtag "
|
||||
"(/tag/<name>), or a search URL. Provide these OR profiles/hashtags/"
|
||||
"search_queries (at least one source is required)."
|
||||
"(/tag/<name>), or a search URL. Provide these OR profiles/hashtags "
|
||||
"(at least one source is required)."
|
||||
),
|
||||
)
|
||||
profiles: list[str] = Field(
|
||||
|
|
@ -40,21 +41,11 @@ class ScrapeInput(BaseModel):
|
|||
max_length=MAX_TIKTOK_SOURCES,
|
||||
description="Hashtag names to scrape, without the leading '#'.",
|
||||
)
|
||||
search_queries: list[str] = Field(
|
||||
default_factory=list,
|
||||
max_length=MAX_TIKTOK_SOURCES,
|
||||
description=(
|
||||
"Search terms resolved via Google (site:tiktok.com) to public TikTok "
|
||||
"videos, since TikTok's own keyword search is login-walled. Slower "
|
||||
"than hashtags/urls. To find accounts by keyword, use "
|
||||
"tiktok.user_search instead."
|
||||
),
|
||||
)
|
||||
results_per_page: int = Field(
|
||||
default=10,
|
||||
ge=1,
|
||||
le=MAX_TIKTOK_ITEMS,
|
||||
description="Max videos to pull per profile/hashtag/search target.",
|
||||
description="Max videos to pull per profile/hashtag target.",
|
||||
)
|
||||
max_items: int = Field(
|
||||
default=10,
|
||||
|
|
@ -65,10 +56,9 @@ class ScrapeInput(BaseModel):
|
|||
|
||||
@model_validator(mode="after")
|
||||
def _require_a_source(self) -> ScrapeInput:
|
||||
if not any((self.urls, self.profiles, self.hashtags, self.search_queries)):
|
||||
if not any((self.urls, self.profiles, self.hashtags)):
|
||||
raise ValueError(
|
||||
"Provide at least one of 'urls', 'profiles', 'hashtags', or "
|
||||
"'search_queries'."
|
||||
"Provide at least one of 'urls', 'profiles', or 'hashtags'."
|
||||
)
|
||||
return self
|
||||
|
||||
|
|
|
|||
|
|
@ -222,9 +222,7 @@ class _RotatingSession:
|
|||
await self.close()
|
||||
self.rotations += 1
|
||||
await self._open()
|
||||
logger.info(
|
||||
"[instagram] rotated proxy session (rotation #%d)", self.rotations
|
||||
)
|
||||
logger.info("[instagram] rotated proxy session (rotation #%d)", self.rotations)
|
||||
return self.session
|
||||
|
||||
async def pace(self) -> None:
|
||||
|
|
@ -378,9 +376,7 @@ async def _fetch(
|
|||
if status == _BACKOFF_STATUS and backoffs < _MAX_BACKOFFS:
|
||||
backoffs += 1
|
||||
delay = _BACKOFF_BASE_S * (2 ** (backoffs - 1))
|
||||
logger.warning(
|
||||
"[instagram] 429 on %s; backing off %.1fs", path, delay
|
||||
)
|
||||
logger.warning("[instagram] 429 on %s; backing off %.1fs", path, delay)
|
||||
await asyncio.sleep(delay + random.uniform(0, 1))
|
||||
continue
|
||||
if status in _ROTATE_STATUSES:
|
||||
|
|
|
|||
|
|
@ -171,7 +171,9 @@ def _relay_child(node: dict[str, Any]) -> dict[str, Any]:
|
|||
mt = node.get("media_type")
|
||||
vv = node.get("video_versions")
|
||||
video_url = (
|
||||
vv[0].get("url") if isinstance(vv, list) and vv and isinstance(vv[0], dict) else None
|
||||
vv[0].get("url")
|
||||
if isinstance(vv, list) and vv and isinstance(vv[0], dict)
|
||||
else None
|
||||
)
|
||||
is_video = mt == 2 or bool(video_url)
|
||||
return {
|
||||
|
|
@ -290,9 +292,7 @@ def parse_profile(user: dict[str, Any]) -> dict[str, Any]:
|
|||
_APP_JSON_RE = re.compile(
|
||||
r'<script type="application/json"[^>]*>(.*?)</script>', re.DOTALL
|
||||
)
|
||||
_OG_RE = re.compile(
|
||||
r'<meta\s+property="og:([^"]+)"\s+content="([^"]*)"', re.IGNORECASE
|
||||
)
|
||||
_OG_RE = re.compile(r'<meta\s+property="og:([^"]+)"\s+content="([^"]*)"', re.IGNORECASE)
|
||||
# og tags are the fallback source (used only when the relay blob is absent). They
|
||||
# follow a fixed English shape because the fetch layer pins Accept-Language en-US:
|
||||
# og:description = "{likes} likes, {comments} comments - {username} on {Month D, YYYY}: "{caption}""
|
||||
|
|
@ -321,7 +321,9 @@ _MEDIA_ID_RE = re.compile(r"instagram://media\?id=(\d+)")
|
|||
def _og_date_to_iso(value: str) -> str | None:
|
||||
"""``"July 9, 2026"`` -> ``"2026-07-09"`` (date-only; og carries no time)."""
|
||||
try:
|
||||
return datetime.strptime(value, "%B %d, %Y").replace(tzinfo=UTC).date().isoformat()
|
||||
return (
|
||||
datetime.strptime(value, "%B %d, %Y").replace(tzinfo=UTC).date().isoformat()
|
||||
)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
|
@ -359,7 +361,7 @@ def _parse_og_meta(og: dict[str, str]) -> dict[str, Any]:
|
|||
elif owner_date:
|
||||
# No usable og:title: fall back to the caption after og:description's
|
||||
# date prefix — still clean (the counts/username/date are stripped).
|
||||
out["caption"] = _clean_caption(desc[owner_date.end():])
|
||||
out["caption"] = _clean_caption(desc[owner_date.end() :])
|
||||
return out
|
||||
|
||||
|
||||
|
|
@ -438,13 +440,21 @@ def _media_from_relay(
|
|||
mt = media.get("media_type")
|
||||
cap = media.get("caption")
|
||||
caption = (
|
||||
cap.get("text") if isinstance(cap, dict) else (cap if isinstance(cap, str) else None)
|
||||
cap.get("text")
|
||||
if isinstance(cap, dict)
|
||||
else (cap if isinstance(cap, str) else None)
|
||||
)
|
||||
carousel = media.get("carousel_media")
|
||||
carousel = [c for c in carousel if isinstance(c, dict)] if isinstance(carousel, list) else []
|
||||
carousel = (
|
||||
[c for c in carousel if isinstance(c, dict)]
|
||||
if isinstance(carousel, list)
|
||||
else []
|
||||
)
|
||||
vv = media.get("video_versions")
|
||||
video_url = (
|
||||
vv[0].get("url") if isinstance(vv, list) and vv and isinstance(vv[0], dict) else None
|
||||
vv[0].get("url")
|
||||
if isinstance(vv, list) and vv and isinstance(vv[0], dict)
|
||||
else None
|
||||
)
|
||||
is_video = mt == 2 or bool(video_url)
|
||||
owner = media.get("user") if isinstance(media.get("user"), dict) else {}
|
||||
|
|
@ -469,13 +479,18 @@ def _media_from_relay(
|
|||
"type": _MEDIA_TYPE.get(mt) or ("Video" if is_video else "Image"),
|
||||
"shortCode": media.get("code") or shortcode,
|
||||
"caption": caption,
|
||||
"hashtags": list(dict.fromkeys(_HASHTAG_RE.findall(caption))) if caption else [],
|
||||
"mentions": list(dict.fromkeys(_MENTION_RE.findall(caption))) if caption else [],
|
||||
"hashtags": list(dict.fromkeys(_HASHTAG_RE.findall(caption)))
|
||||
if caption
|
||||
else [],
|
||||
"mentions": list(dict.fromkeys(_MENTION_RE.findall(caption)))
|
||||
if caption
|
||||
else [],
|
||||
"url": url,
|
||||
"commentsCount": _int(media.get("comment_count")),
|
||||
"dimensionsHeight": _int(media.get("original_height")),
|
||||
"dimensionsWidth": _int(media.get("original_width")),
|
||||
"displayUrl": _iv2_url(media.get("image_versions2")) or media.get("display_uri"),
|
||||
"displayUrl": _iv2_url(media.get("image_versions2"))
|
||||
or media.get("display_uri"),
|
||||
"images": [
|
||||
u
|
||||
for c in carousel
|
||||
|
|
@ -535,8 +550,12 @@ def parse_post(
|
|||
"type": "Video" if is_video else "Image",
|
||||
"shortCode": shortcode,
|
||||
"caption": caption,
|
||||
"hashtags": list(dict.fromkeys(_HASHTAG_RE.findall(caption))) if caption else [],
|
||||
"mentions": list(dict.fromkeys(_MENTION_RE.findall(caption))) if caption else [],
|
||||
"hashtags": list(dict.fromkeys(_HASHTAG_RE.findall(caption)))
|
||||
if caption
|
||||
else [],
|
||||
"mentions": list(dict.fromkeys(_MENTION_RE.findall(caption)))
|
||||
if caption
|
||||
else [],
|
||||
"url": url,
|
||||
"commentsCount": og_meta.get("comments"),
|
||||
"displayUrl": og.get("image"),
|
||||
|
|
|
|||
|
|
@ -328,9 +328,7 @@ async def _discover_via_google(
|
|||
return resolved
|
||||
|
||||
|
||||
async def _discover(
|
||||
query: str, *, search_type: str, limit: int
|
||||
) -> list[ResolvedUrl]:
|
||||
async def _discover(query: str, *, search_type: str, limit: int) -> list[ResolvedUrl]:
|
||||
"""Resolve a discovery query into profile targets - anonymously.
|
||||
|
||||
A query that is a valid handle resolves directly against the anonymous
|
||||
|
|
@ -397,9 +395,7 @@ async def iter_instagram(
|
|||
|
||||
# posts / reels -> media feeds, de-duped by id across targets.
|
||||
jobs = [
|
||||
_media_flow(
|
||||
r, input_model=input_model, cutoff=cutoff, per_target=per_target
|
||||
)
|
||||
_media_flow(r, input_model=input_model, cutoff=cutoff, per_target=per_target)
|
||||
for r in targets
|
||||
]
|
||||
seen: set[str] = set()
|
||||
|
|
|
|||
|
|
@ -25,9 +25,7 @@ from urllib.parse import urlparse
|
|||
|
||||
ResolvedKind = Literal["profile", "post", "reel"]
|
||||
|
||||
_INSTAGRAM_HOSTS = frozenset(
|
||||
{"m.instagram.com", "www.instagram.com", "instagram.com"}
|
||||
)
|
||||
_INSTAGRAM_HOSTS = frozenset({"m.instagram.com", "www.instagram.com", "instagram.com"})
|
||||
_STRIP_SEGMENTS = frozenset({"_u", "profilecard"})
|
||||
_RESERVED = frozenset(
|
||||
{"p", "s", "tv", "reel", "reels", "share", "explore", "stories", "accounts"}
|
||||
|
|
@ -68,9 +66,7 @@ def resolve_url(url: str) -> ResolvedUrl | None:
|
|||
if "instagram.com" not in url.lower():
|
||||
token = url.strip().lstrip("@")
|
||||
if token and "/" not in token and "." not in token:
|
||||
return ResolvedUrl(
|
||||
"profile", token, f"https://www.instagram.com/{token}/"
|
||||
)
|
||||
return ResolvedUrl("profile", token, f"https://www.instagram.com/{token}/")
|
||||
segments = _segments(url)
|
||||
if not segments:
|
||||
return None
|
||||
|
|
@ -83,9 +79,7 @@ def resolve_url(url: str) -> ResolvedUrl | None:
|
|||
return ResolvedUrl("reel", code, url, numeric_post_id=code.isdigit())
|
||||
if head == "stories" and len(segments) >= 2:
|
||||
user = segments[1]
|
||||
return ResolvedUrl(
|
||||
"profile", user, f"https://www.instagram.com/{user}/"
|
||||
)
|
||||
return ResolvedUrl("profile", user, f"https://www.instagram.com/{user}/")
|
||||
if head not in _RESERVED:
|
||||
return ResolvedUrl("profile", head, url)
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -5,11 +5,11 @@ from __future__ import annotations
|
|||
from datetime import UTC, datetime
|
||||
|
||||
|
||||
def epoch_to_iso(seconds: int | None) -> str | None:
|
||||
def epoch_to_iso(seconds: int | str | None) -> str | None:
|
||||
"""Convert a Unix-seconds timestamp to ``YYYY-MM-DDTHH:MM:SS.000Z``."""
|
||||
if not seconds:
|
||||
return None
|
||||
stamp = datetime.fromtimestamp(seconds, tz=UTC)
|
||||
stamp = datetime.fromtimestamp(int(seconds), tz=UTC)
|
||||
return stamp.strftime("%Y-%m-%dT%H:%M:%S.000Z")
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -11,7 +11,9 @@ from ..targets.types import TikTokTarget
|
|||
from . import FetchFn
|
||||
|
||||
|
||||
async def iter_video(target: TikTokTarget, *, fetch: FetchFn) -> AsyncIterator[dict[str, Any]]:
|
||||
async def iter_video(
|
||||
target: TikTokTarget, *, fetch: FetchFn
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
html = await fetch(target.url)
|
||||
if not html:
|
||||
return
|
||||
|
|
|
|||
|
|
@ -10,10 +10,6 @@ from __future__ import annotations
|
|||
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
from app.proprietary.platforms.google_search.schemas import GoogleSearchScrapeInput
|
||||
from app.proprietary.platforms.google_search.scraper import scrape_serps
|
||||
|
||||
from .extraction.timestamps import now_iso
|
||||
from .flows import FetchCommentsFn, FetchFn, FetchListingFn, FetchUsersFn
|
||||
|
|
@ -35,26 +31,15 @@ from .targets.types import TikTokTarget
|
|||
|
||||
_PROFILE_URL = "https://www.tiktok.com/@{name}"
|
||||
_HASHTAG_URL = "https://www.tiktok.com/tag/{tag}"
|
||||
_SEARCH_URL = "https://www.tiktok.com/search?q={query}"
|
||||
_EXPLORE_URL = "https://www.tiktok.com/explore"
|
||||
|
||||
# A ``searchQueries`` term whose Google discovery surfaced no scrapable video
|
||||
# URLs degrades to one honest ErrorItem (mirrors the listing flow's contract:
|
||||
# never vanish silently).
|
||||
_EMPTY_DISCOVERY_MESSAGE = (
|
||||
"No public TikTok videos found for this query via Google discovery. Try a "
|
||||
"narrower phrasing, a hashtag, or a direct video URL."
|
||||
)
|
||||
|
||||
|
||||
def _resolve_targets(input_model: TikTokScrapeInput) -> list[TikTokTarget]:
|
||||
"""Build the target list from the URL/profile/hashtag sources.
|
||||
|
||||
``searchQueries`` is deliberately excluded: TikTok's own keyword search is
|
||||
login-walled for anonymous sessions, so it is routed through Google video
|
||||
discovery in :func:`iter_tiktok` instead. A raw ``tiktok.com/search?...``
|
||||
URL passed explicitly in ``startUrls``/``postURLs`` still resolves here and
|
||||
keeps its native listing routing.
|
||||
A raw ``tiktok.com/search?...`` URL passed explicitly in
|
||||
``startUrls``/``postURLs`` still resolves here and keeps its native listing
|
||||
routing; there is no keyword-search shortcut.
|
||||
"""
|
||||
targets: list[TikTokTarget] = []
|
||||
for entry in input_model.startUrls:
|
||||
|
|
@ -73,39 +58,6 @@ def _resolve_targets(input_model: TikTokScrapeInput) -> list[TikTokTarget]:
|
|||
return targets
|
||||
|
||||
|
||||
async def _discover_via_google(query: str, *, limit: int) -> list[TikTokTarget]:
|
||||
"""Discover public TikTok video targets via Google ``site:tiktok.com``.
|
||||
|
||||
TikTok's anonymous keyword search is login-walled, so we reuse the existing
|
||||
``google_search`` platform, classify each organic URL with ``resolve_target``,
|
||||
and keep only video hits (``/@user/video/<id>``) — the one kind that scrapes
|
||||
reliably over plain HTTP. Profile/hashtag/search/photo/non-tiktok results are
|
||||
dropped (accounts belong to the ``user_search`` verb). De-duped, capped at
|
||||
``limit``.
|
||||
"""
|
||||
serps = await scrape_serps(
|
||||
GoogleSearchScrapeInput(
|
||||
queries=query, site="tiktok.com", maxPagesPerQuery=1
|
||||
),
|
||||
limit=1,
|
||||
)
|
||||
resolved: list[TikTokTarget] = []
|
||||
seen: set[str] = set()
|
||||
for serp in serps:
|
||||
for org in serp.get("organicResults") or []:
|
||||
url = org.get("url", "") if isinstance(org, dict) else ""
|
||||
target = resolve_target(url)
|
||||
if target is None or target.kind != "video":
|
||||
continue
|
||||
if target.value in seen:
|
||||
continue
|
||||
seen.add(target.value)
|
||||
resolved.append(target)
|
||||
if len(resolved) >= limit:
|
||||
return resolved
|
||||
return resolved
|
||||
|
||||
|
||||
def _dispatch(
|
||||
target: TikTokTarget,
|
||||
*,
|
||||
|
|
@ -128,11 +80,9 @@ async def iter_tiktok(
|
|||
) -> AsyncIterator[dict[str, Any]]:
|
||||
"""Yield normalized items for every resolved target, in order.
|
||||
|
||||
Direct sources (URLs, profiles, hashtags) resolve up front; ``searchQueries``
|
||||
then run through Google video discovery. The video flow's ``fetch_html``
|
||||
opens its own warmed proxy session per call when none is bound; the listing
|
||||
flow drives its own browser. Neither binds a ContextVar across these
|
||||
``yield``s, so the generator stays context-safe.
|
||||
The video flow's ``fetch_html`` opens its own warmed proxy session per call
|
||||
when none is bound; the listing flow drives its own browser. Neither binds a
|
||||
ContextVar across these ``yield``s, so the generator stays context-safe.
|
||||
"""
|
||||
cap = input_model.resultsPerPage
|
||||
for target in _resolve_targets(input_model):
|
||||
|
|
@ -141,27 +91,6 @@ async def iter_tiktok(
|
|||
):
|
||||
yield item
|
||||
|
||||
# searchQueries -> Google-discovered public video URLs, de-duped across
|
||||
# queries so the same video surfacing under two terms is scraped once.
|
||||
seen_videos: set[str] = set()
|
||||
for query in input_model.searchQueries:
|
||||
discovered = await _discover_via_google(query, limit=cap)
|
||||
if not discovered:
|
||||
yield ErrorItem(
|
||||
url=_SEARCH_URL.format(query=quote(query)),
|
||||
input=query,
|
||||
error=_EMPTY_DISCOVERY_MESSAGE,
|
||||
errorCode="no_items",
|
||||
scrapedAt=now_iso(),
|
||||
).to_output()
|
||||
continue
|
||||
for target in discovered:
|
||||
if target.value in seen_videos:
|
||||
continue
|
||||
seen_videos.add(target.value)
|
||||
async for item in iter_video(target, fetch=fetch):
|
||||
yield item
|
||||
|
||||
|
||||
async def scrape_tiktok(
|
||||
input_model: TikTokScrapeInput,
|
||||
|
|
@ -174,7 +103,9 @@ async def scrape_tiktok(
|
|||
from app.capabilities.core.progress import emit_progress
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
async for item in iter_tiktok(input_model, fetch=fetch, fetch_listing=fetch_listing):
|
||||
async for item in iter_tiktok(
|
||||
input_model, fetch=fetch, fetch_listing=fetch_listing
|
||||
):
|
||||
results.append(item)
|
||||
emit_progress("scraping", current=len(results), total=limit, unit="item")
|
||||
if limit is not None and len(results) >= limit:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue