mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-16 23:01:06 +02:00
feat(tiktok): emit graceful ErrorItem for blocked/empty listings
Profile and search feeds are trust-gated: an anonymous headless session gets an empty item_list (profile) or no results XHR (search), while hashtag feeds load. A zero-item listing now yields one honest ErrorItem (errorCode="no_items") instead of vanishing silently, and ErrorItems are excluded from billing so a blocked target is surfaced but never charged.
This commit is contained in:
parent
f731d371ab
commit
ba375ea3ee
5 changed files with 96 additions and 14 deletions
|
|
@ -82,5 +82,6 @@ class ScrapeOutput(BaseModel):
|
|||
|
||||
@property
|
||||
def billable_units(self) -> int:
|
||||
"""One returned item = one billable unit."""
|
||||
return len(self.items)
|
||||
"""One returned video = one billable unit; ErrorItems (``errorCode`` set,
|
||||
for blocked/empty targets) are surfaced but never charged."""
|
||||
return sum(1 for item in self.items if not getattr(item, "errorCode", None))
|
||||
|
|
|
|||
|
|
@ -12,9 +12,19 @@ from typing import Any
|
|||
|
||||
from ..extraction import parse_video
|
||||
from ..extraction.timestamps import now_iso
|
||||
from ..schemas import ErrorItem
|
||||
from ..targets.types import TikTokTarget
|
||||
from . import FetchListingFn
|
||||
|
||||
# Profile and search feeds are trust-gated: an anonymous headless session gets an
|
||||
# empty body (profile) or no results XHR (search), while hashtag feeds load. We
|
||||
# can't tell "genuinely empty" from "blocked" here, so a zero-item listing emits
|
||||
# one honest ErrorItem instead of vanishing silently.
|
||||
_EMPTY_LISTING_MESSAGE = (
|
||||
"No videos returned. The target may be empty/private/removed, or TikTok "
|
||||
"withheld this feed from anonymous access (common for profiles and search)."
|
||||
)
|
||||
|
||||
|
||||
async def iter_listing(
|
||||
target: TikTokTarget, *, cap: int, fetch_listing: FetchListingFn
|
||||
|
|
@ -35,3 +45,11 @@ async def iter_listing(
|
|||
emitted += 1
|
||||
if emitted >= cap:
|
||||
return
|
||||
if emitted == 0:
|
||||
yield ErrorItem(
|
||||
url=target.url,
|
||||
input=target.value,
|
||||
error=_EMPTY_LISTING_MESSAGE,
|
||||
errorCode="no_items",
|
||||
scrapedAt=now_iso(),
|
||||
).to_output()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue