apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-18 23:11:12 +02:00
Code Issues Projects Releases Packages Wiki Activity

refactor: streamline TikTok and Instagram scraping logic by removing search_queries and enhancing documentation for clarity

Browse source
This commit is contained in:
DESKTOP-RTLN3BA\$punk 2026-07-13 17:11:25 -07:00
parent e8b3692b54
commit 2b018c4474
111 changed files with 1800 additions and 1580 deletions
Download patch file Download diff file Expand all files Collapse all files

4
surfsense_backend/app/capabilities/instagram/details/schemas.py
View file

@ -54,9 +54,7 @@ class DetailsInput(BaseModel):
@model_validator(mode="after")
def _exactly_one_source(self) -> DetailsInput:
if not self.urls and not self.search_queries:
raise ValueError(
"Provide at least one of 'urls' or 'search_queries'."
)
raise ValueError("Provide at least one of 'urls' or 'search_queries'.")
if self.urls and self.search_queries:
raise ValueError(
"Provide 'urls' OR 'search_queries', not both (they cannot be combined)."

4
surfsense_backend/app/capabilities/instagram/scrape/schemas.py
View file

@ -77,9 +77,7 @@ class ScrapeInput(BaseModel):
@model_validator(mode="after")
def _exactly_one_source(self) -> ScrapeInput:
if not self.urls and not self.search_queries:
raise ValueError(
"Provide at least one of 'urls' or 'search_queries'."
)
raise ValueError("Provide at least one of 'urls' or 'search_queries'.")
if self.urls and self.search_queries:
raise ValueError(
"Provide 'urls' OR 'search_queries', not both (they cannot be combined)."

5
surfsense_backend/app/capabilities/tiktok/scrape/definition.py
View file

@ -10,9 +10,8 @@ from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput
TIKTOK_SCRAPE = Capability(
name="tiktok.scrape",
description=(
"Scrape public TikTok videos. Use urls, profiles, hashtags, or "
"search_queries (search_queries are resolved via Google to public "
"videos; for accounts by keyword use tiktok.user_search)."
"Scrape public TikTok videos. Use urls, profiles, or hashtags. To find "
"accounts by keyword, use tiktok.user_search."
),
input_schema=ScrapeInput,
output_schema=ScrapeOutput,

1
surfsense_backend/app/capabilities/tiktok/scrape/executor.py
View file

@ -26,7 +26,6 @@ def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor:
startUrls=[{"url": url} for url in payload.urls],
profiles=payload.profiles,
hashtags=payload.hashtags,
searchQueries=payload.search_queries,
resultsPerPage=payload.results_per_page,
)
emit_progress(

24
surfsense_backend/app/capabilities/tiktok/scrape/schemas.py
View file

@ -4,7 +4,8 @@ A lean, agent-friendly surface over ``TikTokScrapeInput``
(``app/proprietary/platforms/tiktok``). The executor maps this to the full
scraper input; the scraper's ``TikTokVideoItem`` is reused verbatim as the
output element. Any TikTok URL kind (video, profile, hashtag, search) goes in
``urls``; ``profiles``/``hashtags``/``search_queries`` are typed shortcuts.
``urls``; ``profiles``/``hashtags`` are typed shortcuts. Keyword search is not a
video source here — use ``tiktok.user_search`` to find accounts by keyword.
"""
from __future__ import annotations
@ -26,8 +27,8 @@ class ScrapeInput(BaseModel):
max_length=MAX_TIKTOK_SOURCES,
description=(
"TikTok URLs to scrape: a video, a profile (/@<user>), a hashtag "
"(/tag/<name>), or a search URL. Provide these OR profiles/hashtags/"
"search_queries (at least one source is required)."
"(/tag/<name>), or a search URL. Provide these OR profiles/hashtags "
"(at least one source is required)."
),
)
profiles: list[str] = Field(
@ -40,21 +41,11 @@ class ScrapeInput(BaseModel):
max_length=MAX_TIKTOK_SOURCES,
description="Hashtag names to scrape, without the leading '#'.",
)
search_queries: list[str] = Field(
default_factory=list,
max_length=MAX_TIKTOK_SOURCES,
description=(
"Search terms resolved via Google (site:tiktok.com) to public TikTok "
"videos, since TikTok's own keyword search is login-walled. Slower "
"than hashtags/urls. To find accounts by keyword, use "
"tiktok.user_search instead."
),
)
results_per_page: int = Field(
default=10,
ge=1,
le=MAX_TIKTOK_ITEMS,
description="Max videos to pull per profile/hashtag/search target.",
description="Max videos to pull per profile/hashtag target.",
)
max_items: int = Field(
default=10,
@ -65,10 +56,9 @@ class ScrapeInput(BaseModel):
@model_validator(mode="after")
def _require_a_source(self) -> ScrapeInput:
if not any((self.urls, self.profiles, self.hashtags, self.search_queries)):
if not any((self.urls, self.profiles, self.hashtags)):
raise ValueError(
"Provide at least one of 'urls', 'profiles', 'hashtags', or "
"'search_queries'."
"Provide at least one of 'urls', 'profiles', or 'hashtags'."
)
return self