feat: bumped version to 0.0.32

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md

@@ -13,7 +13,7 @@ Answer the delegated question from live Instagram data gathered with your verbs,
 
 <playbook>
 - Known profile/post/reel links: call `instagram_scrape` with the links in `urls` (use `result_type` to pick posts or reels). Hashtag/place URLs are unsupported (login-walled).
-- Finding a profile on a topic: call `instagram_scrape` with `search_queries` (resolved to public profiles via Google; `search_type` is profile-only).
+- Finding a profile on a topic: call `instagram_scrape` with `search_queries` (resolved to public profiles via Google; `search_type` is profile-only). Google-backed discovery is slow (~30-60s per query), so start with **at most 3** distinct queries per task and only add more if the first round returns nothing significant — never batch many phrasing variants of the same intent.
 - Profile metadata (follower counts, bio, post count): call `instagram_details`.
 - Batch multiple URLs (or queries) into one call rather than many single-item calls.
 <include snippet="run_reader"/>

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md

@@ -14,11 +14,11 @@ Answer the delegated question from live TikTok data gathered with your verb, com
 </available_tools>
 
 <playbook>
-- Finding videos on a topic: call `tiktok_scrape` with `hashtags` (no leading '#'), or pass a TikTok URL in `urls`.
+- Finding videos on a topic: prefer `tiktok_scrape` with `hashtags` (no leading '#') or a direct TikTok URL in `urls` (fastest). `search_queries` also finds videos on a topic, but it is Google-backed and slow, so start with **at most 3** distinct queries and only add more if the first round returns nothing significant — never batch many phrasing variants of the same intent.
 - Scraping a specific video, profile, hashtag, or search page: pass its TikTok URL in `urls`.
 - Profiles: a creator's `profiles` feed returns the account's metadata (name, followers, bio, verification) reliably, but its video list is often withheld by TikTok — treat an empty video list as a known limit, not a failure to retry endlessly. Prefer `hashtags` or a direct video URL for videos.
 - Comments on a video: call `tiktok_comments` with the video URL(s) in `video_urls`.
-- Finding accounts by keyword: call `tiktok_user_search` with `queries`. Keyword search returns no videos, so do not use `search_queries` for videos — use `hashtags` or a video URL.
+- Finding accounts by keyword: call `tiktok_user_search` with `queries` — that is the path for accounts. Use `search_queries` on `tiktok_scrape` only when you want videos, not accounts.
 - "What's trending now": call `tiktok_trending` (no query needed); set `max_items` for how many.
 - Controlling volume: use `max_items` for the total cap and `results_per_page` per target (per-verb equivalents: `comments_per_video`, `results_per_query`).
 - Requested counts: `max_items` defaults low — when the task asks for N items, set `max_items` (and the per-target count) above N. A call that caps below the target can never satisfy it.

surfsense_backend/app/capabilities/tiktok/scrape/definition.py

@@ -11,7 +11,8 @@ TIKTOK_SCRAPE = Capability(
     name="tiktok.scrape",
     description=(
         "Scrape public TikTok videos. Use urls, profiles, hashtags, or "
-        "search_queries."
+        "search_queries (search_queries are resolved via Google to public "
+        "videos; for accounts by keyword use tiktok.user_search)."
     ),
     input_schema=ScrapeInput,
     output_schema=ScrapeOutput,

surfsense_backend/app/capabilities/tiktok/scrape/schemas.py

@@ -43,7 +43,12 @@ class ScrapeInput(BaseModel):
     search_queries: list[str] = Field(
         default_factory=list,
         max_length=MAX_TIKTOK_SOURCES,
-        description="Search terms to run on TikTok.",
+        description=(
+            "Search terms resolved via Google (site:tiktok.com) to public TikTok "
+            "videos, since TikTok's own keyword search is login-walled. Slower "
+            "than hashtags/urls. To find accounts by keyword, use "
+            "tiktok.user_search instead."
+        ),
     )
     results_per_page: int = Field(
         default=10,

surfsense_backend/app/proprietary/platforms/instagram/scraper.py

@@ -306,9 +306,9 @@ async def _discover_via_google(
     """
     serps = await scrape_serps(
         GoogleSearchScrapeInput(
-            queries=query, site="instagram.com", maxPagesPerQuery=2
+            queries=query, site="instagram.com", maxPagesPerQuery=1
         ),
-        limit=2,
+        limit=1,
     )
     resolved: list[ResolvedUrl] = []
     seen: set[tuple[str, str]] = set()

surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py

@@ -12,6 +12,9 @@ from collections.abc import AsyncIterator
 from typing import Any
 from urllib.parse import quote
 
+from app.proprietary.platforms.google_search.schemas import GoogleSearchScrapeInput
+from app.proprietary.platforms.google_search.scraper import scrape_serps
+
 from .extraction.timestamps import now_iso
 from .flows import FetchCommentsFn, FetchFn, FetchListingFn, FetchUsersFn
 from .flows.comments import iter_comments
@@ -35,9 +38,24 @@ _HASHTAG_URL = "https://www.tiktok.com/tag/{tag}"
 _SEARCH_URL = "https://www.tiktok.com/search?q={query}"
 _EXPLORE_URL = "https://www.tiktok.com/explore"
 
+# A ``searchQueries`` term whose Google discovery surfaced no scrapable video
+# URLs degrades to one honest ErrorItem (mirrors the listing flow's contract:
+# never vanish silently).
+_EMPTY_DISCOVERY_MESSAGE = (
+    "No public TikTok videos found for this query via Google discovery. Try a "
+    "narrower phrasing, a hashtag, or a direct video URL."
+)
+
 
 def _resolve_targets(input_model: TikTokScrapeInput) -> list[TikTokTarget]:
-    """Build the target list from every input source, dropping unresolved URLs."""
+    """Build the target list from the URL/profile/hashtag sources.
+
+    ``searchQueries`` is deliberately excluded: TikTok's own keyword search is
+    login-walled for anonymous sessions, so it is routed through Google video
+    discovery in :func:`iter_tiktok` instead. A raw ``tiktok.com/search?...``
+    URL passed explicitly in ``startUrls``/``postURLs`` still resolves here and
+    keeps its native listing routing.
+    """
     targets: list[TikTokTarget] = []
     for entry in input_model.startUrls:
         resolved = resolve_target(entry.url)
@@ -52,13 +70,42 @@ def _resolve_targets(input_model: TikTokScrapeInput) -> list[TikTokTarget]:
         targets.append(TikTokTarget("profile", name, _PROFILE_URL.format(name=name)))
     for tag in input_model.hashtags:
         targets.append(TikTokTarget("hashtag", tag, _HASHTAG_URL.format(tag=tag)))
-    for query in input_model.searchQueries:
-        targets.append(
-            TikTokTarget("search", query, _SEARCH_URL.format(query=quote(query)))
-        )
     return targets
 
 
+async def _discover_via_google(query: str, *, limit: int) -> list[TikTokTarget]:
+    """Discover public TikTok video targets via Google ``site:tiktok.com``.
+
+    TikTok's anonymous keyword search is login-walled, so we reuse the existing
+    ``google_search`` platform, classify each organic URL with ``resolve_target``,
+    and keep only video hits (``/@user/video/<id>``) — the one kind that scrapes
+    reliably over plain HTTP. Profile/hashtag/search/photo/non-tiktok results are
+    dropped (accounts belong to the ``user_search`` verb). De-duped, capped at
+    ``limit``.
+    """
+    serps = await scrape_serps(
+        GoogleSearchScrapeInput(
+            queries=query, site="tiktok.com", maxPagesPerQuery=1
+        ),
+        limit=1,
+    )
+    resolved: list[TikTokTarget] = []
+    seen: set[str] = set()
+    for serp in serps:
+        for org in serp.get("organicResults") or []:
+            url = org.get("url", "") if isinstance(org, dict) else ""
+            target = resolve_target(url)
+            if target is None or target.kind != "video":
+                continue
+            if target.value in seen:
+                continue
+            seen.add(target.value)
+            resolved.append(target)
+            if len(resolved) >= limit:
+                return resolved
+    return resolved
+
+
 def _dispatch(
     target: TikTokTarget,
     *,
@@ -81,9 +128,11 @@ async def iter_tiktok(
 ) -> AsyncIterator[dict[str, Any]]:
     """Yield normalized items for every resolved target, in order.
 
-    The video flow's ``fetch_html`` opens its own warmed proxy session per call
-    when none is bound; the listing flow drives its own browser. Neither binds a
-    ContextVar across these ``yield``s, so the generator stays context-safe.
+    Direct sources (URLs, profiles, hashtags) resolve up front; ``searchQueries``
+    then run through Google video discovery. The video flow's ``fetch_html``
+    opens its own warmed proxy session per call when none is bound; the listing
+    flow drives its own browser. Neither binds a ContextVar across these
+    ``yield``s, so the generator stays context-safe.
     """
     cap = input_model.resultsPerPage
     for target in _resolve_targets(input_model):
@@ -92,6 +141,27 @@ async def iter_tiktok(
         ):
             yield item
 
+    # searchQueries -> Google-discovered public video URLs, de-duped across
+    # queries so the same video surfacing under two terms is scraped once.
+    seen_videos: set[str] = set()
+    for query in input_model.searchQueries:
+        discovered = await _discover_via_google(query, limit=cap)
+        if not discovered:
+            yield ErrorItem(
+                url=_SEARCH_URL.format(query=quote(query)),
+                input=query,
+                error=_EMPTY_DISCOVERY_MESSAGE,
+                errorCode="no_items",
+                scrapedAt=now_iso(),
+            ).to_output()
+            continue
+        for target in discovered:
+            if target.value in seen_videos:
+                continue
+            seen_videos.add(target.value)
+            async for item in iter_video(target, fetch=fetch):
+                yield item
+
 
 async def scrape_tiktok(
     input_model: TikTokScrapeInput,