feat: bumped version to 0.0.32

2026-07-16 23:01:06 +02:00 · 2026-07-13 16:29:39 -07:00 · 2026-07-13 16:29:39 -07:00 · 1131da5ed7
commit 1131da5ed7
parent 1bc7d9f51c
55 changed files with 496 additions and 159 deletions
--- a/2
+++ b/2
@ -1 +1 @@
-0.0.31
+0.0.32
--- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md
+++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md
@ -13,7 +13,7 @@ Answer the delegated question from live Instagram data gathered with your verbs,

 <playbook>
 - Known profile/post/reel links: call `instagram_scrape` with the links in `urls` (use `result_type` to pick posts or reels). Hashtag/place URLs are unsupported (login-walled).
- Finding a profile on a topic: call `instagram_scrape` with `search_queries` (resolved to public profiles via Google; `search_type` is profile-only).
+- Finding a profile on a topic: call `instagram_scrape` with `search_queries` (resolved to public profiles via Google; `search_type` is profile-only). Google-backed discovery is slow (~30-60s per query), so start with **at most 3** distinct queries per task and only add more if the first round returns nothing significant — never batch many phrasing variants of the same intent.
 - Profile metadata (follower counts, bio, post count): call `instagram_details`.
 - Batch multiple URLs (or queries) into one call rather than many single-item calls.
 <include snippet="run_reader"/>
--- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md
+++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md
@ -14,11 +14,11 @@ Answer the delegated question from live TikTok data gathered with your verb, com
 </available_tools>

 <playbook>
- Finding videos on a topic: call `tiktok_scrape` with `hashtags` (no leading '#'), or pass a TikTok URL in `urls`.
+- Finding videos on a topic: prefer `tiktok_scrape` with `hashtags` (no leading '#') or a direct TikTok URL in `urls` (fastest). `search_queries` also finds videos on a topic, but it is Google-backed and slow, so start with **at most 3** distinct queries and only add more if the first round returns nothing significant — never batch many phrasing variants of the same intent.
 - Scraping a specific video, profile, hashtag, or search page: pass its TikTok URL in `urls`.
 - Profiles: a creator's `profiles` feed returns the account's metadata (name, followers, bio, verification) reliably, but its video list is often withheld by TikTok — treat an empty video list as a known limit, not a failure to retry endlessly. Prefer `hashtags` or a direct video URL for videos.
 - Comments on a video: call `tiktok_comments` with the video URL(s) in `video_urls`.
- Finding accounts by keyword: call `tiktok_user_search` with `queries`. Keyword search returns no videos, so do not use `search_queries` for videos — use `hashtags` or a video URL.
+- Finding accounts by keyword: call `tiktok_user_search` with `queries` — that is the path for accounts. Use `search_queries` on `tiktok_scrape` only when you want videos, not accounts.
 - "What's trending now": call `tiktok_trending` (no query needed); set `max_items` for how many.
 - Controlling volume: use `max_items` for the total cap and `results_per_page` per target (per-verb equivalents: `comments_per_video`, `results_per_query`).
 - Requested counts: `max_items` defaults low — when the task asks for N items, set `max_items` (and the per-target count) above N. A call that caps below the target can never satisfy it.
--- a/surfsense_backend/app/capabilities/tiktok/scrape/definition.py
+++ b/surfsense_backend/app/capabilities/tiktok/scrape/definition.py
@ -11,7 +11,8 @@ TIKTOK_SCRAPE = Capability(
    name="tiktok.scrape",
    description=(
        "Scrape public TikTok videos. Use urls, profiles, hashtags, or "
-        "search_queries."
+        "search_queries (search_queries are resolved via Google to public "
+        "videos; for accounts by keyword use tiktok.user_search)."
    ),
    input_schema=ScrapeInput,
    output_schema=ScrapeOutput,
--- a/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py
+++ b/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py
@ -43,7 +43,12 @@ class ScrapeInput(BaseModel):
    search_queries: list[str] = Field(
        default_factory=list,
        max_length=MAX_TIKTOK_SOURCES,
-        description="Search terms to run on TikTok.",
+        description=(
+            "Search terms resolved via Google (site:tiktok.com) to public TikTok "
+            "videos, since TikTok's own keyword search is login-walled. Slower "
+            "than hashtags/urls. To find accounts by keyword, use "
+            "tiktok.user_search instead."
+        ),
    )
    results_per_page: int = Field(
        default=10,
--- a/surfsense_backend/app/proprietary/platforms/instagram/scraper.py
+++ b/surfsense_backend/app/proprietary/platforms/instagram/scraper.py
@ -306,9 +306,9 @@ async def _discover_via_google(
    """
    serps = await scrape_serps(
        GoogleSearchScrapeInput(
-            queries=query, site="instagram.com", maxPagesPerQuery=2
+            queries=query, site="instagram.com", maxPagesPerQuery=1
        ),
-        limit=2,
+        limit=1,
    )
    resolved: list[ResolvedUrl] = []
    seen: set[tuple[str, str]] = set()
--- a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py
@ -12,6 +12,9 @@ from collections.abc import AsyncIterator
 from typing import Any
 from urllib.parse import quote

+from app.proprietary.platforms.google_search.schemas import GoogleSearchScrapeInput
+from app.proprietary.platforms.google_search.scraper import scrape_serps
+
 from .extraction.timestamps import now_iso
 from .flows import FetchCommentsFn, FetchFn, FetchListingFn, FetchUsersFn
 from .flows.comments import iter_comments
@ -35,9 +38,24 @@ _HASHTAG_URL = "https://www.tiktok.com/tag/{tag}"
 _SEARCH_URL = "https://www.tiktok.com/search?q={query}"
 _EXPLORE_URL = "https://www.tiktok.com/explore"

+# A ``searchQueries`` term whose Google discovery surfaced no scrapable video
+# URLs degrades to one honest ErrorItem (mirrors the listing flow's contract:
+# never vanish silently).
+_EMPTY_DISCOVERY_MESSAGE = (
+    "No public TikTok videos found for this query via Google discovery. Try a "
+    "narrower phrasing, a hashtag, or a direct video URL."
+)
+

 def _resolve_targets(input_model: TikTokScrapeInput) -> list[TikTokTarget]:
-    """Build the target list from every input source, dropping unresolved URLs."""
+    """Build the target list from the URL/profile/hashtag sources.
+
+    ``searchQueries`` is deliberately excluded: TikTok's own keyword search is
+    login-walled for anonymous sessions, so it is routed through Google video
+    discovery in :func:`iter_tiktok` instead. A raw ``tiktok.com/search?...``
+    URL passed explicitly in ``startUrls``/``postURLs`` still resolves here and
+    keeps its native listing routing.
+    """
    targets: list[TikTokTarget] = []
    for entry in input_model.startUrls:
        resolved = resolve_target(entry.url)
@ -52,13 +70,42 @@ def _resolve_targets(input_model: TikTokScrapeInput) -> list[TikTokTarget]:
        targets.append(TikTokTarget("profile", name, _PROFILE_URL.format(name=name)))
    for tag in input_model.hashtags:
        targets.append(TikTokTarget("hashtag", tag, _HASHTAG_URL.format(tag=tag)))
-    for query in input_model.searchQueries:
-        targets.append(
-            TikTokTarget("search", query, _SEARCH_URL.format(query=quote(query)))
-        )
    return targets


+async def _discover_via_google(query: str, *, limit: int) -> list[TikTokTarget]:
+    """Discover public TikTok video targets via Google ``site:tiktok.com``.
+
+    TikTok's anonymous keyword search is login-walled, so we reuse the existing
+    ``google_search`` platform, classify each organic URL with ``resolve_target``,
+    and keep only video hits (``/@user/video/<id>``) — the one kind that scrapes
+    reliably over plain HTTP. Profile/hashtag/search/photo/non-tiktok results are
+    dropped (accounts belong to the ``user_search`` verb). De-duped, capped at
+    ``limit``.
+    """
+    serps = await scrape_serps(
+        GoogleSearchScrapeInput(
+            queries=query, site="tiktok.com", maxPagesPerQuery=1
+        ),
+        limit=1,
+    )
+    resolved: list[TikTokTarget] = []
+    seen: set[str] = set()
+    for serp in serps:
+        for org in serp.get("organicResults") or []:
+            url = org.get("url", "") if isinstance(org, dict) else ""
+            target = resolve_target(url)
+            if target is None or target.kind != "video":
+                continue
+            if target.value in seen:
+                continue
+            seen.add(target.value)
+            resolved.append(target)
+            if len(resolved) >= limit:
+                return resolved
+    return resolved
+
+
 def _dispatch(
    target: TikTokTarget,
    *,
@ -81,9 +128,11 @@ async def iter_tiktok(
 ) -> AsyncIterator[dict[str, Any]]:
    """Yield normalized items for every resolved target, in order.

-    The video flow's ``fetch_html`` opens its own warmed proxy session per call
-    when none is bound; the listing flow drives its own browser. Neither binds a
-    ContextVar across these ``yield``s, so the generator stays context-safe.
+    Direct sources (URLs, profiles, hashtags) resolve up front; ``searchQueries``
+    then run through Google video discovery. The video flow's ``fetch_html``
+    opens its own warmed proxy session per call when none is bound; the listing
+    flow drives its own browser. Neither binds a ContextVar across these
+    ``yield``s, so the generator stays context-safe.
    """
    cap = input_model.resultsPerPage
    for target in _resolve_targets(input_model):
@ -92,6 +141,27 @@ async def iter_tiktok(
        ):
            yield item

+    # searchQueries -> Google-discovered public video URLs, de-duped across
+    # queries so the same video surfacing under two terms is scraped once.
+    seen_videos: set[str] = set()
+    for query in input_model.searchQueries:
+        discovered = await _discover_via_google(query, limit=cap)
+        if not discovered:
+            yield ErrorItem(
+                url=_SEARCH_URL.format(query=quote(query)),
+                input=query,
+                error=_EMPTY_DISCOVERY_MESSAGE,
+                errorCode="no_items",
+                scrapedAt=now_iso(),
+            ).to_output()
+            continue
+        for target in discovered:
+            if target.value in seen_videos:
+                continue
+            seen_videos.add(target.value)
+            async for item in iter_video(target, fetch=fetch):
+                yield item
+

 async def scrape_tiktok(
    input_model: TikTokScrapeInput,
--- a/surfsense_backend/pyproject.toml
+++ b/surfsense_backend/pyproject.toml
@ -1,6 +1,6 @@
 [project]
 name = "surf-new-backend"
-version = "0.0.31"
+version = "0.0.32"
 description = "SurfSense Backend"
 requires-python = ">=3.12"
 dependencies = [
--- a/surfsense_backend/tests/unit/platforms/tiktok/test_discovery.py
+++ b/surfsense_backend/tests/unit/platforms/tiktok/test_discovery.py
@ -0,0 +1,149 @@
+"""Offline tests for Google-backed TikTok video discovery.
+
+``searchQueries`` are login-walled on TikTok's native search, so they route
+through the ``google_search`` platform (``site:tiktok.com``): each organic URL
+is classified with ``resolve_target`` and only video hits (``/@user/video/<id>``)
+are kept — profiles/hashtags/search/photo/non-tiktok are dropped (accounts
+belong to the user-search verb). These tests inject a fake ``scrape_serps`` so
+there is no network: they pin the classification, cross-query de-dup, the limit
+cap, the barren-query ErrorItem, and that no ``/search?q=`` listing target is
+ever built.
+"""
+
+from __future__ import annotations
+
+import json
+
+from app.proprietary.platforms.tiktok import (
+    TikTokScrapeInput,
+    orchestrator,
+    scrape_tiktok,
+)
+
+
+def _fake_serps(*organic_urls: str):
+    async def _scrape_serps(input_model, *, limit=None):
+        assert input_model.site == "tiktok.com"
+        assert input_model.maxPagesPerQuery == 1
+        return [{"organicResults": [{"url": u} for u in organic_urls]}]
+
+    return _scrape_serps
+
+
+def _video_page(url: str) -> str:
+    """Render a rehydration blob for a ``/@user/video/<id>`` URL."""
+    video_id = url.rsplit("/", 1)[1]
+    username = url.split("@")[1].split("/")[0]
+    blob = {
+        "__DEFAULT_SCOPE__": {
+            "webapp.video-detail": {
+                "itemInfo": {
+                    "itemStruct": {
+                        "id": video_id,
+                        "desc": "hi",
+                        "author": {"uniqueId": username},
+                        "stats": {"diggCount": 1},
+                    }
+                }
+            }
+        }
+    }
+    return (
+        '<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" '
+        f'type="application/json">{json.dumps(blob)}</script>'
+    )
+
+
+async def _fetch_video(url: str) -> str:
+    return _video_page(url)
+
+
+async def test_search_discovery_keeps_only_videos(monkeypatch):
+    # Only the video URL survives; profile / hashtag / search / photo /
+    # non-tiktok organic results are dropped.
+    monkeypatch.setattr(
+        orchestrator,
+        "scrape_serps",
+        _fake_serps(
+            "https://www.tiktok.com/@nasa/video/123",
+            "https://www.tiktok.com/@nasa",
+            "https://www.tiktok.com/tag/space",
+            "https://www.tiktok.com/search?q=space",
+            "https://www.tiktok.com/@nasa/photo/999",
+            "https://example.com/not-tiktok",
+        ),
+    )
+    items = await scrape_tiktok(
+        TikTokScrapeInput(searchQueries=["space"], resultsPerPage=10),
+        fetch=_fetch_video,
+    )
+    assert [i["id"] for i in items] == ["123"]
+
+
+async def test_search_discovery_dedupes_across_queries(monkeypatch):
+    # The same video surfacing under two queries is scraped once.
+    monkeypatch.setattr(
+        orchestrator,
+        "scrape_serps",
+        _fake_serps("https://www.tiktok.com/@nasa/video/123"),
+    )
+    items = await scrape_tiktok(
+        TikTokScrapeInput(searchQueries=["space", "rockets"], resultsPerPage=10),
+        fetch=_fetch_video,
+    )
+    assert [i["id"] for i in items] == ["123"]
+
+
+async def test_search_discovery_respects_per_target_limit(monkeypatch):
+    monkeypatch.setattr(
+        orchestrator,
+        "scrape_serps",
+        _fake_serps(
+            "https://www.tiktok.com/@a/video/1",
+            "https://www.tiktok.com/@b/video/2",
+            "https://www.tiktok.com/@c/video/3",
+        ),
+    )
+    items = await scrape_tiktok(
+        TikTokScrapeInput(searchQueries=["x"], resultsPerPage=2),
+        fetch=_fetch_video,
+    )
+    assert [i["id"] for i in items] == ["1", "2"]
+
+
+async def test_search_barren_query_emits_error_item(monkeypatch):
+    # A query whose discovery finds no video URLs degrades to one ErrorItem.
+    monkeypatch.setattr(
+        orchestrator,
+        "scrape_serps",
+        _fake_serps(
+            "https://www.tiktok.com/@nasa",
+            "https://example.com/x",
+        ),
+    )
+    items = await scrape_tiktok(
+        TikTokScrapeInput(searchQueries=["space"], resultsPerPage=10),
+        fetch=_fetch_video,
+    )
+    assert len(items) == 1
+    assert items[0]["errorCode"] == "no_items"
+    assert items[0]["input"] == "space"
+
+
+async def test_search_never_builds_listing_target(monkeypatch):
+    # searchQueries must never hit the (login-walled) native search listing flow.
+    monkeypatch.setattr(
+        orchestrator,
+        "scrape_serps",
+        _fake_serps("https://www.tiktok.com/@nasa/video/123"),
+    )
+
+    async def _boom_listing(_url: str, _count: int) -> list[dict]:
+        raise AssertionError("searchQueries must not build a listing target")
+
+    items = await scrape_tiktok(
+        TikTokScrapeInput(searchQueries=["space"], resultsPerPage=10),
+        fetch=_fetch_video,
+        fetch_listing=_boom_listing,
+    )
+    assert [i["id"] for i in items] == ["123"]
--- a/surfsense_backend/uv.lock
+++ b/surfsense_backend/uv.lock
@ -36,6 +36,9 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -70,6 +73,9 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version < '3.13' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -104,6 +110,9 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -138,6 +147,9 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -172,6 +184,9 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -206,6 +221,9 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -240,6 +258,9 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -274,6 +295,9 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -308,6 +332,9 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -342,6 +369,9 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -387,6 +417,10 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version >= '3.14' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -421,6 +455,9 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -455,6 +492,9 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version < '3.13' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -489,6 +529,9 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -523,6 +566,9 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -557,6 +603,9 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -591,6 +640,9 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -625,6 +677,9 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -659,6 +714,9 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -693,6 +751,9 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -727,6 +788,9 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -772,6 +836,10 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version >= '3.14' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -806,6 +874,9 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version == '3.13.*' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -840,6 +911,9 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version < '3.13' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -874,6 +948,9 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version >= '3.14' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -908,6 +985,9 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -942,6 +1022,9 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -976,6 +1059,9 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version == '3.13.*' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -1010,6 +1096,9 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -1044,6 +1133,9 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -1078,6 +1170,9 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -1112,6 +1207,9 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version < '3.13' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -1157,6 +1255,10 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -1191,6 +1293,9 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -1258,6 +1363,12 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -1292,6 +1403,9 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -1326,6 +1440,9 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -1393,6 +1510,12 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -1460,6 +1583,12 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version < '3.13' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
    "python_version < '0'",
    "python_version < '0'",
@ -1494,6 +1623,9 @@ resolution-markers = [
    "python_version < '0'",
    "python_version < '0'",
    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
+    "python_version < '0'",
    "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
 ]
 conflicts = [[
@ -10353,7 +10485,7 @@ wheels = [

 [[package]]
 name = "surf-new-backend"
-version = "0.0.31"
+version = "0.0.32"
 source = { editable = "." }
 dependencies = [
    { name = "alembic" },
--- a/surfsense_browser_extension/package.json
+++ b/surfsense_browser_extension/package.json
@ -1,7 +1,7 @@
 {
 	"name": "surfsense_browser_extension",
 	"displayName": "Surfsense Browser Extension",
-	"version": "0.0.31",
+	"version": "0.0.32",
 	"description": "Extension to collect Browsing History for SurfSense.",
 	"author": "https://github.com/MODSetter",
 	"engines": {
--- a/surfsense_desktop/package.json
+++ b/surfsense_desktop/package.json
@ -1,7 +1,7 @@
 {
  "name": "surfsense-desktop",
  "productName": "SurfSense",
-  "version": "0.0.31",
+  "version": "0.0.32",
  "description": "SurfSense Desktop App",
  "main": "dist/main.js",
  "scripts": {
--- a/surfsense_evals/scripts/analyze_failures.py
+++ b/surfsense_evals/scripts/analyze_failures.py
@ -12,12 +12,10 @@ Outputs (printed to stdout + written to `failures_n171.json`):
 from __future__ import annotations

 import json
-import re
 from collections import Counter, defaultdict
 from pathlib import Path
 from typing import Any

-
 REPO = Path(__file__).resolve().parents[1]
 RUN = REPO / "data" / "multimodal_doc" / "runs" / "2026-05-14T00-53-19Z" / "parser_compare"
 RAW = RUN / "raw.jsonl"
--- a/surfsense_evals/scripts/check_uploaded_status.py
+++ b/surfsense_evals/scripts/check_uploaded_status.py
@ -15,7 +15,6 @@ from pathlib import Path
 import httpx
 from dotenv import load_dotenv

-
 REPO = Path(__file__).resolve().parents[1]
 PDF_DIR = REPO / "data" / "multimodal_doc" / "mmlongbench" / "pdfs"

--- a/surfsense_evals/scripts/compute_post_retry_accuracy.py
+++ b/surfsense_evals/scripts/compute_post_retry_accuracy.py
@ -47,9 +47,7 @@ def _row_key(row: dict) -> tuple[str, str]:
 def _is_failure(row: dict) -> bool:
    if row.get("error"):
        return True
-    if not (row.get("raw_text") or "").strip():
-        return True
-    return False
+    return bool(not (row.get("raw_text") or "").strip())


 def _summarise(rows_by_arm: dict[str, list[dict]]) -> dict[str, dict]:
--- a/surfsense_evals/scripts/patch_manifest_for_parallel_ingest.py
+++ b/surfsense_evals/scripts/patch_manifest_for_parallel_ingest.py
@ -27,7 +27,6 @@ from __future__ import annotations
 import json
 from pathlib import Path

-
 REPO = Path(__file__).resolve().parents[1]
 MAP_PATH = REPO / "data" / "multimodal_doc" / "maps" / "mmlongbench_doc_map.jsonl"
 PDF_DIR = REPO / "data" / "multimodal_doc" / "mmlongbench" / "pdfs"
--- a/surfsense_evals/scripts/peek_crag_run.py
+++ b/surfsense_evals/scripts/peek_crag_run.py
@ -10,19 +10,20 @@ from collections import defaultdict
 def main() -> None:
    raw_path = sorted(glob.glob("data/research/runs/*/crag/raw.jsonl"))[-1]
    print(f"Reading: {raw_path}")
-    rows = [json.loads(line) for line in open(raw_path, encoding="utf-8") if line.strip()]
+    with open(raw_path, encoding="utf-8") as fh:
+        rows = [json.loads(line) for line in fh if line.strip()]
    by_q: dict[str, dict[str, dict]] = defaultdict(dict)
    for r in rows:
        by_q[r["qid"]][r["arm"]] = r

    for qid, arms in list(by_q.items()):
        b = arms.get("bare_llm", {})
-        l = arms.get("long_context", {})
+        lc = arms.get("long_context", {})
        s = arms.get("surfsense", {})
        print(f"\n=== {qid} ({b.get('domain')}/{b.get('question_type')}) ===")
        print(f"  question: {b.get('extra', {}).get('question', '?')!r}")
        print(f"  gold: {b.get('gold')!r}")
-        for arm_name, a in (("bare_llm", b), ("long_context", l), ("surfsense", s)):
+        for arm_name, a in (("bare_llm", b), ("long_context", lc), ("surfsense", s)):
            grade = a.get("graded", {})
            text = (a.get("raw_text") or "").strip()
            tail = text[-200:] if text else ""
--- a/surfsense_evals/scripts/peek_disagreements.py
+++ b/surfsense_evals/scripts/peek_disagreements.py
@ -10,7 +10,8 @@ from collections import defaultdict
 def main() -> None:
    raw_path = sorted(glob.glob("data/research/runs/*/crag/raw.jsonl"))[-1]
    print(f"Reading: {raw_path}")
-    rows = [json.loads(line) for line in open(raw_path, encoding="utf-8") if line.strip()]
+    with open(raw_path, encoding="utf-8") as fh:
+        rows = [json.loads(line) for line in fh if line.strip()]
    by_q: dict[str, dict[str, dict]] = defaultdict(dict)
    for r in rows:
        by_q[r["qid"]][r["arm"]] = r
--- a/surfsense_evals/scripts/retry_failed_questions.py
+++ b/surfsense_evals/scripts/retry_failed_questions.py
@ -106,9 +106,7 @@ def _is_failure_row(row: dict[str, Any]) -> bool:

    if row.get("error"):
        return True
-    if not (row.get("raw_text") or "").strip():
-        return True
-    return False
+    return bool(not (row.get("raw_text") or "").strip())


@dataclass
@ -428,7 +426,7 @@ async def _run(args: argparse.Namespace) -> int:

        if f.arm == "native_pdf":
            pdf_path = Path(map_row["pdf_path"])
-            if not pdf_path.exists():
+            if not await asyncio.to_thread(pdf_path.exists):
                logger.error("PDF missing on disk: %s — skipping", pdf_path)
                continue
            request = _build_native_request(
--- a/surfsense_evals/scripts/summarise_crag_run.py
+++ b/surfsense_evals/scripts/summarise_crag_run.py
@ -11,7 +11,8 @@ def main() -> None:
    if not runs:
        print("(no CRAG runs found)")
        return
-    m = json.load(open(runs[-1], encoding="utf-8"))
+    with open(runs[-1], encoding="utf-8") as fh:
+        m = json.load(fh)
    metrics = m["metrics"]

    print(f"Reading: {runs[-1]}")
--- a/surfsense_evals/scripts/summarise_parser_compare_run.py
+++ b/surfsense_evals/scripts/summarise_parser_compare_run.py
@ -16,7 +16,6 @@ import statistics
 from collections import defaultdict
 from pathlib import Path

-
 REPO = Path(__file__).resolve().parents[1]
 RUN_DIR = REPO / "data" / "multimodal_doc" / "runs" / "2026-05-14T00-53-19Z" / "parser_compare"
 RAW = RUN_DIR / "raw.jsonl"
@ -91,7 +90,7 @@ def main() -> None:

    print()
    print("by answer_format (accuracy):")
-    formats = sorted({f for m in arm_metrics.values() for f in m["by_format"].keys()})
+    formats = sorted({f for m in arm_metrics.values() for f in m["by_format"]})
    header = f"{'arm':<25} " + " ".join(f"{f:>10}" for f in formats)
    print(header)
    print("-" * len(header))
--- a/surfsense_evals/src/surfsense_evals/core/cli.py
+++ b/surfsense_evals/src/surfsense_evals/core/cli.py
@ -32,14 +32,13 @@ from __future__ import annotations

 import argparse
 import asyncio
+import contextlib
 import json
 import logging
 import sys
 from dataclasses import dataclass
 from typing import Any

-import sys
-
 import httpx
 from rich.console import Console
 from rich.table import Table
@ -51,10 +50,8 @@ from rich.table import Table
 # Terminal, PowerShell, cmd) all interpret ANSI escapes natively.
 if sys.platform == "win32":
    for _stream in (sys.stdout, sys.stderr):
-        try:
+        with contextlib.suppress(AttributeError, ValueError):
            _stream.reconfigure(encoding="utf-8", errors="replace")
-        except (AttributeError, ValueError):
-            pass

 from . import registry
 from .auth import CredentialError, acquire_token, client_with_auth
--- a/surfsense_evals/src/surfsense_evals/core/clients/documents.py
+++ b/surfsense_evals/src/surfsense_evals/core/clients/documents.py
@ -18,6 +18,7 @@ Document processing is asynchronous:
 from __future__ import annotations

 import asyncio
+import contextlib
 import logging
 import mimetypes
 from collections.abc import Iterable, Sequence
@ -157,10 +158,8 @@ class DocumentsClient:
            )
        finally:
            for _, (_, file_obj, _) in opened:
-                try:
+                with contextlib.suppress(Exception):
                    file_obj.close()
-                except Exception:  # noqa: BLE001
-                    pass

        response.raise_for_status()
        return FileUploadResult.from_payload(response.json())
--- a/surfsense_evals/src/surfsense_evals/core/metrics/comparison.py
+++ b/surfsense_evals/src/surfsense_evals/core/metrics/comparison.py
@ -71,8 +71,8 @@ def mcnemar_test(
            f"Length mismatch: arm_a={len(arm_a_correct)}, arm_b={len(arm_b_correct)}"
        )
    n = len(arm_a_correct)
-    b = sum(1 for a, c in zip(arm_a_correct, arm_b_correct) if a and not c)
-    c = sum(1 for a, cc in zip(arm_a_correct, arm_b_correct) if (not a) and cc)
+    b = sum(1 for a, c in zip(arm_a_correct, arm_b_correct, strict=False) if a and not c)
+    c = sum(1 for a, cc in zip(arm_a_correct, arm_b_correct, strict=False) if (not a) and cc)
    discordant = b + c
    if discordant == 0:
        return McnemarResult(
--- a/surfsense_evals/src/surfsense_evals/core/parse/init.py
+++ b/surfsense_evals/src/surfsense_evals/core/parse/init.py
@ -3,7 +3,7 @@
 from __future__ import annotations

 from .answer_letter import AnswerLetterResult, extract_answer_letter
-from .citations import CITATION_REGEX, CitationToken, ChunkCitation, UrlCitation, parse_citations
+from .citations import CITATION_REGEX, ChunkCitation, CitationToken, UrlCitation, parse_citations
 from .freeform_answer import extract_freeform_answer
 from .sse import SseEvent, iter_sse_events

--- a/surfsense_evals/src/surfsense_evals/core/parse/citations.py
+++ b/surfsense_evals/src/surfsense_evals/core/parse/citations.py
@ -15,7 +15,7 @@ from __future__ import annotations

 import re
 from dataclasses import dataclass
-from typing import Any, Union
+from typing import Any

 # Pattern preserves the TS source verbatim:
 #   /[\[【]\u200B?citation:\s*(https?:\/\/[^\]】\u200B]+|urlcite\d+|(?:doc-)?-?\d+(?:\s*,\s*(?:doc-)?-?\d+)*)\s*\u200B?[\]】]/g
@ -64,7 +64,7 @@ class UrlCitation:
        return {"kind": "url", "url": self.url}


-CitationToken = Union[ChunkCitation, UrlCitation]
+CitationToken = ChunkCitation | UrlCitation


 def parse_citations(text: str, *, url_map: dict[str, str] | None = None) -> list[CitationToken]:
--- a/surfsense_evals/src/surfsense_evals/core/parsers/azure_di.py
+++ b/surfsense_evals/src/surfsense_evals/core/parsers/azure_di.py
@ -25,6 +25,7 @@ import asyncio
 import logging
 import os
 import random
+from pathlib import Path

 logger = logging.getLogger(__name__)

@ -82,7 +83,7 @@ async def parse_with_azure_di(
        ServiceResponseError,
    )

-    file_size_mb = os.path.getsize(file_path) / (1024 * 1024)
+    file_size_mb = await asyncio.to_thread(os.path.getsize, file_path) / (1024 * 1024)
    logger.info(
        "Azure DI parsing %s (mode=%s, model=%s, size=%.1fMB)",
        file_path, processing_mode, model_id, file_size_mb,
@ -96,12 +97,12 @@ async def parse_with_azure_di(
                credential=AzureKeyCredential(api_key),
            )
            async with client:
-                with open(file_path, "rb") as fh:
-                    poller = await client.begin_analyze_document(
-                        model_id,
-                        body=fh,
-                        output_content_format=DocumentContentFormat.MARKDOWN,
-                    )
+                body = await asyncio.to_thread(Path(file_path).read_bytes)
+                poller = await client.begin_analyze_document(
+                    model_id,
+                    body=body,
+                    output_content_format=DocumentContentFormat.MARKDOWN,
+                )
                result = await poller.result()
            content = (result.content or "").strip()
            if not content:
--- a/surfsense_evals/src/surfsense_evals/core/parsers/llamacloud.py
+++ b/surfsense_evals/src/surfsense_evals/core/parsers/llamacloud.py
@ -98,7 +98,7 @@ async def parse_with_llamacloud(
    from llama_cloud_services.parse.base import JobFailedException
    from llama_cloud_services.parse.utils import ResultType

-    file_size_mb = os.path.getsize(file_path) / (1024 * 1024)
+    file_size_mb = await asyncio.to_thread(os.path.getsize, file_path) / (1024 * 1024)
    # Match backend's per-page timeout heuristic so big PDFs don't drop
    # mid-job: 60s baseline + 30s/page (premium agent runs longer than
    # basic; both fit comfortably here).
--- a/surfsense_evals/src/surfsense_evals/core/providers/openrouter_chat.py
+++ b/surfsense_evals/src/surfsense_evals/core/providers/openrouter_chat.py
@ -29,8 +29,8 @@ from typing import Any
 import httpx

 from .openrouter_pdf import (
-    OpenRouterResponse,
    _DEFAULT_HEADERS,
+    OpenRouterResponse,
    _parse_chat_completion,
 )

--- a/surfsense_evals/src/surfsense_evals/core/providers/openrouter_pdf.py
+++ b/surfsense_evals/src/surfsense_evals/core/providers/openrouter_pdf.py
@ -34,7 +34,7 @@ import base64
 import logging
 import time
 from dataclasses import dataclass
-from enum import Enum
+from enum import StrEnum
 from pathlib import Path
 from typing import Any

@ -43,7 +43,7 @@ import httpx
 logger = logging.getLogger(__name__)


-class PdfEngine(str, Enum):
+class PdfEngine(StrEnum):
    NATIVE = "native"
    MISTRAL_OCR = "mistral-ocr"
    CLOUDFLARE_AI = "cloudflare-ai"
--- a/surfsense_evals/src/surfsense_evals/suites/init.py
+++ b/surfsense_evals/src/surfsense_evals/suites/init.py
@ -20,7 +20,7 @@ from __future__ import annotations
 import importlib
 import logging
 import pkgutil
-from typing import Iterable
+from collections.abc import Iterable

 logger = logging.getLogger(__name__)

--- a/surfsense_evals/src/surfsense_evals/suites/_demo/hello/init.py
+++ b/surfsense_evals/src/surfsense_evals/suites/_demo/hello/init.py
@ -6,7 +6,6 @@ import argparse
 from typing import Any

 from ....core.registry import (
-    Benchmark,
    ReportSection,
    RunArtifact,
    RunContext,
--- a/surfsense_evals/src/surfsense_evals/suites/medical/cure/init.py
+++ b/surfsense_evals/src/surfsense_evals/suites/medical/cure/init.py
@ -12,7 +12,7 @@ Recall@k / MRR / nDCG@10 against qrels.

 from __future__ import annotations

-from .runner import CureBenchmark
 from ....core import registry as _registry
+from .runner import CureBenchmark

 _registry.register(CureBenchmark())
--- a/surfsense_evals/src/surfsense_evals/suites/medical/cure/ingest.py
+++ b/surfsense_evals/src/surfsense_evals/suites/medical/cure/ingest.py
@ -227,12 +227,10 @@ async def run_ingest(


 def _take(it: Iterable, n: int) -> Iterable:
-    yielded = 0
-    for x in it:
-        if yielded >= n:
+    for i, x in enumerate(it):
+        if i >= n:
            return
        yield x
-        yielded += 1


 __all__ = ["DISCIPLINES", "CorpusPassage", "PassageBatch", "run_ingest"]
--- a/surfsense_evals/src/surfsense_evals/suites/medical/cure/runner.py
+++ b/surfsense_evals/src/surfsense_evals/suites/medical/cure/runner.py
@ -34,7 +34,6 @@ from ....core.ingest_settings import (
 )
 from ....core.metrics.retrieval import score_run
 from ....core.registry import (
-    Benchmark,
    ReportSection,
    RunArtifact,
    RunContext,
@ -276,7 +275,7 @@ class CureBenchmark:
        )

        per_query_retrieved: dict[str, list[str]] = {}
-        for q, res in zip(queries, results):
+        for q, res in zip(queries, results, strict=False):
            chunk_ids: list[int] = []
            seen: set[int] = set()
            for citation in res.citations:
@ -311,7 +310,7 @@ class CureBenchmark:
        run_dir = ctx.runs_dir(run_timestamp=run_timestamp)
        raw_path = run_dir / "raw.jsonl"
        with raw_path.open("w", encoding="utf-8") as fh:
-            for q, res in zip(queries, results):
+            for q, res in zip(queries, results, strict=False):
                fh.write(
                    json.dumps(
                        {
--- a/surfsense_evals/src/surfsense_evals/suites/medical/mirage/init.py
+++ b/surfsense_evals/src/surfsense_evals/suites/medical/mirage/init.py
@ -11,7 +11,7 @@ document — the corpus is millions of biomedical snippets.

 from __future__ import annotations

-from .runner import MirageBenchmark
 from ....core import registry as _registry
+from .runner import MirageBenchmark

 _registry.register(MirageBenchmark())
--- a/surfsense_evals/src/surfsense_evals/suites/medical/mirage/ingest.py
+++ b/surfsense_evals/src/surfsense_evals/suites/medical/mirage/ingest.py
@ -93,6 +93,25 @@ class SnippetRow:
 # ---------------------------------------------------------------------------


+def _reuse_cached_dest(dest: Path, *, expect_zip: bool, label: str) -> Path | None:
+    """Return ``dest`` if a usable cache hit, else ``None`` (and delete corrupt zips)."""
+
+    if not dest.exists():
+        return None
+    if expect_zip and not _is_valid_zip(dest):
+        logger.warning(
+            "Cached %s at %s failed ZIP validation (size=%d B); deleting "
+            "and re-downloading.",
+            label,
+            dest,
+            dest.stat().st_size,
+        )
+        dest.unlink(missing_ok=True)
+        return None
+    logger.info("Using cached %s at %s", label, dest)
+    return dest
+
+
 async def _fetch_to_path(
    url: str,
    *,
@ -127,19 +146,9 @@ async def _fetch_to_path(
      surprise-grabbing 16 GB on a slow link.
    """

-    if dest.exists():
-        if expect_zip and not _is_valid_zip(dest):
-            logger.warning(
-                "Cached %s at %s failed ZIP validation (size=%d B); deleting "
-                "and re-downloading.",
-                label,
-                dest,
-                dest.stat().st_size,
-            )
-            dest.unlink(missing_ok=True)
-        else:
-            logger.info("Using cached %s at %s", label, dest)
-            return dest
+    cached = _reuse_cached_dest(dest, expect_zip=expect_zip, label=label)
+    if cached is not None:
+        return cached

    dest.parent.mkdir(parents=True, exist_ok=True)
    partial = dest.with_suffix(dest.suffix + ".partial")
@ -170,39 +179,38 @@ async def _fetch_to_path(
            async with httpx.AsyncClient(
                timeout=httpx.Timeout(timeout_s, connect=20.0),
                follow_redirects=True,
-            ) as client:
-                async with client.stream("GET", url, headers=headers) as response:
-                    if existing_bytes and response.status_code == 200:
-                        logger.warning(
-                            "Server ignored Range header for %s; restarting from 0.",
-                            label,
-                        )
-                        partial.unlink(missing_ok=True)
-                        existing_bytes = 0
-                    elif response.status_code == 416:
-                        # Range not satisfiable — the .partial is at or
-                        # past the end. Treat as "already downloaded";
-                        # validate by closing and re-opening for atomic
-                        # rename below.
-                        logger.info(
-                            "Server reports %s already complete (HTTP 416).",
-                            label,
-                        )
-                    elif response.status_code not in (200, 206):
-                        response.raise_for_status()
+            ) as client, client.stream("GET", url, headers=headers) as response:
+                if existing_bytes and response.status_code == 200:
+                    logger.warning(
+                        "Server ignored Range header for %s; restarting from 0.",
+                        label,
+                    )
+                    partial.unlink(missing_ok=True)
+                    existing_bytes = 0
+                elif response.status_code == 416:
+                    # Range not satisfiable — the .partial is at or
+                    # past the end. Treat as "already downloaded";
+                    # validate by closing and re-opening for atomic
+                    # rename below.
+                    logger.info(
+                        "Server reports %s already complete (HTTP 416).",
+                        label,
+                    )
+                elif response.status_code not in (200, 206):
+                    response.raise_for_status()

-                    total_size = _planned_total_size(response, existing_bytes)
-                    if (
-                        total_size is not None
-                        and total_size > _LARGE_DOWNLOAD_BYTES
-                        and not allow_large_download
-                    ):
-                        raise _LargeDownloadAbort(label, total_size)
+                total_size = _planned_total_size(response, existing_bytes)
+                if (
+                    total_size is not None
+                    and total_size > _LARGE_DOWNLOAD_BYTES
+                    and not allow_large_download
+                ):
+                    raise _LargeDownloadAbort(label, total_size)

-                    mode = "ab" if existing_bytes else "wb"
-                    with partial.open(mode) as fh:
-                        async for chunk in response.aiter_bytes(chunk_size=1 << 18):
-                            fh.write(chunk)
+                mode = "ab" if existing_bytes else "wb"
+                with partial.open(mode) as fh:
+                    async for chunk in response.aiter_bytes(chunk_size=1 << 18):
+                        fh.write(chunk)
            # Optional content sanity check before promoting to dest.
            if expect_zip and not _is_valid_zip(partial):
                raise zipfile.BadZipFile(
--- a/surfsense_evals/src/surfsense_evals/suites/medical/mirage/prompt.py
+++ b/surfsense_evals/src/surfsense_evals/suites/medical/mirage/prompt.py
@ -8,7 +8,6 @@ from __future__ import annotations

 from collections.abc import Mapping

-
 _PROMPT_TEMPLATE = """\
 You are a helpful medical expert. Answer the following multiple-choice
 question using the relevant medical knowledge available to you (and any
--- a/surfsense_evals/src/surfsense_evals/suites/medical/mirage/runner.py
+++ b/surfsense_evals/src/surfsense_evals/suites/medical/mirage/runner.py
@ -29,7 +29,6 @@ from ....core.ingest_settings import (
 )
 from ....core.metrics.mc_accuracy import accuracy_with_wilson_ci, macro_accuracy
 from ....core.registry import (
-    Benchmark,
    ReportSection,
    RunArtifact,
    RunContext,
@ -229,7 +228,7 @@ class MirageBenchmark:
        run_dir = ctx.runs_dir(run_timestamp=run_timestamp)
        raw_path = run_dir / "raw.jsonl"
        with raw_path.open("w", encoding="utf-8") as fh:
-            for q, res in zip(questions, results):
+            for q, res in zip(questions, results, strict=False):
                fh.write(
                    json.dumps(
                        {
@ -246,7 +245,7 @@ class MirageBenchmark:
        for task in tasks:
            n_correct = 0
            n_total = 0
-            for q, res in zip(questions, results):
+            for q, res in zip(questions, results, strict=False):
                if q.task != task:
                    continue
                n_total += 1
--- a/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/parser_compare/ingest.py
+++ b/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/parser_compare/ingest.py
@ -41,8 +41,6 @@ from typing import Any

 from ....core.config import set_suite_state
 from ....core.parsers import (
-    AzureDIError,
-    LlamaCloudError,
    count_pdf_pages,
    parse_with_azure_di,
    parse_with_llamacloud,
@ -131,7 +129,7 @@ async def _run_one_extraction(
    else:
        raise ValueError(f"Unknown parser {parser!r}")
    out_path.parent.mkdir(parents=True, exist_ok=True)
-    out_path.write_text(markdown, encoding="utf-8")
+    await asyncio.to_thread(out_path.write_text, markdown, encoding="utf-8")
    return markdown, time.monotonic() - started


--- a/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/parser_compare/runner.py
+++ b/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/parser_compare/runner.py
@ -603,8 +603,8 @@ class ParserCompareBenchmark:
            f"engine: `{extra.get('pdf_engine', 'native')}`)."
        )
        body.append(
-            f"- Preprocess tariff: basic = $1 / 1k pages, "
-            f"premium = $10 / 1k pages."
+            "- Preprocess tariff: basic = $1 / 1k pages, "
+            "premium = $10 / 1k pages."
        )
        body.append("")
        body.append("### Per-arm summary")
--- a/surfsense_evals/src/surfsense_evals/suites/research/crag/html_extract.py
+++ b/surfsense_evals/src/surfsense_evals/suites/research/crag/html_extract.py
@ -177,10 +177,7 @@ def extract_main_content(
            # Prefer trafilatura output even if short, but only if it
            # contained any prose at all — empty trafilatura fall-through
            # to the stripped form.
-            if body and stripped and len(stripped) > len(body) * 1.5:
-                body = stripped
-                method = "fallback_strip"
-            elif not body and stripped:
+            if body and stripped and len(stripped) > len(body) * 1.5 or not body and stripped:
                body = stripped
                method = "fallback_strip"
            elif body:
--- a/surfsense_evals/src/surfsense_evals/suites/research/crag/prompt.py
+++ b/surfsense_evals/src/surfsense_evals/suites/research/crag/prompt.py
@ -28,7 +28,6 @@ in the runner, so the format is mandatory.

 from __future__ import annotations

-
 _BASE_INSTRUCTIONS = (
    "You are a careful question-answering assistant. The question is a "
    "real-world factual question that may be about finance, music, "
--- a/surfsense_evals/src/surfsense_evals/suites/research/crag/runner.py
+++ b/surfsense_evals/src/surfsense_evals/suites/research/crag/runner.py
@ -830,11 +830,11 @@ def _per_facet_truthfulness(
    """Bucket truthfulness scores by ``key_fn(q)``."""

    buckets: dict[str, dict[str, list[CragGradeResult]]] = {}
-    for q, b, l, s in zip(questions, bare_grades, lc_grades, surf_grades, strict=False):
+    for q, b, lc, s in zip(questions, bare_grades, lc_grades, surf_grades, strict=False):
        key = key_fn(q)
        bucket = buckets.setdefault(key, {"bare_llm": [], "long_context": [], "surfsense": []})
        bucket["bare_llm"].append(b)
-        bucket["long_context"].append(l)
+        bucket["long_context"].append(lc)
        bucket["surfsense"].append(s)
    out: dict[str, Any] = {}
    for key, arms in buckets.items():
--- a/surfsense_evals/src/surfsense_evals/suites/research/frames/dataset.py
+++ b/surfsense_evals/src/surfsense_evals/suites/research/frames/dataset.py
@ -102,7 +102,7 @@ def _parse_wiki_links(raw: Any) -> list[str]:
    except (SyntaxError, ValueError):
        # Fall back: maybe it's a comma-separated string with no quotes.
        return [tok.strip() for tok in text.strip("[]").split(",") if tok.strip()]
-    if isinstance(parsed, (list, tuple)):
+    if isinstance(parsed, list | tuple):
        return [str(x).strip() for x in parsed if str(x).strip()]
    return [str(parsed).strip()]

--- a/surfsense_evals/src/surfsense_evals/suites/research/frames/ingest.py
+++ b/surfsense_evals/src/surfsense_evals/suites/research/frames/ingest.py
@ -34,7 +34,6 @@ filename (without extension), so we round-trip via

 from __future__ import annotations

-import asyncio
 import json
 import logging
 from dataclasses import dataclass
--- a/surfsense_evals/src/surfsense_evals/suites/research/frames/prompt.py
+++ b/surfsense_evals/src/surfsense_evals/suites/research/frames/prompt.py
@ -17,7 +17,6 @@ Format expectations (mirrors the FRAMES paper, section 4):

 from __future__ import annotations

-
 _BASE_INSTRUCTIONS = (
    "You are a careful question-answering assistant. The question may "
    "require combining facts from multiple sources, doing arithmetic, "
--- a/surfsense_evals/tests/suites/test_crag_dataset.py
+++ b/surfsense_evals/tests/suites/test_crag_dataset.py
@ -11,8 +11,6 @@ import bz2
 import json
 from pathlib import Path

-import pytest
-
 from surfsense_evals.suites.research.crag.dataset import (
    CragPage,
    CragQuestion,
--- a/surfsense_evals/tests/suites/test_crag_grader.py
+++ b/surfsense_evals/tests/suites/test_crag_grader.py
@ -7,8 +7,6 @@ exercise the deterministic shortcut + the special-case routing for

 from __future__ import annotations

-import pytest
-
 from surfsense_evals.suites.research.crag.grader import (
    CragGradeResult,
    _flags_false_premise,
--- a/surfsense_evals/tests/suites/test_crag_html_extract.py
+++ b/surfsense_evals/tests/suites/test_crag_html_extract.py
@ -11,13 +11,10 @@ We don't network-fetch trafilatura; we just verify the wrapper:

 from __future__ import annotations

-import pytest
-
 from surfsense_evals.suites.research.crag.html_extract import (
    extract_main_content,
 )

-
 _RICH_HTML = """\
 <!DOCTYPE html>
 <html>
--- a/surfsense_evals/tests/suites/test_frames_dataset.py
+++ b/surfsense_evals/tests/suites/test_frames_dataset.py
@ -16,8 +16,6 @@ from __future__ import annotations
 import textwrap
 from pathlib import Path

-import pytest
-
 from surfsense_evals.suites.research.frames.dataset import (
    FramesQuestion,
    _parse_reasoning_types,
@ -25,7 +23,6 @@ from surfsense_evals.suites.research.frames.dataset import (
    load_questions,
 )

-
 # ---------------------------------------------------------------------------
 # Pure-function tests
 # ---------------------------------------------------------------------------
--- a/surfsense_evals/tests/suites/test_frames_grader.py
+++ b/surfsense_evals/tests/suites/test_frames_grader.py
@ -8,10 +8,7 @@ runner knows to consult the judge.

 from __future__ import annotations

-import pytest
-
 from surfsense_evals.suites.research.frames.grader import (
-    GradeResult,
    _maybe_number,
    _normalise,
    _whole_word_substring,
--- a/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py
+++ b/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py
@ -41,7 +41,9 @@ def register(
            list[str] | None,
            Field(
                description="Terms to discover public profiles for (resolved via "
-                "Google). Provide search_queries OR urls."
+                "Google). Google-backed discovery is slow (~30-60s per query), so "
+                "start with at most 3 queries and expand only if nothing "
+                "significant is found. Provide search_queries OR urls."
            ),
        ] = None,
        search_type: Annotated[
@ -100,7 +102,9 @@ def register(
        search_queries: Annotated[
            list[str] | None,
            Field(
-                description="Terms to discover public profiles for. Provide "
+                description="Terms to discover public profiles for. Google-backed "
+                "discovery is slow (~30-60s per query), so start with at most 3 "
+                "queries and expand only if nothing significant is found. Provide "
                "search_queries OR urls."
            ),
        ] = None,
--- a/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py
+++ b/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py
@ -52,9 +52,11 @@ def register(
        search_queries: Annotated[
            list[str] | None,
            Field(
-                description="Keyword search terms. Returns no videos — use "
-                "hashtags/profiles/urls for videos, or the user-search tool for "
-                "accounts."
+                description="Keyword search terms, resolved via Google to public "
+                "TikTok videos (TikTok's own keyword search is login-walled). "
+                "Slower than hashtags/urls, so start with at most 3 queries and "
+                "expand only if nothing significant is found. For accounts by "
+                "keyword use surfsense_tiktok_user_search."
            ),
        ] = None,
        results_per_page: Annotated[
@ -67,13 +69,15 @@ def register(
        workspace: WorkspaceParam = None,
        response_format: ResponseFormatParam = "markdown",
    ) -> str:
-        """Scrape public TikTok videos by hashtag, profile, or URL.
+        """Scrape public TikTok videos by hashtag, profile, URL, or keyword.

        Use for TikTok video research — a creator's videos, a hashtag feed, or a
        specific video/profile/hashtag URL — instead of a generic web search.
-        Returns videos with text, author, stats, music, and the web URL. For
-        accounts by keyword use the user-search tool; keyword search returns no
-        videos. Example: hashtags=['food'], max_items=20.
+        search_queries also finds videos on a topic (resolved via Google), but is
+        slower: start with at most 3 queries and expand only if nothing
+        significant is found. Returns videos with text, author, stats, music, and
+        the web URL. For accounts by keyword use surfsense_tiktok_user_search.
+        Example: hashtags=['food'], max_items=20.
        """
        return await run_scraper(
            client,
--- a/surfsense_web/package.json
+++ b/surfsense_web/package.json
@ -1,6 +1,6 @@
 {
 	"name": "surfsense_web",
-	"version": "0.0.31",
+	"version": "0.0.32",
 	"private": true,
 	"packageManager": "pnpm@10.26.0",
 	"description": "SurfSense Frontend",
 @ -1 +1 @@
 .0.31
 .0.32