Merge remote-tracking branch 'upstream/dev' into feat/instagram-scraper

surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py

@@ -38,6 +38,7 @@ _EXPECTED_SUBAGENTS = frozenset(
         "memory",
         "onedrive",
         "reddit",
+        "tiktok",
         "web_crawler",
         "youtube",
     }

surfsense_backend/tests/unit/capabilities/access/test_agent_tools.py

@@ -78,8 +78,14 @@ async def test_registry_becomes_one_tool_per_verb_plus_readers(isolate):
     tools = isolate.module.build_capability_tools(workspace_id=7, capabilities=caps)
 
     by_name = {t.name: t for t in tools}
-    # One tool per verb, plus the two shared run-reader tools.
-    assert set(by_name) == {"web_scrape", "web_discover", "read_run", "search_run"}
+    # One tool per verb, plus the shared run-reader tools.
+    assert set(by_name) == {
+        "web_scrape",
+        "web_discover",
+        "read_run",
+        "search_run",
+        "export_run",
+    }
     assert by_name["web_scrape"].description == "web.scrape does a thing."
     assert by_name["web_scrape"].args_schema is _EchoInput
 

surfsense_backend/tests/unit/capabilities/tiktok/comments/test_executor.py

@@ -0,0 +1,48 @@
+"""``tiktok.comments`` executor: verb input → scraper args → typed comment items.
+
+Boundary mocked: the proprietary comments actor (injected fake). NOT mocked: the
+verb's own payload→args forwarding and the dict→CommentItem wrapping.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from app.capabilities.tiktok.comments.executor import build_comments_executor
+from app.capabilities.tiktok.comments.schemas import CommentsInput, CommentsOutput
+
+pytestmark = pytest.mark.unit
+
+_VIDEO = "https://www.tiktok.com/@bob/video/123"
+
+
+class _FakeComments:
+    """Records the urls + kwargs it was called with; returns canned items."""
+
+    def __init__(self, items: list[dict]):
+        self._items = items
+        self.calls: list[tuple[list[str], int, int | None]] = []
+
+    async def __call__(
+        self, video_urls: list[str], *, per_video: int, limit: int | None = None
+    ) -> list[dict]:
+        self.calls.append((video_urls, per_video, limit))
+        return self._items
+
+
+async def test_forwards_urls_and_limits_and_wraps_items():
+    comments = _FakeComments([{"id": "1", "text": "hi"}])
+    execute = build_comments_executor(comments_fn=comments)
+
+    out = await execute(
+        CommentsInput(video_urls=[_VIDEO], comments_per_video=15, max_items=40)
+    )
+
+    assert isinstance(out, CommentsOutput)
+    assert len(out.items) == 1
+    assert out.items[0].text == "hi"
+
+    (urls, per_video, limit) = comments.calls[0]
+    assert urls == [_VIDEO]
+    assert per_video == 15
+    assert limit == 40

surfsense_backend/tests/unit/capabilities/tiktok/comments/test_schemas.py

@@ -0,0 +1,48 @@
+"""``tiktok.comments`` input guards and billing: a video URL is required, bounded,
+and ErrorItems are surfaced free."""
+
+from __future__ import annotations
+
+import pytest
+from pydantic import ValidationError
+
+from app.capabilities.tiktok.comments.schemas import CommentsInput, CommentsOutput
+from app.capabilities.tiktok.scrape.schemas import MAX_TIKTOK_ITEMS, MAX_TIKTOK_SOURCES
+
+pytestmark = pytest.mark.unit
+
+_VIDEO = "https://www.tiktok.com/@bob/video/123"
+
+
+def test_rejects_input_with_no_url():
+    with pytest.raises(ValidationError):
+        CommentsInput(video_urls=[])
+
+
+def test_defaults_and_bounds():
+    payload = CommentsInput(video_urls=[_VIDEO])
+    assert payload.max_items == 20
+    assert payload.comments_per_video == 20
+    assert payload.estimated_units == 20
+    with pytest.raises(ValidationError):
+        CommentsInput(video_urls=[_VIDEO], max_items=0)
+    with pytest.raises(ValidationError):
+        CommentsInput(video_urls=[_VIDEO], max_items=MAX_TIKTOK_ITEMS + 1)
+
+
+def test_rejects_more_urls_than_the_cap():
+    too_many = [f"{_VIDEO}{i}" for i in range(MAX_TIKTOK_SOURCES + 1)]
+    with pytest.raises(ValidationError):
+        CommentsInput(video_urls=too_many)
+
+
+def test_error_items_are_not_billed():
+    # Real comments count; ErrorItems (bad URL / empty video) are surfaced free.
+    out = CommentsOutput(
+        items=[
+            {"id": "1", "text": "hi"},
+            {"errorCode": "no_comments", "input": "123", "error": "empty"},
+        ]
+    )
+    assert len(out.items) == 2
+    assert out.billable_units == 1

surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_executor.py

@@ -0,0 +1,79 @@
+"""``tiktok.scrape`` executor: verb input → actor input mapping → typed items.
+
+Boundary mocked: the proprietary scraper (injected fake). NOT mocked: the verb's
+own payload→TikTokScrapeInput mapping and the dict→TikTokVideoItem wrapping.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from app.capabilities.tiktok.scrape.executor import build_scrape_executor
+from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput
+from app.exceptions import ForbiddenError
+from app.proprietary.platforms.tiktok import TikTokAccessBlockedError, TikTokScrapeInput
+
+pytestmark = pytest.mark.unit
+
+
+class _FakeScraper:
+    """Records the actor input + limit it was called with; returns canned items."""
+
+    def __init__(self, items: list[dict]):
+        self._items = items
+        self.calls: list[tuple[TikTokScrapeInput, int | None]] = []
+
+    async def __call__(
+        self, actor_input: TikTokScrapeInput, *, limit: int | None = None
+    ) -> list[dict]:
+        self.calls.append((actor_input, limit))
+        return self._items
+
+
+async def test_maps_urls_to_start_urls_and_wraps_items():
+    scraper = _FakeScraper([{"id": "123", "text": "hello"}])
+    execute = build_scrape_executor(scrape_fn=scraper)
+
+    out = await execute(ScrapeInput(urls=["https://www.tiktok.com/@nasa/video/123"]))
+
+    assert isinstance(out, ScrapeOutput)
+    assert len(out.items) == 1
+    assert out.items[0].id == "123"
+
+    (actor_input, _limit) = scraper.calls[0]
+    assert [u.url for u in actor_input.startUrls] == [
+        "https://www.tiktok.com/@nasa/video/123"
+    ]
+
+
+async def test_forwards_typed_sources_and_limit():
+    scraper = _FakeScraper([])
+    execute = build_scrape_executor(scrape_fn=scraper)
+
+    await execute(
+        ScrapeInput(
+            profiles=["nasa"],
+            hashtags=["food"],
+            search_queries=["cats"],
+            results_per_page=7,
+            max_items=25,
+        )
+    )
+
+    (actor_input, limit) = scraper.calls[0]
+    assert actor_input.profiles == ["nasa"]
+    assert actor_input.hashtags == ["food"]
+    assert actor_input.searchQueries == ["cats"]
+    assert actor_input.resultsPerPage == 7
+    # The outer collection limit is the caller's total-item cap.
+    assert limit == 25
+
+
+async def test_access_blocked_maps_to_forbidden():
+    async def _blocked(actor_input: TikTokScrapeInput, *, limit: int | None = None):
+        raise TikTokAccessBlockedError("all IPs refused")
+
+    execute = build_scrape_executor(scrape_fn=_blocked)
+
+    with pytest.raises(ForbiddenError):
+        await execute(ScrapeInput(hashtags=["food"]))

surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_schemas.py

@@ -0,0 +1,58 @@
+"""``tiktok.scrape`` input guards: a source is required and the batch is bounded."""
+
+from __future__ import annotations
+
+import pytest
+from pydantic import ValidationError
+
+from app.capabilities.tiktok.scrape.schemas import (
+    MAX_TIKTOK_ITEMS,
+    MAX_TIKTOK_SOURCES,
+    ScrapeInput,
+    ScrapeOutput,
+)
+
+pytestmark = pytest.mark.unit
+
+
+def test_rejects_input_with_no_source():
+    with pytest.raises(ValidationError):
+        ScrapeInput()
+
+
+def test_accepts_urls_only():
+    payload = ScrapeInput(urls=["https://www.tiktok.com/@nasa"])
+    assert payload.profiles == []
+
+
+def test_accepts_hashtags_only():
+    payload = ScrapeInput(hashtags=["food"])
+    assert payload.hashtags == ["food"]
+
+
+def test_defaults_and_bounds():
+    payload = ScrapeInput(hashtags=["food"])
+    assert payload.max_items == 10
+    assert payload.results_per_page == 10
+    with pytest.raises(ValidationError):
+        ScrapeInput(hashtags=["food"], max_items=0)
+    with pytest.raises(ValidationError):
+        ScrapeInput(hashtags=["food"], max_items=MAX_TIKTOK_ITEMS + 1)
+
+
+def test_rejects_more_sources_than_the_cap():
+    too_many = [f"tag{i}" for i in range(MAX_TIKTOK_SOURCES + 1)]
+    with pytest.raises(ValidationError):
+        ScrapeInput(hashtags=too_many)
+
+
+def test_error_items_are_not_billed():
+    # Real videos count; ErrorItems (blocked/empty targets) are surfaced free.
+    out = ScrapeOutput(
+        items=[
+            {"id": "1", "webVideoUrl": "https://tiktok.com/@a/video/1"},
+            {"errorCode": "no_items", "input": "nasa", "error": "blocked"},
+        ]
+    )
+    assert len(out.items) == 2
+    assert out.billable_units == 1

surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py

@@ -0,0 +1,57 @@
+"""The tiktok namespace registers its verb as one Capability the doors/agent read."""
+
+from __future__ import annotations
+
+import pytest
+
+from app.capabilities import (
+    tiktok,  # noqa: F401  — importing the namespace registers its verbs
+)
+from app.capabilities.core import BillingUnit
+from app.capabilities.core.store import get_capability
+from app.capabilities.tiktok.comments.schemas import CommentsInput, CommentsOutput
+from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput
+from app.capabilities.tiktok.trending.schemas import TrendingInput, TrendingOutput
+from app.capabilities.tiktok.user_search.schemas import (
+    UserSearchInput,
+    UserSearchOutput,
+)
+
+pytestmark = pytest.mark.unit
+
+
+def test_tiktok_scrape_is_registered_and_billed_per_video():
+    cap = get_capability("tiktok.scrape")
+
+    assert cap.name == "tiktok.scrape"
+    assert cap.input_schema is ScrapeInput
+    assert cap.output_schema is ScrapeOutput
+    assert cap.billing_unit is BillingUnit.TIKTOK_VIDEO
+
+
+def test_tiktok_user_search_is_registered_and_billed_per_profile():
+    cap = get_capability("tiktok.user_search")
+
+    assert cap.name == "tiktok.user_search"
+    assert cap.input_schema is UserSearchInput
+    assert cap.output_schema is UserSearchOutput
+    assert cap.billing_unit is BillingUnit.TIKTOK_USER
+
+
+def test_tiktok_comments_is_registered_and_billed_per_comment():
+    cap = get_capability("tiktok.comments")
+
+    assert cap.name == "tiktok.comments"
+    assert cap.input_schema is CommentsInput
+    assert cap.output_schema is CommentsOutput
+    assert cap.billing_unit is BillingUnit.TIKTOK_COMMENT
+
+
+def test_tiktok_trending_is_registered_and_billed_per_video():
+    cap = get_capability("tiktok.trending")
+
+    assert cap.name == "tiktok.trending"
+    assert cap.input_schema is TrendingInput
+    assert cap.output_schema is TrendingOutput
+    # Trending returns videos, so it shares the per-video meter.
+    assert cap.billing_unit is BillingUnit.TIKTOK_VIDEO

surfsense_backend/tests/unit/capabilities/tiktok/trending/test_executor.py

@@ -0,0 +1,38 @@
+"""``tiktok.trending`` executor: verb input → scraper count → typed video items.
+
+Boundary mocked: the proprietary trending actor (injected fake). NOT mocked: the
+verb's own count forwarding and the dict→TikTokVideoItem wrapping.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from app.capabilities.tiktok.trending.executor import build_trending_executor
+from app.capabilities.tiktok.trending.schemas import TrendingInput, TrendingOutput
+
+pytestmark = pytest.mark.unit
+
+
+class _FakeTrending:
+    """Records the count it was called with; returns canned items."""
+
+    def __init__(self, items: list[dict]):
+        self._items = items
+        self.calls: list[int] = []
+
+    async def __call__(self, *, count: int) -> list[dict]:
+        self.calls.append(count)
+        return self._items
+
+
+async def test_forwards_count_and_wraps_items():
+    trending = _FakeTrending([{"id": "1", "text": "viral"}])
+    execute = build_trending_executor(trending_fn=trending)
+
+    out = await execute(TrendingInput(max_items=30))
+
+    assert isinstance(out, TrendingOutput)
+    assert len(out.items) == 1
+    assert out.items[0].id == "1"
+    assert trending.calls == [30]

surfsense_backend/tests/unit/capabilities/tiktok/trending/test_schemas.py

@@ -0,0 +1,33 @@
+"""``tiktok.trending`` input guards and billing: bounded count, ErrorItems free."""
+
+from __future__ import annotations
+
+import pytest
+from pydantic import ValidationError
+
+from app.capabilities.tiktok.scrape.schemas import MAX_TIKTOK_ITEMS
+from app.capabilities.tiktok.trending.schemas import TrendingInput, TrendingOutput
+
+pytestmark = pytest.mark.unit
+
+
+def test_defaults_and_bounds():
+    payload = TrendingInput()
+    assert payload.max_items == 20
+    assert payload.estimated_units == 20
+    with pytest.raises(ValidationError):
+        TrendingInput(max_items=0)
+    with pytest.raises(ValidationError):
+        TrendingInput(max_items=MAX_TIKTOK_ITEMS + 1)
+
+
+def test_error_items_are_not_billed():
+    # Real videos count; an ErrorItem (empty/withheld feed) is surfaced free.
+    out = TrendingOutput(
+        items=[
+            {"id": "1", "webVideoUrl": "https://tiktok.com/@a/video/1"},
+            {"errorCode": "no_items", "input": "explore", "error": "empty"},
+        ]
+    )
+    assert len(out.items) == 2
+    assert out.billable_units == 1

surfsense_backend/tests/unit/capabilities/tiktok/user_search/test_executor.py

@@ -0,0 +1,49 @@
+"""``tiktok.user_search`` executor: verb input → search args → typed profile items.
+
+Boundary mocked: the proprietary search actor (injected fake). NOT mocked: the
+verb's own payload→args forwarding and the dict→TikTokProfileItem wrapping.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from app.capabilities.tiktok.user_search.executor import build_user_search_executor
+from app.capabilities.tiktok.user_search.schemas import (
+    UserSearchInput,
+    UserSearchOutput,
+)
+
+pytestmark = pytest.mark.unit
+
+
+class _FakeSearch:
+    """Records the queries + kwargs it was called with; returns canned items."""
+
+    def __init__(self, items: list[dict]):
+        self._items = items
+        self.calls: list[tuple[list[str], int, int | None]] = []
+
+    async def __call__(
+        self, queries: list[str], *, per_query: int, limit: int | None = None
+    ) -> list[dict]:
+        self.calls.append((queries, per_query, limit))
+        return self._items
+
+
+async def test_forwards_queries_and_limits_and_wraps_items():
+    search = _FakeSearch([{"id": "1", "name": "nasa"}])
+    execute = build_user_search_executor(search_fn=search)
+
+    out = await execute(
+        UserSearchInput(queries=["nasa"], results_per_query=7, max_items=25)
+    )
+
+    assert isinstance(out, UserSearchOutput)
+    assert len(out.items) == 1
+    assert out.items[0].name == "nasa"
+
+    (queries, per_query, limit) = search.calls[0]
+    assert queries == ["nasa"]
+    assert per_query == 7
+    assert limit == 25

surfsense_backend/tests/unit/capabilities/tiktok/user_search/test_schemas.py

@@ -0,0 +1,49 @@
+"""``tiktok.user_search`` input guards and billing: a query is required, bounded,
+and ErrorItems are surfaced free."""
+
+from __future__ import annotations
+
+import pytest
+from pydantic import ValidationError
+
+from app.capabilities.tiktok.scrape.schemas import MAX_TIKTOK_ITEMS, MAX_TIKTOK_SOURCES
+from app.capabilities.tiktok.user_search.schemas import (
+    UserSearchInput,
+    UserSearchOutput,
+)
+
+pytestmark = pytest.mark.unit
+
+
+def test_rejects_input_with_no_query():
+    with pytest.raises(ValidationError):
+        UserSearchInput(queries=[])
+
+
+def test_defaults_and_bounds():
+    payload = UserSearchInput(queries=["nasa"])
+    assert payload.max_items == 10
+    assert payload.results_per_query == 10
+    assert payload.estimated_units == 10
+    with pytest.raises(ValidationError):
+        UserSearchInput(queries=["nasa"], max_items=0)
+    with pytest.raises(ValidationError):
+        UserSearchInput(queries=["nasa"], max_items=MAX_TIKTOK_ITEMS + 1)
+
+
+def test_rejects_more_queries_than_the_cap():
+    too_many = [f"q{i}" for i in range(MAX_TIKTOK_SOURCES + 1)]
+    with pytest.raises(ValidationError):
+        UserSearchInput(queries=too_many)
+
+
+def test_error_items_are_not_billed():
+    # Real accounts count; ErrorItems (empty/withheld queries) are surfaced free.
+    out = UserSearchOutput(
+        items=[
+            {"id": "1", "name": "nasa"},
+            {"errorCode": "no_users", "input": "ghost", "error": "empty"},
+        ]
+    )
+    assert len(out.items) == 2
+    assert out.billable_units == 1

surfsense_backend/tests/unit/platforms/tiktok/test_comments.py

@@ -0,0 +1,89 @@
+"""Comments orchestration over a fake fetch (no network).
+
+Drives ``scrape_tiktok_comments``: video URLs -> captured raw comments -> items.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from app.proprietary.platforms.tiktok import scrape_tiktok_comments
+
+_VIDEO = "https://www.tiktok.com/@bob/video/123"
+
+
+def _comment(cid: str, reply_id: str = "0") -> dict[str, Any]:
+    return {
+        "cid": cid,
+        "text": f"comment {cid}",
+        "digg_count": 7,
+        "reply_comment_total": 2,
+        "create_time": 1700000000,
+        "reply_id": reply_id,
+        "user": {
+            "uid": "u1",
+            "unique_id": "alice",
+            "nickname": "Alice",
+            "avatar_thumb": {"url_list": ["https://cdn/a.webp"]},
+        },
+    }
+
+
+async def test_comments_parse_dedupe_and_cap():
+    async def fake_fetch(_url: str, _cap: int) -> list[dict]:
+        return [_comment("1"), _comment("1"), _comment("2", reply_id="1")]
+
+    items = await scrape_tiktok_comments(
+        [_VIDEO], per_video=2, fetch_comments_fn=fake_fetch
+    )
+
+    assert [i["id"] for i in items] == ["1", "2"]
+    first = items[0]
+    assert first["text"] == "comment 1"
+    assert first["videoWebUrl"] == _VIDEO
+    assert first["diggCount"] == 7
+    assert first["uniqueId"] == "alice"
+    assert first["avatar"] == "https://cdn/a.webp"
+    assert first["createTimeISO"] is not None
+    assert first["repliesToId"] is None  # reply_id "0" == top-level
+    assert first["scrapedAt"] is not None
+    assert items[1]["repliesToId"] == "1"  # a reply carries its parent id
+
+
+async def test_empty_video_emits_error_item():
+    async def fake_fetch(_url: str, _cap: int) -> list[dict]:
+        return []
+
+    items = await scrape_tiktok_comments(
+        [_VIDEO], per_video=5, fetch_comments_fn=fake_fetch
+    )
+
+    assert len(items) == 1
+    assert items[0]["errorCode"] == "no_comments"
+    assert items[0]["input"] == "123"
+
+
+async def test_non_video_url_emits_bad_url_error():
+    async def fake_fetch(_url: str, _cap: int) -> list[dict]:
+        raise AssertionError("should not fetch for a non-video URL")
+
+    items = await scrape_tiktok_comments(
+        ["https://www.tiktok.com/@bob"], per_video=5, fetch_comments_fn=fake_fetch
+    )
+
+    assert len(items) == 1
+    assert items[0]["errorCode"] == "bad_url"
+
+
+async def test_comments_honor_limit_across_videos():
+    async def fake_fetch(_url: str, _cap: int) -> list[dict]:
+        return [_comment("1"), _comment("2")]
+
+    items = await scrape_tiktok_comments(
+        [_VIDEO, "https://www.tiktok.com/@bob/video/456"],
+        per_video=5,
+        limit=3,
+        fetch_comments_fn=fake_fetch,
+    )
+
+    assert len(items) == 3

surfsense_backend/tests/unit/platforms/tiktok/test_fetch_resilience.py

@@ -0,0 +1,135 @@
+"""Fetch-seam resilience for the TikTok scraper (no network, fake sessions).
+
+Fake sessions drive the cookie warm-up + rotate-on-block + backoff branches
+deterministically; a live first IP normally warms and returns 200s.
+"""
+
+from __future__ import annotations
+
+from app.proprietary.platforms.tiktok.session import (
+    TikTokAccessBlockedError,
+    client,
+)
+from app.proprietary.platforms.tiktok.session.proxy import _current_session
+
+_HTML = "<html><body>ok</body></html>"
+
+
+class _FakePage:
+    def __init__(self, status: int, *, cookies: dict | None = None, body: str = _HTML):
+        self.status = status
+        self.cookies = cookies or {}
+        self.body = body
+
+    @property
+    def text(self) -> str:
+        return self.body
+
+
+class _FakeSession:
+    """One 'IP': homepage warm mints ``ttwid`` per flag; page GETs return ``status``."""
+
+    def __init__(self, status: int = 200, *, warms: bool = True, body: str = _HTML):
+        self.status = status
+        self.warms = warms
+        self.body = body
+        self.page_calls = 0
+        self.warm_calls = 0
+
+    async def get(self, url, headers=None, cookies=None):
+        if url.rstrip("/") == "https://www.tiktok.com":
+            self.warm_calls += 1
+            return _FakePage(200, cookies={"ttwid": "x"} if self.warms else {})
+        self.page_calls += 1
+        return _FakePage(self.status, body=self.body)
+
+
+class _FakeHolder:
+    def __init__(self, sessions: list[_FakeSession]) -> None:
+        self._sessions = sessions
+        self.session = sessions[0]
+        self.rotations = 0
+        self.warmed = False
+
+    async def rotate(self):
+        self.rotations += 1
+        self.session = self._sessions[min(self.rotations, len(self._sessions) - 1)]
+        self.warmed = False
+        return self.session
+
+    async def pace(self) -> None:
+        return None
+
+    async def close(self) -> None:
+        return None
+
+
+def _no_sleep(monkeypatch) -> None:
+    async def _noop(_seconds):
+        return None
+
+    monkeypatch.setattr(client.asyncio, "sleep", _noop)
+
+
+async def test_warms_then_returns_html():
+    holder = _FakeHolder([_FakeSession(200, warms=True)])
+    token = _current_session.set(holder)
+    try:
+        result = await client.fetch_html("https://www.tiktok.com/@scout2015")
+    finally:
+        _current_session.reset(token)
+    assert result == _HTML
+    assert holder.rotations == 0
+    assert holder.session.warm_calls == 1
+
+
+async def test_rotates_when_warm_fails_then_succeeds():
+    holder = _FakeHolder([_FakeSession(200, warms=False), _FakeSession(200, warms=True)])
+    token = _current_session.set(holder)
+    try:
+        result = await client.fetch_html("https://www.tiktok.com/@scout2015")
+    finally:
+        _current_session.reset(token)
+    assert result == _HTML
+    assert holder.rotations == 1
+
+
+async def test_404_returns_none_without_rotating():
+    holder = _FakeHolder([_FakeSession(404), _FakeSession(200)])
+    token = _current_session.set(holder)
+    try:
+        result = await client.fetch_html("https://www.tiktok.com/@missing")
+    finally:
+        _current_session.reset(token)
+    assert result is None
+    assert holder.rotations == 0
+
+
+async def test_rotates_and_rewarms_on_403():
+    holder = _FakeHolder([_FakeSession(403), _FakeSession(200, warms=True)])
+    token = _current_session.set(holder)
+    try:
+        result = await client.fetch_html("https://www.tiktok.com/@scout2015")
+    finally:
+        _current_session.reset(token)
+    assert result == _HTML
+    assert holder.rotations == 1
+    assert holder.session.warm_calls == 1
+
+
+async def test_persistent_403_raises_blocked(monkeypatch):
+    _no_sleep(monkeypatch)
+    holder = _FakeHolder(
+        [_FakeSession(403) for _ in range(client._MAX_ROTATIONS + 1)]
+    )
+    token = _current_session.set(holder)
+    try:
+        raised = False
+        try:
+            await client.fetch_html("https://www.tiktok.com/@scout2015")
+        except TikTokAccessBlockedError:
+            raised = True
+    finally:
+        _current_session.reset(token)
+    assert raised
+    assert holder.rotations == client._MAX_ROTATIONS

surfsense_backend/tests/unit/platforms/tiktok/test_hydration.py

@@ -0,0 +1,32 @@
+"""Rehydration-blob extraction from TikTok page HTML (pure, no network)."""
+
+from __future__ import annotations
+
+from app.proprietary.platforms.tiktok.extraction import extract_rehydration_data
+
+_BLOB = (
+    '<html><body><script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" '
+    'type="application/json">'
+    '{"__DEFAULT_SCOPE__":{"webapp.video-detail":{"itemInfo":{"itemStruct":'
+    '{"id":"123"}}}}}'
+    "</script></body></html>"
+)
+
+
+def test_extracts_default_scope_from_blob():
+    data = extract_rehydration_data(_BLOB)
+    assert data is not None
+    item = data["__DEFAULT_SCOPE__"]["webapp.video-detail"]["itemInfo"]["itemStruct"]
+    assert item["id"] == "123"
+
+
+def test_returns_none_when_blob_absent():
+    assert extract_rehydration_data("<html><body>no blob here</body></html>") is None
+
+
+def test_returns_none_when_blob_json_malformed():
+    broken = (
+        '<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" '
+        'type="application/json">{not json}</script>'
+    )
+    assert extract_rehydration_data(broken) is None

surfsense_backend/tests/unit/platforms/tiktok/test_input.py

@@ -0,0 +1,26 @@
+"""Input surface for the TikTok scraper (anonymous, Apify-shaped)."""
+
+from __future__ import annotations
+
+from app.proprietary.platforms.tiktok.schemas import TikTokScrapeInput
+
+
+def test_input_has_no_auth_fields():
+    forbidden = {"username", "password", "token", "login", "auth", "credentials"}
+    assert forbidden.isdisjoint(TikTokScrapeInput.model_fields)
+
+
+def test_input_defaults():
+    model = TikTokScrapeInput()
+    assert model.resultsPerPage == 1
+    assert model.profileSorting == "latest"
+    assert model.proxyCountryCode == "None"
+    assert model.hashtags == []
+    assert model.profiles == []
+    assert model.searchQueries == []
+    assert model.postURLs == []
+
+
+def test_input_allows_extra_inert_fields():
+    model = TikTokScrapeInput(shouldDownloadVideos=True, videoKvStoreIdOrName="x")
+    assert model.model_dump().get("shouldDownloadVideos") is True

surfsense_backend/tests/unit/platforms/tiktok/test_item_list.py

@@ -0,0 +1,25 @@
+"""Pulling item structs out of captured item_list / search API responses."""
+
+from __future__ import annotations
+
+from app.proprietary.platforms.tiktok.extraction import items_from_response
+
+
+def test_reads_item_list_shape():
+    body = {"itemList": [{"id": "1"}, {"id": "2"}], "hasMore": True}
+    assert items_from_response(body) == [{"id": "1"}, {"id": "2"}]
+
+
+def test_reads_search_data_shape():
+    body = {"data": [{"type": 1, "item": {"id": "9"}}, {"type": 4, "item": {}}]}
+    assert items_from_response(body) == [{"id": "9"}, {}]
+
+
+def test_skips_malformed_entries():
+    body = {"data": [{"type": 1}, "junk", {"item": {"id": "7"}}]}
+    assert items_from_response(body) == [{"id": "7"}]
+
+
+def test_returns_empty_for_unrelated_json():
+    assert items_from_response({"statusCode": 0}) == []
+    assert items_from_response("nope") == []

surfsense_backend/tests/unit/platforms/tiktok/test_listing_retry.py

@@ -0,0 +1,60 @@
+"""Retry-on-empty for the browser ``item_list`` seam (no browser, fake fetch).
+
+``fetch_item_list`` re-fetches an empty capture up to ``TIKTOK_LISTING_MAX_ATTEMPTS``
+so a flagged rotating exit IP on the first draw doesn't collapse straight to an
+``ErrorItem``. These drive that loop deterministically by faking ``_fetch_sync``.
+"""
+
+from __future__ import annotations
+
+from app.proprietary.platforms.tiktok.session import listing
+
+
+class _Fake:
+    """Returns each queued result once, repeating the last; counts calls."""
+
+    def __init__(self, results: list[list[dict]]):
+        self.results = results
+        self.calls = 0
+
+    def __call__(self, *args, **kwargs):
+        out = self.results[min(self.calls, len(self.results) - 1)]
+        self.calls += 1
+        return out
+
+
+def _patch(monkeypatch, fake: _Fake, attempts: int) -> None:
+    monkeypatch.setattr(listing, "_fetch_sync", fake)
+    monkeypatch.setattr(listing.config, "TIKTOK_LISTING_MAX_ATTEMPTS", attempts)
+
+
+async def test_returns_first_nonempty_without_retrying(monkeypatch):
+    fake = _Fake([[{"id": "1"}]])
+    _patch(monkeypatch, fake, 3)
+    items = await listing.fetch_item_list("https://tt/@x", 5)
+    assert items == [{"id": "1"}]
+    assert fake.calls == 1  # a draw with items never retries
+
+
+async def test_retries_past_empty_draws_then_hits(monkeypatch):
+    fake = _Fake([[], [], [{"id": "9"}]])
+    _patch(monkeypatch, fake, 3)
+    items = await listing.fetch_item_list("https://tt/@x", 5)
+    assert items == [{"id": "9"}]
+    assert fake.calls == 3  # two empty (flagged-IP) draws retried, third lands
+
+
+async def test_stops_at_attempt_ceiling_when_always_empty(monkeypatch):
+    fake = _Fake([[]])
+    _patch(monkeypatch, fake, 3)
+    items = await listing.fetch_item_list("https://tt/@x", 5)
+    assert items == []
+    assert fake.calls == 3  # capped; caller then emits the ErrorItem
+
+
+async def test_single_attempt_config_disables_retry(monkeypatch):
+    fake = _Fake([[]])
+    _patch(monkeypatch, fake, 1)
+    items = await listing.fetch_item_list("https://tt/@x", 5)
+    assert items == []
+    assert fake.calls == 1  # static-IP setups opt out via attempts=1

surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py

@@ -0,0 +1,185 @@
+"""End-to-end orchestration over a fake fetch (no network).
+
+Drives the public collector: input -> target -> blob-first flow -> items.
+"""
+
+from __future__ import annotations
+
+import json
+
+from app.proprietary.platforms.tiktok import TikTokScrapeInput, scrape_tiktok
+
+
+async def _no_html(_url: str) -> str:
+    """Fetch stub that yields no rehydration blob (skips profile metadata)."""
+    return ""
+
+
+def _profile_page(username: str, followers: int, videos: int) -> str:
+    blob = {
+        "__DEFAULT_SCOPE__": {
+            "webapp.user-detail": {
+                "userInfo": {
+                    "user": {
+                        "id": "u1",
+                        "uniqueId": username,
+                        "nickname": "Nick",
+                        "verified": True,
+                    },
+                    "stats": {"followerCount": followers, "videoCount": videos},
+                }
+            }
+        }
+    }
+    return (
+        '<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" '
+        f'type="application/json">{json.dumps(blob)}</script>'
+    )
+
+
+def _video_page(video_id: str, username: str) -> str:
+    blob = {
+        "__DEFAULT_SCOPE__": {
+            "webapp.video-detail": {
+                "itemInfo": {
+                    "itemStruct": {
+                        "id": video_id,
+                        "desc": "hello",
+                        "author": {"uniqueId": username},
+                        "stats": {"diggCount": 5},
+                    }
+                }
+            }
+        }
+    }
+    return (
+        '<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" '
+        f'type="application/json">{json.dumps(blob)}</script>'
+    )
+
+
+async def test_scrape_video_url_returns_parsed_item():
+    url = "https://www.tiktok.com/@scout2015/video/123"
+
+    async def fake_fetch(_url: str) -> str:
+        return _video_page("123", "scout2015")
+
+    items = await scrape_tiktok(TikTokScrapeInput(postURLs=[url]), fetch=fake_fetch)
+
+    assert len(items) == 1
+    assert items[0]["id"] == "123"
+    assert items[0]["diggCount"] == 5
+    assert items[0]["webVideoUrl"] == "https://www.tiktok.com/@scout2015/video/123"
+    assert items[0]["scrapedAt"] is not None
+
+
+async def test_scrape_honors_limit_across_targets():
+    urls = [
+        "https://www.tiktok.com/@a/video/1",
+        "https://www.tiktok.com/@b/video/2",
+    ]
+
+    async def fake_fetch(url: str) -> str:
+        vid = url.rsplit("/", 1)[1]
+        user = url.split("@")[1].split("/")[0]
+        return _video_page(vid, user)
+
+    items = await scrape_tiktok(
+        TikTokScrapeInput(postURLs=urls), limit=1, fetch=fake_fetch
+    )
+    assert len(items) == 1
+
+
+async def test_scrape_skips_unrecognized_urls():
+    async def fake_fetch(_url: str) -> str:
+        return ""
+
+    items = await scrape_tiktok(
+        TikTokScrapeInput(postURLs=["https://example.com/x"]), fetch=fake_fetch
+    )
+    assert items == []
+
+
+async def test_scrape_profile_returns_listing_items():
+    async def fake_listing(_url: str, _count: int) -> list[dict]:
+        return [
+            {"id": "1", "author": {"uniqueId": "a"}},
+            {"id": "2", "author": {"uniqueId": "a"}},
+        ]
+
+    items = await scrape_tiktok(
+        TikTokScrapeInput(profiles=["a"], resultsPerPage=5),
+        fetch=_no_html,
+        fetch_listing=fake_listing,
+    )
+    assert [i["id"] for i in items] == ["1", "2"]
+    assert items[0]["webVideoUrl"] == "https://www.tiktok.com/@a/video/1"
+
+
+async def test_profile_emits_metadata_then_videos():
+    # The blob metadata item comes first and is billable; videos follow.
+    async def fake_fetch(_url: str) -> str:
+        return _profile_page("a", followers=100, videos=2)
+
+    async def fake_listing(_url: str, _count: int) -> list[dict]:
+        return [{"id": "1", "author": {"uniqueId": "a"}}]
+
+    items = await scrape_tiktok(
+        TikTokScrapeInput(profiles=["a"], resultsPerPage=5),
+        fetch=fake_fetch,
+        fetch_listing=fake_listing,
+    )
+    assert len(items) == 2
+    profile, video = items
+    assert "webVideoUrl" not in profile  # metadata item, not a video
+    assert profile["name"] == "a"
+    assert profile["fans"] == 100
+    assert profile["verified"] is True
+    assert profile["scrapedAt"] is not None
+    assert video["id"] == "1"
+
+
+async def test_profile_metadata_survives_blocked_listing():
+    # Videos withheld from anonymous access: we still return the profile metadata
+    # (not just an ErrorItem), so a blocked profile isn't a total loss.
+    async def fake_fetch(_url: str) -> str:
+        return _profile_page("a", followers=100, videos=9)
+
+    async def fake_listing(_url: str, _count: int) -> list[dict]:
+        return []
+
+    items = await scrape_tiktok(
+        TikTokScrapeInput(profiles=["a"], resultsPerPage=5),
+        fetch=fake_fetch,
+        fetch_listing=fake_listing,
+    )
+    assert len(items) == 2
+    assert items[0]["name"] == "a"
+    assert items[0]["fans"] == 100
+    assert items[1]["errorCode"] == "no_items"
+
+
+async def test_listing_dedupes_then_caps_per_target():
+    async def fake_listing(_url: str, _count: int) -> list[dict]:
+        return [{"id": "1"}, {"id": "1"}, {"id": "2"}, {"id": "3"}]
+
+    items = await scrape_tiktok(
+        TikTokScrapeInput(hashtags=["x"], resultsPerPage=2), fetch_listing=fake_listing
+    )
+    assert [i["id"] for i in items] == ["1", "2"]
+
+
+async def test_empty_listing_emits_error_item():
+    # A trust-gated/empty feed (0 videos) must surface one honest ErrorItem,
+    # tagged with errorCode, rather than vanishing silently.
+    async def fake_listing(_url: str, _count: int) -> list[dict]:
+        return []
+
+    items = await scrape_tiktok(
+        TikTokScrapeInput(profiles=["nasa"], resultsPerPage=5),
+        fetch=_no_html,
+        fetch_listing=fake_listing,
+    )
+    assert len(items) == 1
+    assert items[0]["errorCode"] == "no_items"
+    assert items[0]["input"] == "nasa"

surfsense_backend/tests/unit/platforms/tiktok/test_parsers.py

@@ -0,0 +1,102 @@
+"""Raw TikTok payload -> normalized item mapping (pure, no network)."""
+
+from __future__ import annotations
+
+from app.proprietary.platforms.tiktok.extraction import parse_author, parse_video
+
+_ITEM_STRUCT = {
+    "id": "7534061113365859586",
+    "desc": "haha #comeramabanana",
+    "createTime": 1_700_000_000,
+    "author": {
+        "id": "6733",
+        "uniqueId": "bruniela_",
+        "nickname": "Bruni",
+        "verified": False,
+        "signature": "bio here",
+        "avatarLarger": "https://cdn/avatar.jpg",
+    },
+    "authorStats": {
+        "followerCount": 51200,
+        "followingCount": 269,
+        "heartCount": 3_000_000,
+        "videoCount": 259,
+    },
+    "stats": {
+        "diggCount": 5344,
+        "shareCount": 701,
+        "playCount": 55700,
+        "commentCount": 24,
+        "collectCount": 291,
+    },
+    "music": {
+        "id": "7529",
+        "title": "som original",
+        "authorName": "fox_rus0",
+        "original": True,
+        "playUrl": "https://cdn/music.mp3",
+    },
+    "video": {
+        "height": 1024,
+        "width": 576,
+        "duration": 16,
+        "cover": "https://cdn/cover.jpg",
+        "format": "mp4",
+        "definition": "540p",
+    },
+    "challenges": [{"id": "4982299", "title": "comeramabanana"}],
+}
+
+
+def test_parse_video_maps_core_and_derived_fields():
+    item = parse_video(_ITEM_STRUCT)
+
+    assert item["id"] == "7534061113365859586"
+    assert item["text"] == "haha #comeramabanana"
+    assert item["createTimeISO"] == "2023-11-14T22:13:20.000Z"
+
+    assert item["authorMeta"]["name"] == "bruniela_"
+    assert item["authorMeta"]["nickName"] == "Bruni"
+    assert item["authorMeta"]["profileUrl"] == "https://www.tiktok.com/@bruniela_"
+    assert item["authorMeta"]["fans"] == 51200
+
+    assert item["musicMeta"]["musicName"] == "som original"
+    assert item["videoMeta"]["duration"] == 16
+
+    assert item["diggCount"] == 5344
+    assert item["playCount"] == 55700
+
+    assert item["hashtags"] == [{"id": "4982299", "name": "comeramabanana"}]
+    assert (
+        item["webVideoUrl"]
+        == "https://www.tiktok.com/@bruniela_/video/7534061113365859586"
+    )
+
+
+_USER_INFO = {
+    "user": {
+        "id": "6733",
+        "uniqueId": "bruniela_",
+        "nickname": "Bruni",
+        "verified": True,
+        "signature": "bio here",
+        "avatarLarger": "https://cdn/avatar.jpg",
+        "privateAccount": False,
+    },
+    "stats": {
+        "followerCount": 51200,
+        "followingCount": 269,
+        "heartCount": 3_000_000,
+        "videoCount": 259,
+    },
+}
+
+
+def test_parse_author_maps_user_and_stats():
+    author = parse_author(_USER_INFO)
+    assert author["name"] == "bruniela_"
+    assert author["nickName"] == "Bruni"
+    assert author["verified"] is True
+    assert author["profileUrl"] == "https://www.tiktok.com/@bruniela_"
+    assert author["fans"] == 51200
+    assert author["video"] == 259

surfsense_backend/tests/unit/platforms/tiktok/test_scopes.py

@@ -0,0 +1,30 @@
+"""Navigating the rehydration blob to its useful scopes (pure, no network)."""
+
+from __future__ import annotations
+
+from app.proprietary.platforms.tiktok.extraction import user_info, video_item_struct
+
+
+def test_video_item_struct_navigates_video_detail_scope():
+    data = {
+        "__DEFAULT_SCOPE__": {
+            "webapp.video-detail": {"itemInfo": {"itemStruct": {"id": "123"}}}
+        }
+    }
+    item = video_item_struct(data)
+    assert item == {"id": "123"}
+
+
+def test_user_info_navigates_user_detail_scope():
+    data = {
+        "__DEFAULT_SCOPE__": {
+            "webapp.user-detail": {"userInfo": {"user": {"uniqueId": "scout2015"}}}
+        }
+    }
+    info = user_info(data)
+    assert info == {"user": {"uniqueId": "scout2015"}}
+
+
+def test_scopes_return_none_when_absent():
+    assert video_item_struct({}) is None
+    assert user_info({"__DEFAULT_SCOPE__": {}}) is None

surfsense_backend/tests/unit/platforms/tiktok/test_target_resolver.py

@@ -0,0 +1,47 @@
+"""URL classification for the TikTok scraper (pure, no network)."""
+
+from __future__ import annotations
+
+from app.proprietary.platforms.tiktok.targets import resolve_target
+
+
+def test_resolve_video_carries_username_and_id():
+    target = resolve_target("https://www.tiktok.com/@scout2015/video/6718335390845095173")
+    assert target is not None
+    assert target.kind == "video"
+    assert target.value == "6718335390845095173"
+    assert target.username == "scout2015"
+
+
+def test_resolve_profile():
+    target = resolve_target("https://www.tiktok.com/@scout2015")
+    assert target is not None
+    assert target.kind == "profile"
+    assert target.value == "scout2015"
+
+
+def test_resolve_hashtag():
+    target = resolve_target("https://www.tiktok.com/tag/funny")
+    assert target is not None
+    assert target.kind == "hashtag"
+    assert target.value == "funny"
+
+
+def test_resolve_search_top_video_and_user_sections():
+    top = resolve_target("https://www.tiktok.com/search?q=cats")
+    assert top is not None
+    assert top.kind == "search"
+    assert top.value == "cats"
+    assert top.section is None
+
+    videos = resolve_target("https://www.tiktok.com/search/video?q=cats")
+    assert videos is not None and videos.section == "video"
+
+    users = resolve_target("https://www.tiktok.com/search/user?q=cats")
+    assert users is not None and users.section == "user"
+
+
+def test_resolve_rejects_non_tiktok_and_unknown_paths():
+    assert resolve_target("https://example.com/@scout2015") is None
+    assert resolve_target("https://www.tiktok.com/") is None
+    assert resolve_target("https://www.tiktok.com/foundation") is None

surfsense_backend/tests/unit/platforms/tiktok/test_trending.py

@@ -0,0 +1,35 @@
+"""Trending orchestration over a fake fetch (no network).
+
+Drives ``scrape_tiktok_trending``: the Explore feed -> captured itemStructs ->
+video items, reusing the listing flow (parse/dedupe/cap/empty-ErrorItem).
+"""
+
+from __future__ import annotations
+
+from app.proprietary.platforms.tiktok import scrape_tiktok_trending
+
+
+async def test_trending_parses_dedupes_and_caps():
+    async def fake_fetch(url: str, _cap: int) -> list[dict]:
+        assert url == "https://www.tiktok.com/explore"
+        return [
+            {"id": "1", "author": {"uniqueId": "a"}},
+            {"id": "1", "author": {"uniqueId": "a"}},
+            {"id": "2", "author": {"uniqueId": "b"}},
+        ]
+
+    items = await scrape_tiktok_trending(count=2, fetch_trending_fn=fake_fetch)
+
+    assert [i["id"] for i in items] == ["1", "2"]
+    assert items[0]["webVideoUrl"] == "https://www.tiktok.com/@a/video/1"
+    assert items[0]["scrapedAt"] is not None
+
+
+async def test_trending_empty_feed_emits_error_item():
+    async def fake_fetch(_url: str, _cap: int) -> list[dict]:
+        return []
+
+    items = await scrape_tiktok_trending(count=5, fetch_trending_fn=fake_fetch)
+
+    assert len(items) == 1
+    assert items[0]["errorCode"] == "no_items"

surfsense_backend/tests/unit/platforms/tiktok/test_user_search.py

@@ -0,0 +1,70 @@
+"""User-search orchestration over a fake fetch (no network).
+
+Drives ``search_tiktok_users``: queries -> captured ``user_info`` -> profile items.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from app.proprietary.platforms.tiktok import search_tiktok_users
+
+
+def _user(uid: str, unique_id: str, followers: int = 10) -> dict[str, Any]:
+    return {
+        "uid": uid,
+        "unique_id": unique_id,
+        "nickname": unique_id.upper(),
+        "signature": "bio",
+        "follower_count": followers,
+        "total_favorited": 999,
+        "sec_uid": f"sec-{uid}",
+        "enterprise_verify_reason": "official" if uid == "1" else "",
+        "avatar_thumb": {"url_list": [f"https://cdn/{uid}.webp"]},
+    }
+
+
+async def test_user_search_parses_dedupes_and_caps():
+    async def fake_fetch(_url: str, _cap: int) -> list[dict]:
+        return [_user("1", "nasa"), _user("1", "nasa"), _user("2", "nasa2")]
+
+    items = await search_tiktok_users(
+        ["nasa"], per_query=2, fetch_users=fake_fetch
+    )
+
+    assert [i["id"] for i in items] == ["1", "2"]
+    first = items[0]
+    assert first["name"] == "nasa"
+    assert first["nickName"] == "NASA"
+    assert first["profileUrl"] == "https://www.tiktok.com/@nasa"
+    assert first["verified"] is True
+    assert first["fans"] == 10
+    assert first["avatar"] == "https://cdn/1.webp"
+    assert first["secUid"] == "sec-1"
+    assert first["scrapedAt"] is not None
+    assert items[1]["verified"] is False
+
+
+async def test_user_search_empty_query_emits_error_item():
+    async def fake_fetch(_url: str, _cap: int) -> list[dict]:
+        return []
+
+    items = await search_tiktok_users(
+        ["ghost"], per_query=5, fetch_users=fake_fetch
+    )
+
+    assert len(items) == 1
+    assert items[0]["errorCode"] == "no_users"
+    assert items[0]["input"] == "ghost"
+
+
+async def test_user_search_honors_limit_across_queries():
+    async def fake_fetch(_url: str, _cap: int) -> list[dict]:
+        return [_user("1", "a"), _user("2", "b")]
+
+    items = await search_tiktok_users(
+        ["q1", "q2"], per_query=5, limit=3, fetch_users=fake_fetch
+    )
+
+    # 2 from q1 + 1 from q2, then the cross-query limit stops it.
+    assert len(items) == 3