Merge remote-tracking branch 'upstream/dev' into feat/instagram-scraper

surfsense_backend/tests/unit/capabilities/access/test_agent_tools.py

@@ -78,8 +78,14 @@ async def test_registry_becomes_one_tool_per_verb_plus_readers(isolate):
     tools = isolate.module.build_capability_tools(workspace_id=7, capabilities=caps)
 
     by_name = {t.name: t for t in tools}
-    # One tool per verb, plus the two shared run-reader tools.
-    assert set(by_name) == {"web_scrape", "web_discover", "read_run", "search_run"}
+    # One tool per verb, plus the shared run-reader tools.
+    assert set(by_name) == {
+        "web_scrape",
+        "web_discover",
+        "read_run",
+        "search_run",
+        "export_run",
+    }
     assert by_name["web_scrape"].description == "web.scrape does a thing."
     assert by_name["web_scrape"].args_schema is _EchoInput
 

surfsense_backend/tests/unit/capabilities/tiktok/comments/test_executor.py

@@ -0,0 +1,48 @@
+"""``tiktok.comments`` executor: verb input → scraper args → typed comment items.
+
+Boundary mocked: the proprietary comments actor (injected fake). NOT mocked: the
+verb's own payload→args forwarding and the dict→CommentItem wrapping.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from app.capabilities.tiktok.comments.executor import build_comments_executor
+from app.capabilities.tiktok.comments.schemas import CommentsInput, CommentsOutput
+
+pytestmark = pytest.mark.unit
+
+_VIDEO = "https://www.tiktok.com/@bob/video/123"
+
+
+class _FakeComments:
+    """Records the urls + kwargs it was called with; returns canned items."""
+
+    def __init__(self, items: list[dict]):
+        self._items = items
+        self.calls: list[tuple[list[str], int, int | None]] = []
+
+    async def __call__(
+        self, video_urls: list[str], *, per_video: int, limit: int | None = None
+    ) -> list[dict]:
+        self.calls.append((video_urls, per_video, limit))
+        return self._items
+
+
+async def test_forwards_urls_and_limits_and_wraps_items():
+    comments = _FakeComments([{"id": "1", "text": "hi"}])
+    execute = build_comments_executor(comments_fn=comments)
+
+    out = await execute(
+        CommentsInput(video_urls=[_VIDEO], comments_per_video=15, max_items=40)
+    )
+
+    assert isinstance(out, CommentsOutput)
+    assert len(out.items) == 1
+    assert out.items[0].text == "hi"
+
+    (urls, per_video, limit) = comments.calls[0]
+    assert urls == [_VIDEO]
+    assert per_video == 15
+    assert limit == 40

surfsense_backend/tests/unit/capabilities/tiktok/comments/test_schemas.py

@@ -0,0 +1,48 @@
+"""``tiktok.comments`` input guards and billing: a video URL is required, bounded,
+and ErrorItems are surfaced free."""
+
+from __future__ import annotations
+
+import pytest
+from pydantic import ValidationError
+
+from app.capabilities.tiktok.comments.schemas import CommentsInput, CommentsOutput
+from app.capabilities.tiktok.scrape.schemas import MAX_TIKTOK_ITEMS, MAX_TIKTOK_SOURCES
+
+pytestmark = pytest.mark.unit
+
+_VIDEO = "https://www.tiktok.com/@bob/video/123"
+
+
+def test_rejects_input_with_no_url():
+    with pytest.raises(ValidationError):
+        CommentsInput(video_urls=[])
+
+
+def test_defaults_and_bounds():
+    payload = CommentsInput(video_urls=[_VIDEO])
+    assert payload.max_items == 20
+    assert payload.comments_per_video == 20
+    assert payload.estimated_units == 20
+    with pytest.raises(ValidationError):
+        CommentsInput(video_urls=[_VIDEO], max_items=0)
+    with pytest.raises(ValidationError):
+        CommentsInput(video_urls=[_VIDEO], max_items=MAX_TIKTOK_ITEMS + 1)
+
+
+def test_rejects_more_urls_than_the_cap():
+    too_many = [f"{_VIDEO}{i}" for i in range(MAX_TIKTOK_SOURCES + 1)]
+    with pytest.raises(ValidationError):
+        CommentsInput(video_urls=too_many)
+
+
+def test_error_items_are_not_billed():
+    # Real comments count; ErrorItems (bad URL / empty video) are surfaced free.
+    out = CommentsOutput(
+        items=[
+            {"id": "1", "text": "hi"},
+            {"errorCode": "no_comments", "input": "123", "error": "empty"},
+        ]
+    )
+    assert len(out.items) == 2
+    assert out.billable_units == 1

surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_executor.py

@@ -0,0 +1,79 @@
+"""``tiktok.scrape`` executor: verb input → actor input mapping → typed items.
+
+Boundary mocked: the proprietary scraper (injected fake). NOT mocked: the verb's
+own payload→TikTokScrapeInput mapping and the dict→TikTokVideoItem wrapping.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from app.capabilities.tiktok.scrape.executor import build_scrape_executor
+from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput
+from app.exceptions import ForbiddenError
+from app.proprietary.platforms.tiktok import TikTokAccessBlockedError, TikTokScrapeInput
+
+pytestmark = pytest.mark.unit
+
+
+class _FakeScraper:
+    """Records the actor input + limit it was called with; returns canned items."""
+
+    def __init__(self, items: list[dict]):
+        self._items = items
+        self.calls: list[tuple[TikTokScrapeInput, int | None]] = []
+
+    async def __call__(
+        self, actor_input: TikTokScrapeInput, *, limit: int | None = None
+    ) -> list[dict]:
+        self.calls.append((actor_input, limit))
+        return self._items
+
+
+async def test_maps_urls_to_start_urls_and_wraps_items():
+    scraper = _FakeScraper([{"id": "123", "text": "hello"}])
+    execute = build_scrape_executor(scrape_fn=scraper)
+
+    out = await execute(ScrapeInput(urls=["https://www.tiktok.com/@nasa/video/123"]))
+
+    assert isinstance(out, ScrapeOutput)
+    assert len(out.items) == 1
+    assert out.items[0].id == "123"
+
+    (actor_input, _limit) = scraper.calls[0]
+    assert [u.url for u in actor_input.startUrls] == [
+        "https://www.tiktok.com/@nasa/video/123"
+    ]
+
+
+async def test_forwards_typed_sources_and_limit():
+    scraper = _FakeScraper([])
+    execute = build_scrape_executor(scrape_fn=scraper)
+
+    await execute(
+        ScrapeInput(
+            profiles=["nasa"],
+            hashtags=["food"],
+            search_queries=["cats"],
+            results_per_page=7,
+            max_items=25,
+        )
+    )
+
+    (actor_input, limit) = scraper.calls[0]
+    assert actor_input.profiles == ["nasa"]
+    assert actor_input.hashtags == ["food"]
+    assert actor_input.searchQueries == ["cats"]
+    assert actor_input.resultsPerPage == 7
+    # The outer collection limit is the caller's total-item cap.
+    assert limit == 25
+
+
+async def test_access_blocked_maps_to_forbidden():
+    async def _blocked(actor_input: TikTokScrapeInput, *, limit: int | None = None):
+        raise TikTokAccessBlockedError("all IPs refused")
+
+    execute = build_scrape_executor(scrape_fn=_blocked)
+
+    with pytest.raises(ForbiddenError):
+        await execute(ScrapeInput(hashtags=["food"]))

surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_schemas.py

@@ -0,0 +1,58 @@
+"""``tiktok.scrape`` input guards: a source is required and the batch is bounded."""
+
+from __future__ import annotations
+
+import pytest
+from pydantic import ValidationError
+
+from app.capabilities.tiktok.scrape.schemas import (
+    MAX_TIKTOK_ITEMS,
+    MAX_TIKTOK_SOURCES,
+    ScrapeInput,
+    ScrapeOutput,
+)
+
+pytestmark = pytest.mark.unit
+
+
+def test_rejects_input_with_no_source():
+    with pytest.raises(ValidationError):
+        ScrapeInput()
+
+
+def test_accepts_urls_only():
+    payload = ScrapeInput(urls=["https://www.tiktok.com/@nasa"])
+    assert payload.profiles == []
+
+
+def test_accepts_hashtags_only():
+    payload = ScrapeInput(hashtags=["food"])
+    assert payload.hashtags == ["food"]
+
+
+def test_defaults_and_bounds():
+    payload = ScrapeInput(hashtags=["food"])
+    assert payload.max_items == 10
+    assert payload.results_per_page == 10
+    with pytest.raises(ValidationError):
+        ScrapeInput(hashtags=["food"], max_items=0)
+    with pytest.raises(ValidationError):
+        ScrapeInput(hashtags=["food"], max_items=MAX_TIKTOK_ITEMS + 1)
+
+
+def test_rejects_more_sources_than_the_cap():
+    too_many = [f"tag{i}" for i in range(MAX_TIKTOK_SOURCES + 1)]
+    with pytest.raises(ValidationError):
+        ScrapeInput(hashtags=too_many)
+
+
+def test_error_items_are_not_billed():
+    # Real videos count; ErrorItems (blocked/empty targets) are surfaced free.
+    out = ScrapeOutput(
+        items=[
+            {"id": "1", "webVideoUrl": "https://tiktok.com/@a/video/1"},
+            {"errorCode": "no_items", "input": "nasa", "error": "blocked"},
+        ]
+    )
+    assert len(out.items) == 2
+    assert out.billable_units == 1

surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py

@@ -0,0 +1,57 @@
+"""The tiktok namespace registers its verb as one Capability the doors/agent read."""
+
+from __future__ import annotations
+
+import pytest
+
+from app.capabilities import (
+    tiktok,  # noqa: F401  — importing the namespace registers its verbs
+)
+from app.capabilities.core import BillingUnit
+from app.capabilities.core.store import get_capability
+from app.capabilities.tiktok.comments.schemas import CommentsInput, CommentsOutput
+from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput
+from app.capabilities.tiktok.trending.schemas import TrendingInput, TrendingOutput
+from app.capabilities.tiktok.user_search.schemas import (
+    UserSearchInput,
+    UserSearchOutput,
+)
+
+pytestmark = pytest.mark.unit
+
+
+def test_tiktok_scrape_is_registered_and_billed_per_video():
+    cap = get_capability("tiktok.scrape")
+
+    assert cap.name == "tiktok.scrape"
+    assert cap.input_schema is ScrapeInput
+    assert cap.output_schema is ScrapeOutput
+    assert cap.billing_unit is BillingUnit.TIKTOK_VIDEO
+
+
+def test_tiktok_user_search_is_registered_and_billed_per_profile():
+    cap = get_capability("tiktok.user_search")
+
+    assert cap.name == "tiktok.user_search"
+    assert cap.input_schema is UserSearchInput
+    assert cap.output_schema is UserSearchOutput
+    assert cap.billing_unit is BillingUnit.TIKTOK_USER
+
+
+def test_tiktok_comments_is_registered_and_billed_per_comment():
+    cap = get_capability("tiktok.comments")
+
+    assert cap.name == "tiktok.comments"
+    assert cap.input_schema is CommentsInput
+    assert cap.output_schema is CommentsOutput
+    assert cap.billing_unit is BillingUnit.TIKTOK_COMMENT
+
+
+def test_tiktok_trending_is_registered_and_billed_per_video():
+    cap = get_capability("tiktok.trending")
+
+    assert cap.name == "tiktok.trending"
+    assert cap.input_schema is TrendingInput
+    assert cap.output_schema is TrendingOutput
+    # Trending returns videos, so it shares the per-video meter.
+    assert cap.billing_unit is BillingUnit.TIKTOK_VIDEO

surfsense_backend/tests/unit/capabilities/tiktok/trending/test_executor.py

@@ -0,0 +1,38 @@
+"""``tiktok.trending`` executor: verb input → scraper count → typed video items.
+
+Boundary mocked: the proprietary trending actor (injected fake). NOT mocked: the
+verb's own count forwarding and the dict→TikTokVideoItem wrapping.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from app.capabilities.tiktok.trending.executor import build_trending_executor
+from app.capabilities.tiktok.trending.schemas import TrendingInput, TrendingOutput
+
+pytestmark = pytest.mark.unit
+
+
+class _FakeTrending:
+    """Records the count it was called with; returns canned items."""
+
+    def __init__(self, items: list[dict]):
+        self._items = items
+        self.calls: list[int] = []
+
+    async def __call__(self, *, count: int) -> list[dict]:
+        self.calls.append(count)
+        return self._items
+
+
+async def test_forwards_count_and_wraps_items():
+    trending = _FakeTrending([{"id": "1", "text": "viral"}])
+    execute = build_trending_executor(trending_fn=trending)
+
+    out = await execute(TrendingInput(max_items=30))
+
+    assert isinstance(out, TrendingOutput)
+    assert len(out.items) == 1
+    assert out.items[0].id == "1"
+    assert trending.calls == [30]

surfsense_backend/tests/unit/capabilities/tiktok/trending/test_schemas.py

@@ -0,0 +1,33 @@
+"""``tiktok.trending`` input guards and billing: bounded count, ErrorItems free."""
+
+from __future__ import annotations
+
+import pytest
+from pydantic import ValidationError
+
+from app.capabilities.tiktok.scrape.schemas import MAX_TIKTOK_ITEMS
+from app.capabilities.tiktok.trending.schemas import TrendingInput, TrendingOutput
+
+pytestmark = pytest.mark.unit
+
+
+def test_defaults_and_bounds():
+    payload = TrendingInput()
+    assert payload.max_items == 20
+    assert payload.estimated_units == 20
+    with pytest.raises(ValidationError):
+        TrendingInput(max_items=0)
+    with pytest.raises(ValidationError):
+        TrendingInput(max_items=MAX_TIKTOK_ITEMS + 1)
+
+
+def test_error_items_are_not_billed():
+    # Real videos count; an ErrorItem (empty/withheld feed) is surfaced free.
+    out = TrendingOutput(
+        items=[
+            {"id": "1", "webVideoUrl": "https://tiktok.com/@a/video/1"},
+            {"errorCode": "no_items", "input": "explore", "error": "empty"},
+        ]
+    )
+    assert len(out.items) == 2
+    assert out.billable_units == 1

surfsense_backend/tests/unit/capabilities/tiktok/user_search/test_executor.py

@@ -0,0 +1,49 @@
+"""``tiktok.user_search`` executor: verb input → search args → typed profile items.
+
+Boundary mocked: the proprietary search actor (injected fake). NOT mocked: the
+verb's own payload→args forwarding and the dict→TikTokProfileItem wrapping.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from app.capabilities.tiktok.user_search.executor import build_user_search_executor
+from app.capabilities.tiktok.user_search.schemas import (
+    UserSearchInput,
+    UserSearchOutput,
+)
+
+pytestmark = pytest.mark.unit
+
+
+class _FakeSearch:
+    """Records the queries + kwargs it was called with; returns canned items."""
+
+    def __init__(self, items: list[dict]):
+        self._items = items
+        self.calls: list[tuple[list[str], int, int | None]] = []
+
+    async def __call__(
+        self, queries: list[str], *, per_query: int, limit: int | None = None
+    ) -> list[dict]:
+        self.calls.append((queries, per_query, limit))
+        return self._items
+
+
+async def test_forwards_queries_and_limits_and_wraps_items():
+    search = _FakeSearch([{"id": "1", "name": "nasa"}])
+    execute = build_user_search_executor(search_fn=search)
+
+    out = await execute(
+        UserSearchInput(queries=["nasa"], results_per_query=7, max_items=25)
+    )
+
+    assert isinstance(out, UserSearchOutput)
+    assert len(out.items) == 1
+    assert out.items[0].name == "nasa"
+
+    (queries, per_query, limit) = search.calls[0]
+    assert queries == ["nasa"]
+    assert per_query == 7
+    assert limit == 25

surfsense_backend/tests/unit/capabilities/tiktok/user_search/test_schemas.py

@@ -0,0 +1,49 @@
+"""``tiktok.user_search`` input guards and billing: a query is required, bounded,
+and ErrorItems are surfaced free."""
+
+from __future__ import annotations
+
+import pytest
+from pydantic import ValidationError
+
+from app.capabilities.tiktok.scrape.schemas import MAX_TIKTOK_ITEMS, MAX_TIKTOK_SOURCES
+from app.capabilities.tiktok.user_search.schemas import (
+    UserSearchInput,
+    UserSearchOutput,
+)
+
+pytestmark = pytest.mark.unit
+
+
+def test_rejects_input_with_no_query():
+    with pytest.raises(ValidationError):
+        UserSearchInput(queries=[])
+
+
+def test_defaults_and_bounds():
+    payload = UserSearchInput(queries=["nasa"])
+    assert payload.max_items == 10
+    assert payload.results_per_query == 10
+    assert payload.estimated_units == 10
+    with pytest.raises(ValidationError):
+        UserSearchInput(queries=["nasa"], max_items=0)
+    with pytest.raises(ValidationError):
+        UserSearchInput(queries=["nasa"], max_items=MAX_TIKTOK_ITEMS + 1)
+
+
+def test_rejects_more_queries_than_the_cap():
+    too_many = [f"q{i}" for i in range(MAX_TIKTOK_SOURCES + 1)]
+    with pytest.raises(ValidationError):
+        UserSearchInput(queries=too_many)
+
+
+def test_error_items_are_not_billed():
+    # Real accounts count; ErrorItems (empty/withheld queries) are surfaced free.
+    out = UserSearchOutput(
+        items=[
+            {"id": "1", "name": "nasa"},
+            {"errorCode": "no_users", "input": "ghost", "error": "empty"},
+        ]
+    )
+    assert len(out.items) == 2
+    assert out.billable_units == 1