feat: api playground, other tweaks

surfsense_backend/tests/unit/agents/multi_agent_chat/test_web_search_removed.py

@@ -17,7 +17,10 @@ from app.agents.chat.multi_agent_chat.main_agent.tools.index import (
 from app.agents.chat.multi_agent_chat.subagents.registry import (
     SUBAGENT_BUILDERS_BY_NAME,
 )
-from app.utils.validators import DEPRECATED_CONNECTOR_TYPES, raise_if_connector_deprecated
+from app.utils.validators import (
+    DEPRECATED_CONNECTOR_TYPES,
+    raise_if_connector_deprecated,
+)
 
 pytestmark = pytest.mark.unit
 

surfsense_backend/tests/unit/capabilities/access/test_rest_router.py

@@ -105,6 +105,24 @@ def test_registered_verbs_appear_on_rest():
     assert "/workspaces/{workspace_id}/scrapers/web/crawl" in paths
 
 
+@pytest.mark.asyncio
+async def test_capabilities_endpoint_lists_verbs_with_input_schema(monkeypatch):
+    """The playground reads verb identity + input JSON schema from one GET."""
+    app = _build_app([_ECHO], monkeypatch)
+    async with _client(app) as client:
+        resp = await client.get("/api/v1/workspaces/7/scrapers/capabilities")
+    assert resp.status_code == 200
+    body = resp.json()
+    assert len(body) == 1
+    entry = body[0]
+    assert entry["name"] == "test.echo"
+    assert entry["description"] == "Echo the input back for tests."
+    # The schemas are the pydantic models' JSON schemas: the form renders the
+    # input schema, the API reference docs render both.
+    assert "value" in entry["input_schema"]["properties"]
+    assert "properties" in entry["output_schema"]
+
+
 @pytest.mark.asyncio
 async def test_over_budget_is_blocked_before_the_executor(monkeypatch):
     from app.capabilities.core.access import rest
@@ -243,6 +261,7 @@ def _fake_run_row(**overrides):
         "thread_id": None,
         "input": {"value": "hi"},
         "output_text": '{"echo": "hi"}',
+        "progress": None,
     }
     defaults.update(overrides)
     return SimpleNamespace(**defaults)
@@ -337,3 +356,111 @@ async def test_success_charges_once(monkeypatch):
     assert isinstance(output, _EchoOutput)
     assert unit is None
     assert ctx.workspace_id == 7
+
+
+@pytest.mark.asyncio
+async def test_async_mode_returns_202_and_pending_run(monkeypatch):
+    """``?mode=async`` inserts a pending run and returns its id without blocking."""
+    from unittest.mock import AsyncMock
+
+    from app.capabilities.core.access import rest
+
+    monkeypatch.setattr(
+        rest, "create_pending_run", AsyncMock(return_value="async-id"), raising=True
+    )
+    # Don't actually run the scrape in the background during this unit test.
+    monkeypatch.setattr(rest, "_execute_async_run", AsyncMock(), raising=True)
+
+    app = _build_app([_ECHO], monkeypatch)
+    async with _client(app) as client:
+        resp = await client.post(
+            "/api/v1/workspaces/7/scrapers/test/echo?mode=async",
+            json={"value": "hi"},
+        )
+    assert resp.status_code == 202
+    assert resp.json() == {"run_id": "run_async-id", "status": "running"}
+
+
+@pytest.mark.asyncio
+async def test_run_events_replays_buffer_then_finishes(monkeypatch):
+    """The SSE endpoint replays buffered events and closes on ``run.finished``."""
+    from app.capabilities.core.events import run_event_bus
+
+    row = _fake_run_row(status="running")
+    raw = str(row.id)
+    run_event_bus.publish(raw, {"type": "run.progress", "phase": "scraping", "current": 1})
+    run_event_bus.publish(raw, {"type": "run.finished", "status": "success", "item_count": 2})
+
+    app = _build_app_with_rows(monkeypatch, [row])
+    try:
+        async with _client(app) as client:
+            resp = await client.get(
+                f"/api/v1/workspaces/7/scrapers/runs/run_{raw}/events"
+            )
+        assert resp.status_code == 200
+        body = resp.text
+        assert '"type": "run.progress"' in body
+        assert '"type": "run.finished"' in body
+        assert '"status": "success"' in body
+    finally:
+        run_event_bus.close(raw)
+
+
+@pytest.mark.asyncio
+async def test_cancel_finalizes_running_run(monkeypatch):
+    """Cancel signals the task, finalizes as ``cancelled``, and emits a terminal."""
+    import asyncio
+    from unittest.mock import AsyncMock
+
+    from app.capabilities.core.access import rest
+    from app.capabilities.core.events import run_event_bus
+
+    finalize = AsyncMock(return_value=True)
+    monkeypatch.setattr(rest, "finalize_run", finalize, raising=True)
+
+    row = _fake_run_row(status="running")
+    raw = str(row.id)
+    task = asyncio.create_task(asyncio.sleep(60))
+    run_event_bus.register_task(raw, task)
+
+    app = _build_app_with_rows(monkeypatch, [row])
+    try:
+        async with _client(app) as client:
+            resp = await client.post(
+                f"/api/v1/workspaces/7/scrapers/runs/run_{raw}/cancel"
+            )
+        assert resp.status_code == 200
+        assert resp.json() == {"run_id": f"run_{raw}", "status": "cancelled"}
+        finalize.assert_awaited_once()
+        assert finalize.await_args.kwargs["status"] == "cancelled"
+    finally:
+        task.cancel()
+        await asyncio.gather(task, return_exceptions=True)
+        run_event_bus.close(raw)
+
+
+@pytest.mark.asyncio
+async def test_cancel_conflicts_when_not_running(monkeypatch):
+    """Cancelling a terminal run is a 409, not a silent overwrite."""
+    row = _fake_run_row(status="success")
+    app = _build_app_with_rows(monkeypatch, [row])
+    async with _client(app) as client:
+        resp = await client.post(
+            f"/api/v1/workspaces/7/scrapers/runs/run_{row.id}/cancel"
+        )
+    assert resp.status_code == 409
+
+
+def test_emit_progress_is_a_noop_without_context():
+    """Scraper code can call ``emit_progress`` freely; unset context = no-op."""
+    from app.capabilities.core.progress import emit_progress, progress_scope
+
+    # No active reporter -> returns without raising, records nothing.
+    emit_progress("phase", "message", current=1, total=2, unit="item")
+
+    # Inside a scope, coarse events are buffered for persistence.
+    with progress_scope() as reporter:
+        emit_progress("starting", "go")
+        emit_progress("done", current=5, unit="item")
+    assert len(reporter.coarse) == 2
+    assert reporter.coarse[0]["phase"] == "starting"

surfsense_backend/tests/unit/platforms/google_search/test_browser_loop.py

@@ -0,0 +1,36 @@
+"""The Windows regression this guards: main.py runs the server on a
+SelectorEventLoop (psycopg needs it), and Selector loops cannot spawn
+subprocesses — so patchright's Chromium launch died with NotImplementedError
+on every render. fetch.py now marshals all browser work onto a dedicated
+subprocess-capable loop; this check proves that marshalling works from a
+Selector main loop without paying for a real browser launch.
+"""
+
+import asyncio
+import sys
+
+import pytest
+
+from app.proprietary.platforms.google_search import fetch
+
+
+def test_browser_loop_can_spawn_subprocess_from_selector_loop():
+    async def spawn():
+        proc = await asyncio.create_subprocess_exec(
+            sys.executable,
+            "-c",
+            "print('ok')",
+            stdout=asyncio.subprocess.PIPE,
+        )
+        out, _ = await proc.communicate()
+        return out
+
+    async def main():
+        if sys.platform == "win32":
+            # The original bug: the server's own loop cannot do this.
+            with pytest.raises(NotImplementedError):
+                await spawn()
+        # The fix: the same work marshalled onto the browser loop succeeds.
+        assert b"ok" in await fetch._in_browser_loop(spawn())
+
+    asyncio.run(main(), loop_factory=asyncio.SelectorEventLoop)

surfsense_backend/tests/unit/platforms/youtube/test_subtitles_retry.py

@@ -0,0 +1,77 @@
+"""Offline tests for the subtitles blocked-IP retry (rotate-on-block, no network.
+
+Stubs ``_build_client``/``get_requests_proxies`` so the RequestBlocked retry
+path is exercised deterministically: retries only when a proxy is configured,
+each attempt gets a fresh client, and the final block is re-raised.
+"""
+
+from __future__ import annotations
+
+import pytest
+from youtube_transcript_api import RequestBlocked
+
+from app.proprietary.platforms.youtube import subtitles
+
+
+class _FakeTranscript:
+    is_generated = True
+    language_code = "en"
+
+    def fetch(self):
+        return []  # formatters iterate snippets; empty is fine here
+
+
+class _FakeTranscriptList:
+    def find_transcript(self, codes):
+        return _FakeTranscript()
+
+
+class _FakeApi:
+    """One 'IP': blocks if ``blocked``, else returns a transcript list."""
+
+    def __init__(self, blocked: bool) -> None:
+        self.blocked = blocked
+
+    def list(self, video_id: str):
+        if self.blocked:
+            raise RequestBlocked(video_id)
+        return _FakeTranscriptList()
+
+
+def _install(monkeypatch, outcomes: list[bool], proxied: bool) -> list[_FakeApi]:
+    """Each ``_build_client`` call pops the next outcome (True = blocked)."""
+    built: list[_FakeApi] = []
+
+    def fake_build():
+        api = _FakeApi(outcomes[len(built)])
+        built.append(api)
+        return api
+
+    monkeypatch.setattr(subtitles, "_build_client", fake_build)
+    monkeypatch.setattr(
+        subtitles,
+        "get_requests_proxies",
+        lambda: {"http": "http://p", "https": "http://p"} if proxied else None,
+    )
+    return built
+
+
+def test_blocked_then_success_retries_with_fresh_client(monkeypatch):
+    built = _install(monkeypatch, [True, True, False], proxied=True)
+    track = subtitles._fetch_subtitles_sync("vid", "en", "plaintext", False)
+    assert track.language == "en"
+    assert len(built) == 3  # two blocked attempts + one success, each a new client
+
+
+def test_all_attempts_blocked_reraises(monkeypatch):
+    built = _install(monkeypatch, [True] * 10, proxied=True)
+    with pytest.raises(RequestBlocked):
+        subtitles._fetch_subtitles_sync("vid", "en", "plaintext", False)
+    assert len(built) == subtitles._MAX_ROTATIONS + 1
+
+
+def test_no_proxy_means_single_attempt(monkeypatch):
+    built = _install(monkeypatch, [True] * 10, proxied=False)
+    with pytest.raises(RequestBlocked):
+        subtitles._fetch_subtitles_sync("vid", "en", "plaintext", False)
+    assert len(built) == 1  # same egress IP; retrying would be futile