Merge commit 'fa7075fde6' into dev

surfsense_backend/tests/unit/agents/chat/shared/tools/test_web_search.py

@@ -1,93 +0,0 @@
-"""Tests for the shared ``web_search`` tool's citable-result adaptation.
-
-The tool's network path (SearXNG + live connectors) is out of scope here; these
-cover the pure mapping from raw web results to renderable, citable documents and
-the end-to-end registration of ``WEB_RESULT`` ``[n]`` labels.
-"""
-
-from __future__ import annotations
-
-import pytest
-
-from app.agents.chat.multi_agent_chat.shared.citations import (
-    CitationRegistry,
-    CitationSourceType,
-)
-from app.agents.chat.multi_agent_chat.shared.document_render import render_web_results
-from app.agents.chat.shared.tools.web_search import (
-    _to_renderable_web_documents,
-    _web_source_label,
-)
-
-pytestmark = pytest.mark.unit
-
-
-def _raw_result(url: str, title: str, content: str) -> dict:
-    return {
-        "document": {"title": title, "metadata": {"url": url}},
-        "content": content,
-    }
-
-
-def test_web_source_label_strips_scheme_and_www() -> None:
-    assert _web_source_label("https://www.example.com/path") == "Web · example.com"
-    assert _web_source_label("http://news.site.org/a/b") == "Web · news.site.org"
-    assert _web_source_label("") == "Web"
-
-
-def test_adapter_maps_each_result_to_one_web_passage() -> None:
-    docs = _to_renderable_web_documents(
-        [
-            _raw_result("https://a.com/x", "Alpha", "alpha body"),
-            _raw_result("https://b.com/y", "Beta", "beta body"),
-        ]
-    )
-
-    assert [d.title for d in docs] == ["Alpha", "Beta"]
-    passages = [p for d in docs for p in d.passages]
-    assert all(p.source_type is CitationSourceType.WEB_RESULT for p in passages)
-    assert passages[0].locator == {"url": "https://a.com/x"}
-    assert passages[0].content == "alpha body"
-
-
-def test_adapter_skips_results_without_url_or_content() -> None:
-    docs = _to_renderable_web_documents(
-        [
-            _raw_result("", "No URL", "has content"),
-            _raw_result("https://c.com/z", "Empty", "   "),
-            _raw_result("https://d.com/w", "Good", "real content"),
-        ]
-    )
-
-    assert [d.title for d in docs] == ["Good"]
-
-
-def test_adapter_truncates_on_char_budget() -> None:
-    big = "x" * 30
-    docs = _to_renderable_web_documents(
-        [
-            _raw_result("https://a.com", "A", big),
-            _raw_result("https://b.com", "B", big),
-            _raw_result("https://c.com", "C", big),
-        ],
-        max_chars=50,
-    )
-
-    # First fits (30), second crosses 50 and stops the loop.
-    assert [d.title for d in docs] == ["A"]
-
-
-def test_end_to_end_registers_web_results_for_citation() -> None:
-    registry = CitationRegistry()
-    docs = _to_renderable_web_documents(
-        [_raw_result("https://example.com/a", "Example", "the answer is 42")]
-    )
-
-    block = render_web_results(docs, registry)
-
-    assert block is not None
-    assert "[1] the answer is 42" in block
-    entry = registry.resolve(1)
-    assert entry is not None
-    assert entry.source_type is CitationSourceType.WEB_RESULT
-    assert entry.locator == {"url": "https://example.com/a"}

surfsense_backend/tests/unit/agents/multi_agent_chat/shared/document_render/test_web_results.py

@@ -1,84 +0,0 @@
-"""Tests for the ``<web_results>`` wrapper around web-result excerpt documents."""
-
-from __future__ import annotations
-
-import pytest
-
-from app.agents.chat.multi_agent_chat.shared.citations import (
-    CitationRegistry,
-    CitationSourceType,
-)
-from app.agents.chat.multi_agent_chat.shared.document_render import (
-    RenderableDocument,
-    RenderablePassage,
-    render_web_results,
-)
-
-pytestmark = pytest.mark.unit
-
-
-def _web_doc(url: str, title: str, content: str) -> RenderableDocument:
-    return RenderableDocument(
-        title=title,
-        source=f"Web · {url.split('//', 1)[-1].split('/', 1)[0]}",
-        passages=[
-            RenderablePassage(
-                content=content,
-                locator={"url": url},
-                source_type=CitationSourceType.WEB_RESULT,
-            )
-        ],
-    )
-
-
-def test_returns_none_when_nothing_to_show() -> None:
-    registry = CitationRegistry()
-
-    assert render_web_results([], registry) is None
-
-
-def test_wraps_in_web_results_container() -> None:
-    registry = CitationRegistry()
-
-    block = render_web_results(
-        [_web_doc("https://example.com/a", "Example", "the answer is 42")],
-        registry,
-    )
-
-    assert block is not None
-    assert block.startswith("<web_results>")
-    assert block.endswith("</web_results>")
-    assert "cite a result with its [n]" in block
-    assert (
-        '<document title="Example" source="Web · example.com" view="excerpt">' in block
-    )
-    assert "[1] the answer is 42" in block
-
-
-def test_registers_each_result_as_web_result_with_url_locator() -> None:
-    registry = CitationRegistry()
-
-    render_web_results(
-        [
-            _web_doc("https://a.com/x", "A", "alpha"),
-            _web_doc("https://b.com/y", "B", "beta"),
-        ],
-        registry,
-    )
-
-    first = registry.resolve(1)
-    second = registry.resolve(2)
-    assert first is not None and second is not None
-    assert first.source_type is CitationSourceType.WEB_RESULT
-    assert first.locator == {"url": "https://a.com/x"}
-    assert second.locator == {"url": "https://b.com/y"}
-
-
-def test_same_url_reuses_label_across_calls() -> None:
-    registry = CitationRegistry()
-    doc = _web_doc("https://example.com/a", "Example", "stable fact")
-
-    render_web_results([doc], registry)
-    render_web_results([doc], registry)
-
-    assert registry.next_n == 2

surfsense_backend/tests/unit/agents/multi_agent_chat/subagents/shared/test_subagent_builder.py

@@ -25,6 +25,7 @@ from app.agents.chat.multi_agent_chat.shared.permissions.middleware.core import
     PermissionMiddleware,
 )
 from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import (
+    append_today_utc,
     pack_subagent,
 )
 
@@ -104,6 +105,35 @@ async def test_subagent_recovers_when_primary_llm_fails():
     assert final.content == "recovered via fallback"
 
 
+def test_packed_subagent_prompt_carries_todays_utc_date():
+    """Subagents must inherit the main agent's clock, not guess the date."""
+    from datetime import UTC, datetime
+
+    today = datetime.now(UTC).date().isoformat()
+
+    result = pack_subagent(
+        name="date_test",
+        description="test",
+        system_prompt="be helpful",
+        tools=[],
+        ruleset=Ruleset(origin="date_test", rules=[]),
+        dependencies={"flags": AgentFeatureFlags()},
+    )
+
+    prompt = result.spec["system_prompt"]
+    assert prompt.startswith("be helpful")
+    assert f"Today (UTC): {today}" in prompt
+
+
+def test_append_today_utc_is_idempotent_shape():
+    """Helper appends exactly one dated line and preserves the original body."""
+    from datetime import UTC, datetime
+
+    today = datetime.now(UTC).date().isoformat()
+    out = append_today_utc("body")
+    assert out == f"body\n\nToday (UTC): {today}\n"
+
+
 def _extract_permission_mw(spec) -> PermissionMiddleware:
     """Find the lone PermissionMiddleware in a subagent's middleware list."""
     matches = [m for m in spec["middleware"] if isinstance(m, PermissionMiddleware)]
@@ -191,25 +221,25 @@ def test_missing_user_allowlist_keeps_coded_behaviour():
 
 
 def test_user_allowlist_for_different_subagent_does_not_leak():
-    """User trust for ``linear`` must not affect a ``jira`` subagent compile."""
+    """User trust for one subagent must not affect a different subagent compile."""
     coded = Ruleset(
-        origin="jira",
+        origin="mcp_discovery",
         rules=[Rule(permission="save_issue", pattern="*", action="ask")],
     )
-    linear_allowlist = Ruleset(
-        origin="user_allowlist:linear",
+    other_allowlist = Ruleset(
+        origin="user_allowlist:knowledge_base",
         rules=[Rule(permission="save_issue", pattern="*", action="allow")],
     )
 
     result = pack_subagent(
-        name="jira",
+        name="mcp_discovery",
         description="test",
         system_prompt="x",
         tools=[],
         ruleset=coded,
         dependencies={
             "flags": AgentFeatureFlags(),
-            "user_allowlist_by_subagent": {"linear": linear_allowlist},
+            "user_allowlist_by_subagent": {"knowledge_base": other_allowlist},
         },
     )
 

surfsense_backend/tests/unit/agents/multi_agent_chat/test_mcp_allowlist_fallback.py

@@ -0,0 +1,52 @@
+"""Allowlist filtering in ``_load_http_mcp_tools``.
+
+Covers the fallback added after Notion renamed its MCP tools ("notion-"
+prefix) and a stale allowlist silently disabled the connector: when the
+allowlist matches zero advertised tools, load everything (HITL-gated)
+instead of nothing.
+"""
+
+from datetime import UTC, datetime
+
+from app.agents.chat.multi_agent_chat.shared.tools.mcp.cache import (
+    CachedMCPTools,
+)
+from app.agents.chat.multi_agent_chat.shared.tools.mcp.tool import (
+    _load_http_mcp_tools,
+)
+
+_CACHED = CachedMCPTools(
+    discovered_at=datetime.now(UTC),
+    tools=[
+        {"name": "notion-search", "description": "search", "input_schema": {}},
+        {"name": "notion-update-page", "description": "write", "input_schema": {}},
+    ],
+)
+_SERVER_CONFIG = {"url": "https://example.com/mcp", "headers": {}}
+
+
+async def test_allowlist_match_filters_and_flags_readonly():
+    tools = await _load_http_mcp_tools(
+        1,
+        "Notion",
+        _SERVER_CONFIG,
+        allowed_tools=["notion-search"],
+        readonly_tools=frozenset({"notion-search"}),
+        cached_tools=_CACHED,
+    )
+    assert [t.name for t in tools] == ["notion-search"]
+    assert tools[0].metadata["hitl"] is False
+
+
+async def test_stale_allowlist_falls_back_to_all_tools_hitl_gated():
+    tools = await _load_http_mcp_tools(
+        1,
+        "Notion",
+        _SERVER_CONFIG,
+        allowed_tools=["search", "update-page"],  # server renamed everything
+        readonly_tools=frozenset({"search"}),
+        cached_tools=_CACHED,
+    )
+    assert sorted(t.name for t in tools) == ["notion-search", "notion-update-page"]
+    # Renamed tools match no readonly entry -> every tool requires approval.
+    assert all(t.metadata["hitl"] is True for t in tools)

surfsense_backend/tests/unit/agents/multi_agent_chat/test_mcp_discovery_migration.py

@@ -0,0 +1,162 @@
+"""Guardrails for the MCP-consolidation migration.
+
+Every MCP-backed connector now routes to the single ``mcp_discovery`` subagent
+(file connectors stay native; Discord/Teams/Luma are deprecated). These tests
+pin the pieces that make that safe: connector→route mapping, any-of gating,
+legacy checkpoint aliasing, tool-name collision prefixing, the metadata-derived
+approval ruleset, and the KB indexing-deprecation sets.
+"""
+
+from __future__ import annotations
+
+import pytest
+from langchain_core.tools import StructuredTool
+
+from app.agents.chat.multi_agent_chat.constants import (
+    CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS,
+    LEGACY_SUBAGENT_ALIASES,
+    SUBAGENT_TO_REQUIRED_CONNECTOR_MAP,
+)
+from app.agents.chat.multi_agent_chat.subagents.builtins.mcp_discovery.tools.index import (
+    NAME as MCP_DISCOVERY_NAME,
+    build_ruleset,
+)
+from app.agents.chat.multi_agent_chat.subagents.mcp_tools.index import (
+    resolve_tool_name_collisions,
+)
+from app.agents.chat.multi_agent_chat.subagents.registry import (
+    SUBAGENT_BUILDERS_BY_NAME,
+    get_subagents_to_exclude,
+)
+from app.services.mcp_oauth.registry import (
+    DEPRECATED_INDEXING_CONNECTOR_TYPES,
+    LIVE_CONNECTOR_TYPES,
+)
+
+pytestmark = pytest.mark.unit
+
+# Connectors that must all funnel into ``mcp_discovery`` (not their own routes).
+_MCP_ROUTED = {
+    "SLACK_CONNECTOR",
+    "JIRA_CONNECTOR",
+    "LINEAR_CONNECTOR",
+    "CLICKUP_CONNECTOR",
+    "AIRTABLE_CONNECTOR",
+    "NOTION_CONNECTOR",
+    "CONFLUENCE_CONNECTOR",
+    "GOOGLE_GMAIL_CONNECTOR",
+    "GOOGLE_CALENDAR_CONNECTOR",
+    "MCP_CONNECTOR",
+}
+
+
+def _tool(name: str, metadata: dict) -> StructuredTool:
+    return StructuredTool.from_function(
+        func=lambda: name,
+        name=name,
+        description=name,
+        metadata=metadata,
+    )
+
+
+def test_all_mcp_connectors_route_to_discovery():
+    for connector_type in _MCP_ROUTED:
+        assert (
+            CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS[connector_type] == MCP_DISCOVERY_NAME
+        ), connector_type
+
+
+def test_file_connectors_keep_native_routes():
+    assert (
+        CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS["GOOGLE_DRIVE_CONNECTOR"]
+        == "google_drive"
+    )
+    assert CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS["DROPBOX_CONNECTOR"] == "dropbox"
+    assert CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS["ONEDRIVE_CONNECTOR"] == "onedrive"
+
+
+def test_deprecated_connectors_have_no_route():
+    for connector_type in ("DISCORD_CONNECTOR", "TEAMS_CONNECTOR", "LUMA_CONNECTOR"):
+        assert connector_type not in CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS
+
+
+def test_discovery_gating_is_any_of():
+    """One connected app is enough to keep ``mcp_discovery``; none excludes it."""
+    assert MCP_DISCOVERY_NAME not in get_subagents_to_exclude(["SLACK_CONNECTOR"])
+    assert MCP_DISCOVERY_NAME in get_subagents_to_exclude([])
+    # A generic user MCP server alone still unlocks it.
+    assert MCP_DISCOVERY_NAME not in get_subagents_to_exclude(["MCP_CONNECTOR"])
+
+
+def test_discovery_gating_tokens_match_routed_connectors():
+    assert SUBAGENT_TO_REQUIRED_CONNECTOR_MAP[MCP_DISCOVERY_NAME] == frozenset(
+        _MCP_ROUTED
+    )
+
+
+def test_legacy_aliases_resolve_to_a_live_subagent():
+    """Old per-connector task names must alias onto a subagent that still exists."""
+    for legacy, target in LEGACY_SUBAGENT_ALIASES.items():
+        assert legacy not in SUBAGENT_BUILDERS_BY_NAME, legacy
+        assert target in SUBAGENT_BUILDERS_BY_NAME, target
+
+
+def test_collision_only_prefixes_shared_names():
+    """A name on two connectors is prefixed; a unique name is left untouched."""
+    tools = [
+        _tool("search", {"mcp_connector_id": 1, "mcp_transport": "http"}),
+        _tool("search", {"mcp_connector_id": 2, "mcp_transport": "http"}),
+        _tool("list_bases", {"mcp_connector_id": 2, "mcp_transport": "http"}),
+    ]
+    resolved = {
+        t.name: t
+        for t in resolve_tool_name_collisions(
+            tools, {1: "NOTION_CONNECTOR", 2: "AIRTABLE_CONNECTOR"}
+        )
+    }
+
+    # The unique tool keeps its bare name (trusted_tools / history stay valid).
+    assert "list_bases" in resolved
+    # The colliding name is gone; both are prefixed with service + connector id.
+    assert "search" not in resolved
+    assert "notion_1_search" in resolved
+    assert "airtable_2_search" in resolved
+    # Original name preserved for the "Always Allow" fallback key.
+    for name in ("notion_1_search", "airtable_2_search"):
+        meta = resolved[name].metadata or {}
+        assert meta["mcp_original_tool_name"] == "search"
+        assert meta["mcp_collision_prefixed"] is True
+
+
+def test_collision_noop_without_collisions():
+    tools = [
+        _tool("a", {"mcp_connector_id": 1}),
+        _tool("b", {"mcp_connector_id": 2}),
+    ]
+    assert [t.name for t in resolve_tool_name_collisions(tools, {})] == ["a", "b"]
+
+
+def test_ruleset_reads_hitl_from_metadata():
+    """Read-only MCP tools ``allow``; every other MCP tool ``ask``; natives skip."""
+    tools = [
+        _tool("readonly_search", {"mcp_transport": "http", "hitl": False}),
+        _tool("mutating_create", {"mcp_transport": "http", "hitl": True}),
+        _tool("native_helper", {}),  # no mcp_transport => no rule
+    ]
+    rules = {r.permission: r.action for r in build_ruleset(tools).rules}
+    assert rules == {"readonly_search": "allow", "mutating_create": "ask"}
+
+
+def test_indexing_deprecation_sets():
+    """Indexing-only connectors are deprecated; migrated ones are LIVE; Obsidian stays."""
+    deprecated = {t.value for t in DEPRECATED_INDEXING_CONNECTOR_TYPES}
+    assert deprecated == {
+        "GITHUB_CONNECTOR",
+        "BOOKSTACK_CONNECTOR",
+        "ELASTICSEARCH_CONNECTOR",
+        "CIRCLEBACK_CONNECTOR",
+    }
+    assert "OBSIDIAN_CONNECTOR" not in deprecated
+
+    live = {t.value for t in LIVE_CONNECTOR_TYPES}
+    assert {"NOTION_CONNECTOR", "CONFLUENCE_CONNECTOR"} <= live

surfsense_backend/tests/unit/agents/multi_agent_chat/test_prompt_resources.py

@@ -45,7 +45,7 @@ def test_every_subagent_has_description_md(name: str):
     [
         "core_behavior.md",
         "routing.md",
-        "tools/web_search/description.md",
+        "tools/task/description.md",
     ],
 )
 def test_main_agent_prompt_fragments_resolve(filename: str):

surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py

@@ -19,36 +19,32 @@ from app.agents.chat.multi_agent_chat.subagents.registry import (
 
 pytestmark = pytest.mark.unit
 
-# The full specialist roster the main agent composes from: 4 builtins + 15
-# connector routes. Adding/removing a specialist is a deliberate product change
-# and must be reflected here.
+# The full specialist roster the main agent composes from after the MCP
+# migration: builtins + the three file-connector routes (Drive/Dropbox/
+# OneDrive). Every MCP-backed connector (Slack/Jira/Linear/ClickUp/Airtable/
+# Notion/Confluence/Gmail/Calendar) now lives behind the single
+# ``mcp_discovery`` route; Discord/Teams/Luma were deprecated. Adding/removing a
+# specialist is a deliberate product change and must be reflected here.
 _EXPECTED_SUBAGENTS = frozenset(
     {
-        "airtable",
-        "calendar",
-        "clickup",
-        "confluence",
         "deliverables",
-        "discord",
         "dropbox",
-        "gmail",
         "google_drive",
-        "jira",
+        "google_maps",
+        "google_search",
         "knowledge_base",
-        "linear",
-        "luma",
+        "mcp_discovery",
         "memory",
-        "notion",
         "onedrive",
-        "research",
-        "slack",
-        "teams",
+        "reddit",
+        "web_crawler",
+        "youtube",
     }
 )
 
 # Specialists that are always available regardless of connected sources, so they
 # carry no required-connector entry.
-_CONNECTORLESS = frozenset({"memory", "research"})
+_CONNECTORLESS = frozenset({"memory"})
 
 
 def test_registry_contains_exactly_expected_subagents():

surfsense_backend/tests/unit/agents/multi_agent_chat/test_web_search_removed.py

@@ -0,0 +1,48 @@
+"""Guardrail: the multi-engine ``web_search`` tool is fully retired.
+
+Public web search now runs exclusively through the ``google_search`` subagent,
+and the four search connector types (Tavily/SearXNG/Linkup/Baidu) are soft-
+deprecated. This pins those invariants so a future change can't quietly bring
+the ``web_search`` tool back or un-deprecate a search connector.
+"""
+
+from __future__ import annotations
+
+import pytest
+from fastapi import HTTPException
+
+from app.agents.chat.multi_agent_chat.main_agent.tools.index import (
+    MAIN_AGENT_SURFSENSE_TOOL_NAMES,
+)
+from app.agents.chat.multi_agent_chat.subagents.registry import (
+    SUBAGENT_BUILDERS_BY_NAME,
+)
+from app.utils.validators import (
+    DEPRECATED_CONNECTOR_TYPES,
+    raise_if_connector_deprecated,
+)
+
+pytestmark = pytest.mark.unit
+
+_DEPRECATED_SEARCH_TYPES = (
+    "TAVILY_API",
+    "SEARXNG_API",
+    "LINKUP_API",
+    "BAIDU_SEARCH_API",
+)
+
+
+def test_web_search_tool_removed_from_main_agent():
+    assert "web_search" not in MAIN_AGENT_SURFSENSE_TOOL_NAMES
+
+
+def test_google_search_specialist_present():
+    assert "google_search" in SUBAGENT_BUILDERS_BY_NAME
+
+
+@pytest.mark.parametrize("connector_type", _DEPRECATED_SEARCH_TYPES)
+def test_search_connectors_deprecated(connector_type):
+    assert connector_type in DEPRECATED_CONNECTOR_TYPES
+    with pytest.raises(HTTPException) as excinfo:
+        raise_if_connector_deprecated(connector_type)
+    assert excinfo.value.status_code == 410

surfsense_backend/tests/unit/agents/new_chat/test_action_log.py

@@ -94,7 +94,7 @@ def fake_session_factory():
 class TestActionLogMiddlewareDisabled:
     @pytest.mark.asyncio
     async def test_no_op_when_flag_off(self, patch_get_flags) -> None:
-        mw = ActionLogMiddleware(thread_id=1, search_space_id=1, user_id=None)
+        mw = ActionLogMiddleware(thread_id=1, workspace_id=1, user_id=None)
         request = _FakeRequest(
             tool_call={
                 "name": "make_widget",
@@ -110,7 +110,7 @@ class TestActionLogMiddlewareDisabled:
 
     @pytest.mark.asyncio
     async def test_no_op_when_thread_id_none(self, patch_get_flags) -> None:
-        mw = ActionLogMiddleware(thread_id=None, search_space_id=1, user_id=None)
+        mw = ActionLogMiddleware(thread_id=None, workspace_id=1, user_id=None)
         request = _FakeRequest(
             tool_call={"name": "make_widget", "args": {}, "id": "tc1"}
         )
@@ -126,7 +126,7 @@ class TestActionLogMiddlewarePersistence:
         self, patch_get_flags, fake_session_factory
     ) -> None:
         captured, factory = fake_session_factory
-        mw = ActionLogMiddleware(thread_id=42, search_space_id=7, user_id="u1")
+        mw = ActionLogMiddleware(thread_id=42, workspace_id=7, user_id="u1")
         request = _FakeRequest(
             tool_call={
                 "name": "make_widget",
@@ -150,7 +150,7 @@ class TestActionLogMiddlewarePersistence:
         assert len(captured["rows"]) == 1
         row = captured["rows"][0]
         assert row.thread_id == 42
-        assert row.search_space_id == 7
+        assert row.workspace_id == 7
         assert row.user_id == "u1"
         assert row.tool_name == "make_widget"
         assert row.args == {"color": "red", "size": 3}
@@ -170,7 +170,7 @@ class TestActionLogMiddlewarePersistence:
     ) -> None:
         """``chat_turn_id`` falls back to NULL when ``runtime.config`` is absent."""
         captured, factory = fake_session_factory
-        mw = ActionLogMiddleware(thread_id=1, search_space_id=1, user_id=None)
+        mw = ActionLogMiddleware(thread_id=1, workspace_id=1, user_id=None)
         request = _FakeRequest(
             tool_call={"name": "make_widget", "args": {}, "id": "tc-1"},
             runtime=None,
@@ -190,7 +190,7 @@ class TestActionLogMiddlewarePersistence:
         self, patch_get_flags, fake_session_factory
     ) -> None:
         captured, factory = fake_session_factory
-        mw = ActionLogMiddleware(thread_id=42, search_space_id=7, user_id="u1")
+        mw = ActionLogMiddleware(thread_id=42, workspace_id=7, user_id="u1")
         request = _FakeRequest(
             tool_call={"name": "make_widget", "args": {"color": "red"}, "id": "tc1"}
         )
@@ -214,7 +214,7 @@ class TestActionLogMiddlewarePersistence:
         self, patch_get_flags
     ) -> None:
         """Even if the DB write blows up, the tool's result must reach the model."""
-        mw = ActionLogMiddleware(thread_id=1, search_space_id=1, user_id=None)
+        mw = ActionLogMiddleware(thread_id=1, workspace_id=1, user_id=None)
         request = _FakeRequest(
             tool_call={"name": "make_widget", "args": {}, "id": "tc1"}
         )
@@ -250,7 +250,7 @@ class TestReverseDescriptor:
         )
         mw = ActionLogMiddleware(
             thread_id=1,
-            search_space_id=1,
+            workspace_id=1,
             user_id="u",
             tool_definitions={"make_widget": tool_def},
         )
@@ -296,7 +296,7 @@ class TestReverseDescriptor:
         )
         mw = ActionLogMiddleware(
             thread_id=1,
-            search_space_id=1,
+            workspace_id=1,
             user_id=None,
             tool_definitions={"make_widget": tool_def},
         )
@@ -321,7 +321,7 @@ class TestReverseDescriptor:
         self, patch_get_flags, fake_session_factory
     ) -> None:
         captured, factory = fake_session_factory
-        mw = ActionLogMiddleware(thread_id=1, search_space_id=1, user_id=None)
+        mw = ActionLogMiddleware(thread_id=1, workspace_id=1, user_id=None)
         request = _FakeRequest(
             tool_call={"name": "unknown_tool", "args": {}, "id": "tc1"}
         )
@@ -343,7 +343,7 @@ class TestActionLogDispatch:
         self, patch_get_flags, fake_session_factory
     ) -> None:
         _captured, factory = fake_session_factory
-        mw = ActionLogMiddleware(thread_id=42, search_space_id=7, user_id="u1")
+        mw = ActionLogMiddleware(thread_id=42, workspace_id=7, user_id="u1")
         request = _FakeRequest(
             tool_call={
                 "name": "make_widget",
@@ -383,7 +383,7 @@ class TestActionLogDispatch:
     @pytest.mark.asyncio
     async def test_no_dispatch_when_persistence_fails(self, patch_get_flags) -> None:
         """If commit fails the dispatch is suppressed (no row to surface)."""
-        mw = ActionLogMiddleware(thread_id=1, search_space_id=1, user_id=None)
+        mw = ActionLogMiddleware(thread_id=1, workspace_id=1, user_id=None)
         request = _FakeRequest(
             tool_call={"name": "make_widget", "args": {}, "id": "tc1"}
         )
@@ -411,7 +411,7 @@ class TestArgsTruncation:
         self, patch_get_flags, fake_session_factory
     ) -> None:
         captured, factory = fake_session_factory
-        mw = ActionLogMiddleware(thread_id=1, search_space_id=1, user_id=None)
+        mw = ActionLogMiddleware(thread_id=1, workspace_id=1, user_id=None)
         # Build a > 32KB string so the persisted payload triggers the truncation path.
         huge = "x" * (40 * 1024)
         request = _FakeRequest(

surfsense_backend/tests/unit/agents/new_chat/test_context_editing.py

@@ -59,7 +59,7 @@ class TestSpillEdit:
         assert edit.pending_spills == []
 
     def test_above_trigger_clears_and_records(self) -> None:
-        edit = SpillToBackendEdit(trigger=100, keep=1, path_prefix="/tool_outputs")
+        edit = SpillToBackendEdit(trigger=100, keep=1)
         msgs = _build_history(4)
         edit.apply(msgs, count_tokens=_approx_count)
 
@@ -102,7 +102,9 @@ class TestSpillEdit:
         assert edit.drain_pending() == []
 
     def test_placeholder_format(self) -> None:
-        path = "/tool_outputs/thread-1/tool-msg-0.txt"
-        text = _build_spill_placeholder(path)
-        assert path in text
-        assert "explore" in text  # mentions the recovery agent
+        import uuid
+
+        spill_id = uuid.uuid4()
+        text = _build_spill_placeholder(spill_id)
+        assert f"spill_{spill_id}" in text
+        assert "read_run" in text  # points at the recovery tools

surfsense_backend/tests/unit/agents/new_chat/test_default_permissions_layering.py

@@ -9,7 +9,7 @@ earliest -> latest:
 3. (future) user-defined rules from the Agent Permissions UI
 
 Without #1 every read-only built-in (``ls``, ``read_file``, ``grep``,
-``glob``, ``web_search`` …) defaulted to ``ask`` because
+``glob`` …) defaulted to ``ask`` because
 ``permissions.evaluate`` returns ``ask`` when no rule matches. That
 caused two production-painful behaviors:
 
@@ -58,8 +58,6 @@ class TestReadOnlyToolsAllowed:
             "read_file",
             "grep",
             "glob",
-            "web_search",
-            "scrape_webpage",
             "get_connected_accounts",
             "write_todos",
             "task",

surfsense_backend/tests/unit/agents/new_chat/test_mention_resolver.py

@@ -99,7 +99,7 @@ class TestResolveMentions:
         session.execute = AsyncMock()
         result = await resolve_mentions(
             session,
-            search_space_id=1,
+            workspace_id=1,
             mentioned_documents=None,
         )
         assert isinstance(result, ResolvedMentionSet)
@@ -134,7 +134,7 @@ class TestResolveMentions:
 
         out = await resolve_mentions(
             session,
-            search_space_id=5,
+            workspace_id=5,
             mentioned_documents=[chip],
         )
         assert len(out.mentions) == 1
@@ -170,7 +170,7 @@ class TestResolveMentions:
 
         out = await resolve_mentions(
             session,
-            search_space_id=3,
+            workspace_id=3,
             mentioned_documents=[chip],
         )
         assert len(out.mentions) == 1
@@ -201,7 +201,7 @@ class TestResolveMentions:
 
         out = await resolve_mentions(
             session,
-            search_space_id=1,
+            workspace_id=1,
             mentioned_documents=[chip],
         )
         assert out.mentions == []
@@ -238,7 +238,7 @@ class TestResolveMentions:
 
         out = await resolve_mentions(
             session,
-            search_space_id=1,
+            workspace_id=1,
             mentioned_documents=[chip_short, chip_long],
         )
         tokens = [tok for tok, _ in out.token_to_path]
@@ -265,7 +265,7 @@ class TestResolveMentions:
 
         out = await resolve_mentions(
             session,
-            search_space_id=2,
+            workspace_id=2,
             mentioned_documents=None,
             mentioned_document_ids=[7],
         )

surfsense_backend/tests/unit/agents/new_chat/test_otel_span.py

@@ -65,8 +65,8 @@ class TestResolveToolName:
     def test_falls_back_to_tool_call_name(self) -> None:
         request = MagicMock()
         request.tool = None
-        request.tool_call = {"name": "web_search", "args": {}}
-        assert _resolve_tool_name(request) == "web_search"
+        request.tool_call = {"name": "create_automation", "args": {}}
+        assert _resolve_tool_name(request) == "create_automation"
 
     def test_unknown_when_nothing_resolves(self) -> None:
         request = MagicMock()
@@ -278,8 +278,8 @@ class TestMiddlewareIntegration:
 
             request = MagicMock()
             request.tool = MagicMock()
-            request.tool.name = "web_search"
+            request.tool.name = "scrape_webpage"
             await mw.awrap_tool_call(request, handler)
-            assert errors == ["web_search"]
+            assert errors == ["scrape_webpage"]
         finally:
             ot.reload_for_tests()

surfsense_backend/tests/unit/agents/new_chat/test_path_resolver.py

@@ -149,7 +149,7 @@ class TestVirtualPathToDoc:
 
         document = await virtual_path_to_doc(
             session,
-            search_space_id=5,
+            workspace_id=5,
             virtual_path=f"{DOCUMENTS_ROOT}/{encoded_basename}",
         )
         assert document is target_doc
@@ -169,7 +169,7 @@ class TestVirtualPathToDoc:
 
         document = await virtual_path_to_doc(
             session,
-            search_space_id=5,
+            workspace_id=5,
             virtual_path=f"{DOCUMENTS_ROOT}/Calendar_ Happy birthday!.xml",
         )
         assert document is None
@@ -191,7 +191,7 @@ class TestVirtualPathToDoc:
 
         document = await virtual_path_to_doc(
             session,
-            search_space_id=5,
+            workspace_id=5,
             virtual_path=f"{DOCUMENTS_ROOT}/Plain Note.xml",
         )
         assert document is target_doc
@@ -217,7 +217,7 @@ class TestVirtualPathToDoc:
 
         document = await virtual_path_to_doc(
             session,
-            search_space_id=5,
+            workspace_id=5,
             virtual_path=f"{DOCUMENTS_ROOT}/2025-W2.pdf.xml",
         )
         assert document is target_doc
@@ -239,7 +239,7 @@ class TestVirtualPathToDoc:
 
         document = await virtual_path_to_doc(
             session,
-            search_space_id=5,
+            workspace_id=5,
             virtual_path=f"{DOCUMENTS_ROOT}/2025-W2.pdf",
         )
         assert document is target_doc

surfsense_backend/tests/unit/agents/new_chat/test_plugin_loader.py

@@ -30,7 +30,7 @@ class _DummyMiddleware(AgentMiddleware):
 
 def _ctx() -> PluginContext:
     return PluginContext.build(
-        search_space_id=1,
+        workspace_id=1,
         user_id="u",
         thread_visibility="PRIVATE",  # type: ignore[arg-type]
         llm=MagicMock(),
@@ -165,12 +165,12 @@ class TestPluginContext:
     def test_build_includes_required_fields(self) -> None:
         llm = MagicMock()
         ctx = PluginContext.build(
-            search_space_id=42,
+            workspace_id=42,
             user_id="user-1",
             thread_visibility="PRIVATE",  # type: ignore[arg-type]
             llm=llm,
         )
-        assert ctx["search_space_id"] == 42
+        assert ctx["workspace_id"] == 42
         assert ctx["user_id"] == "user-1"
         assert ctx["llm"] is llm
 

surfsense_backend/tests/unit/agents/new_chat/test_skills_backends.py

@@ -11,7 +11,7 @@ from app.agents.chat.multi_agent_chat.main_agent.skills.backends import (
     SKILLS_BUILTIN_PREFIX,
     SKILLS_SPACE_PREFIX,
     BuiltinSkillsBackend,
-    SearchSpaceSkillsBackend,
+    WorkspaceSkillsBackend,
     build_skills_backend_factory,
     default_skills_sources,
 )
@@ -176,7 +176,7 @@ class _FakeKBBackend:
         return out
 
 
-class TestSearchSpaceSkillsBackend:
+class TestWorkspaceSkillsBackend:
     def test_remaps_paths_when_listing(self) -> None:
         listing = [
             {"path": "/documents/_skills/policy", "is_dir": True},
@@ -184,7 +184,7 @@ class TestSearchSpaceSkillsBackend:
             {"path": "/documents/other-folder/x.md", "is_dir": False},
         ]
         kb = _FakeKBBackend(listing=listing, file_contents={})
-        backend = SearchSpaceSkillsBackend(kb)
+        backend = WorkspaceSkillsBackend(kb)
         infos = asyncio.run(backend.als_info("/"))
         assert kb.last_ls_path == "/documents/_skills"
         paths = [info["path"] for info in infos]
@@ -200,7 +200,7 @@ class TestSearchSpaceSkillsBackend:
                 "/documents/_skills/policy/SKILL.md": b"---\nname: policy\n---\n",
             },
         )
-        backend = SearchSpaceSkillsBackend(kb)
+        backend = WorkspaceSkillsBackend(kb)
         responses = asyncio.run(backend.adownload_files(["/policy/SKILL.md"]))
         assert kb.last_download_paths == ["/documents/_skills/policy/SKILL.md"]
         assert responses[0].path == "/policy/SKILL.md"
@@ -208,7 +208,7 @@ class TestSearchSpaceSkillsBackend:
         assert responses[0].content is not None
 
     def test_sync_methods_raise_not_implemented(self) -> None:
-        backend = SearchSpaceSkillsBackend(_FakeKBBackend([], {}))
+        backend = WorkspaceSkillsBackend(_FakeKBBackend([], {}))
         with pytest.raises(NotImplementedError):
             backend.ls_info("/")
         with pytest.raises(NotImplementedError):
@@ -221,7 +221,7 @@ class TestSearchSpaceSkillsBackend:
             ],
             file_contents={},
         )
-        backend = SearchSpaceSkillsBackend(kb, kb_root="/skills_admin")
+        backend = WorkspaceSkillsBackend(kb, kb_root="/skills_admin")
         infos = asyncio.run(backend.als_info("/"))
         assert kb.last_ls_path == "/skills_admin"
         assert infos[0]["path"] == "/x"

surfsense_backend/tests/unit/agents/new_chat/tools/test_resume_page_limits.py

@@ -148,7 +148,7 @@ async def test_generate_resume_defaults_to_one_page_target(monkeypatch) -> None:
     monkeypatch.setattr(resume_tool, "_compile_typst", lambda _source: b"pdf")
     monkeypatch.setattr(resume_tool, "_count_pdf_pages", lambda _pdf: 1)
 
-    tool = resume_tool.create_generate_resume_tool(search_space_id=1, thread_id=1)
+    tool = resume_tool.create_generate_resume_tool(workspace_id=1, thread_id=1)
     result = await _invoke(tool, {"user_info": "Jane Doe experience"})
 
     assert result["status"] == "ready"
@@ -178,7 +178,7 @@ async def test_generate_resume_compresses_when_over_limit(monkeypatch) -> None:
     page_counts = iter([2, 1])
     monkeypatch.setattr(resume_tool, "_count_pdf_pages", lambda _pdf: next(page_counts))
 
-    tool = resume_tool.create_generate_resume_tool(search_space_id=1, thread_id=1)
+    tool = resume_tool.create_generate_resume_tool(workspace_id=1, thread_id=1)
     result = await _invoke(tool, {"user_info": "Jane Doe experience", "max_pages": 1})
 
     assert result["status"] == "ready"
@@ -213,7 +213,7 @@ async def test_generate_resume_returns_ready_when_target_not_met(monkeypatch) ->
     page_counts = iter([3, 3, 2])
     monkeypatch.setattr(resume_tool, "_count_pdf_pages", lambda _pdf: next(page_counts))
 
-    tool = resume_tool.create_generate_resume_tool(search_space_id=1, thread_id=1)
+    tool = resume_tool.create_generate_resume_tool(workspace_id=1, thread_id=1)
     result = await _invoke(tool, {"user_info": "Jane Doe experience", "max_pages": 1})
 
     assert result["status"] == "ready"
@@ -246,7 +246,7 @@ async def test_generate_resume_fails_when_hard_limit_exceeded(monkeypatch) -> No
     page_counts = iter([7, 6, 6])
     monkeypatch.setattr(resume_tool, "_count_pdf_pages", lambda _pdf: next(page_counts))
 
-    tool = resume_tool.create_generate_resume_tool(search_space_id=1, thread_id=1)
+    tool = resume_tool.create_generate_resume_tool(workspace_id=1, thread_id=1)
     result = await _invoke(tool, {"user_info": "Jane Doe experience", "max_pages": 1})
 
     assert result["status"] == "failed"

surfsense_backend/tests/unit/automations/actions/builtin/agent_task/test_dependencies.py

@@ -1,10 +1,10 @@
 """Lock the runtime model-policy backstop in ``build_dependencies``.
 
 Automations resolve their LLM from the *captured* ``chat_model_id`` snapshot (so
-runs are insulated from later chat/search-space model changes), and the model
+runs are insulated from later chat/workspace model changes), and the model
 policy is re-checked at run time so a captured model that is no longer billable
 fails the run clearly. When no snapshot is present, resolution falls back to the
-live search space.
+live workspace.
 """
 
 from __future__ import annotations
@@ -25,66 +25,66 @@ pytestmark = pytest.mark.unit
 
 
 class _FakeSession:
-    """Minimal async session whose ``get`` returns a preset search space."""
+    """Minimal async session whose ``get`` returns a preset workspace."""
 
-    def __init__(self, search_space: Any) -> None:
-        self._search_space = search_space
+    def __init__(self, workspace: Any) -> None:
+        self._workspace = workspace
 
     async def get(self, _model: Any, _pk: int) -> Any:
-        return self._search_space
+        return self._workspace
 
 
 @pytest.fixture
 def patched_side_effects(monkeypatch: pytest.MonkeyPatch):
     """Stub the connector setup + checkpointer so only policy/LLM logic runs."""
 
-    async def _fake_setup(_session, *, search_space_id):
-        return (SimpleNamespace(name="connector"), "fc-key")
+    async def _fake_setup(_session, *, workspace_id):
+        return SimpleNamespace(name="connector")
 
-    monkeypatch.setattr(deps_mod, "setup_connector_and_firecrawl", _fake_setup)
+    monkeypatch.setattr(deps_mod, "setup_connector_service", _fake_setup)
     return None
 
 
 async def test_build_dependencies_resolves_captured_chat_model_id(
     monkeypatch: pytest.MonkeyPatch, patched_side_effects
 ) -> None:
-    """The bundle loads with the *captured* ``chat_model_id``, not the live search space."""
+    """The bundle loads with the *captured* ``chat_model_id``, not the live workspace."""
     captured: dict[str, Any] = {}
 
-    async def _fake_load(_session, *, config_id, search_space_id):
+    async def _fake_load(_session, *, config_id, workspace_id):
         captured["config_id"] = config_id
-        captured["search_space_id"] = search_space_id
+        captured["workspace_id"] = workspace_id
         return (SimpleNamespace(name="llm"), SimpleNamespace(name="agent_config"), None)
 
     monkeypatch.setattr(deps_mod, "load_llm_bundle", _fake_load)
     # Captured path validates the explicit ids; passes for this test.
     monkeypatch.setattr(deps_mod, "assert_models_billable", lambda **_kw: None)
-    # A different value on the live search space proves we ignore it when a
+    # A different value on the live workspace proves we ignore it when a
     # snapshot is supplied.
     monkeypatch.setattr(
         deps_mod,
         "assert_automation_models_billable",
-        lambda _ss: pytest.fail("search-space policy should not run on captured path"),
+        lambda _ss: pytest.fail("workspace policy should not run on captured path"),
     )
 
-    search_space = SimpleNamespace(chat_model_id=-99)
+    workspace = SimpleNamespace(chat_model_id=-99)
     result = await build_dependencies(
-        session=_FakeSession(search_space),
-        search_space_id=42,
+        session=_FakeSession(workspace),
+        workspace_id=42,
         chat_model_id=-7,
         image_gen_model_id=5,
         vision_model_id=-1,
     )
 
-    assert captured == {"config_id": -7, "search_space_id": 42}
+    assert captured == {"config_id": -7, "workspace_id": 42}
     assert result.llm.name == "llm"
-    assert result.firecrawl_api_key == "fc-key"
+    assert result.connector_service.name == "connector"
 
 
 async def test_build_dependencies_validates_captured_ids(
     monkeypatch: pytest.MonkeyPatch, patched_side_effects
 ) -> None:
-    """The captured ids (not the search space) are what gets policy-checked."""
+    """The captured ids (not the workspace) are what gets policy-checked."""
     seen: dict[str, Any] = {}
 
     def _capture(**kwargs):
@@ -92,14 +92,14 @@ async def test_build_dependencies_validates_captured_ids(
 
     monkeypatch.setattr(deps_mod, "assert_models_billable", _capture)
 
-    async def _fake_load(_session, *, config_id, search_space_id):
+    async def _fake_load(_session, *, config_id, workspace_id):
         return (SimpleNamespace(name="llm"), SimpleNamespace(name="agent_config"), None)
 
     monkeypatch.setattr(deps_mod, "load_llm_bundle", _fake_load)
 
     await build_dependencies(
         session=_FakeSession(SimpleNamespace(chat_model_id=0)),
-        search_space_id=42,
+        workspace_id=42,
         chat_model_id=-7,
         image_gen_model_id=5,
         vision_model_id=-1,
@@ -132,20 +132,20 @@ async def test_build_dependencies_raises_on_captured_policy_violation(
     with pytest.raises(DependencyError):
         await build_dependencies(
             session=_FakeSession(SimpleNamespace(chat_model_id=-7)),
-            search_space_id=42,
+            workspace_id=42,
             chat_model_id=-7,
             image_gen_model_id=-2,
             vision_model_id=-1,
         )
 
 
-async def test_build_dependencies_falls_back_to_search_space(
+async def test_build_dependencies_falls_back_to_workspace(
     monkeypatch: pytest.MonkeyPatch, patched_side_effects
 ) -> None:
-    """With no captured snapshot, resolve + validate the live search space."""
+    """With no captured snapshot, resolve + validate the live workspace."""
     captured: dict[str, Any] = {}
 
-    async def _fake_load(_session, *, config_id, search_space_id):
+    async def _fake_load(_session, *, config_id, workspace_id):
         captured["config_id"] = config_id
         return (SimpleNamespace(name="llm"), SimpleNamespace(name="agent_config"), None)
 
@@ -157,18 +157,16 @@ async def test_build_dependencies_falls_back_to_search_space(
         lambda **_kw: pytest.fail("captured policy should not run on fallback path"),
     )
 
-    search_space = SimpleNamespace(chat_model_id=-7)
-    result = await build_dependencies(
-        session=_FakeSession(search_space), search_space_id=42
-    )
+    workspace = SimpleNamespace(chat_model_id=-7)
+    result = await build_dependencies(session=_FakeSession(workspace), workspace_id=42)
 
     assert captured == {"config_id": -7}
     assert result.llm.name == "llm"
 
 
-async def test_build_dependencies_raises_when_search_space_missing(
+async def test_build_dependencies_raises_when_workspace_missing(
     patched_side_effects,
 ) -> None:
-    """A missing search space (fallback path) surfaces as a ``DependencyError``."""
+    """A missing workspace (fallback path) surfaces as a ``DependencyError``."""
     with pytest.raises(DependencyError):
-        await build_dependencies(session=_FakeSession(None), search_space_id=999)
+        await build_dependencies(session=_FakeSession(None), workspace_id=999)

surfsense_backend/tests/unit/automations/runtime/test_execute_step.py

@@ -34,7 +34,7 @@ def _action_context() -> ActionContext:
         session=cast(AsyncSession, None),
         run_id=1,
         step_id="s1",
-        search_space_id=1,
+        workspace_id=1,
         creator_user_id=None,
     )
 

surfsense_backend/tests/unit/automations/runtime/test_executor_action_ctx.py

@@ -1,6 +1,6 @@
 """Lock that the executor propagates the captured model snapshot into the
 ``ActionContext``, so runs resolve their own model (insulated from chat /
-search-space changes) and not the live search space.
+workspace changes) and not the live workspace.
 """
 
 from __future__ import annotations
@@ -21,7 +21,7 @@ pytestmark = pytest.mark.unit
 def _run() -> SimpleNamespace:
     return SimpleNamespace(
         id=1,
-        automation=SimpleNamespace(search_space_id=42, created_by_user_id="u-1"),
+        automation=SimpleNamespace(workspace_id=42, created_by_user_id="u-1"),
     )
 
 
@@ -39,7 +39,7 @@ def test_build_action_ctx_propagates_captured_models() -> None:
         models,
     )
 
-    assert ctx.search_space_id == 42
+    assert ctx.workspace_id == 42
     assert ctx.chat_model_id == -1
     assert ctx.image_gen_model_id == 5
     assert ctx.vision_model_id == -1

surfsense_backend/tests/unit/automations/schemas/api/test_api_automation.py

@@ -17,11 +17,11 @@ _VALID_DEFINITION = {
 
 
 def test_automation_create_accepts_valid_minimal_payload() -> None:
-    """Happy path: just search_space_id, name, and a valid definition.
+    """Happy path: just workspace_id, name, and a valid definition.
     Triggers default to ``[]`` so users can attach them later."""
     payload = AutomationCreate.model_validate(
         {
-            "search_space_id": 1,
+            "workspace_id": 1,
             "name": "Daily digest",
             "definition": _VALID_DEFINITION,
         }
@@ -39,7 +39,7 @@ def test_automation_create_cascades_validation_into_nested_definition() -> None:
     with pytest.raises(ValidationError):
         AutomationCreate.model_validate(
             {
-                "search_space_id": 1,
+                "workspace_id": 1,
                 "name": "Bad",
                 "definition": {"name": "X", "plan": []},  # empty plan
             }
@@ -51,7 +51,7 @@ def test_automation_create_rejects_unknown_top_level_field() -> None:
     with pytest.raises(ValidationError):
         AutomationCreate.model_validate(
             {
-                "search_space_id": 1,
+                "workspace_id": 1,
                 "name": "X",
                 "definition": _VALID_DEFINITION,
                 "owner": "tg",  # not allowed
@@ -64,7 +64,7 @@ def test_automation_create_rejects_empty_name() -> None:
     with pytest.raises(ValidationError):
         AutomationCreate.model_validate(
             {
-                "search_space_id": 1,
+                "workspace_id": 1,
                 "name": "",
                 "definition": _VALID_DEFINITION,
             }

surfsense_backend/tests/unit/automations/services/test_automation_service_policy.py

@@ -1,6 +1,6 @@
 """Lock creation-time model-policy enforcement in ``AutomationService``.
 
-Creation (REST + manual builder) rejects search spaces whose models aren't
+Creation (REST + manual builder) rejects workspaces whose models aren't
 billable for automations with HTTP 422, mirroring the runtime backstop. These
 tests isolate the new ``_assert_models_billable`` / ``model_eligibility`` paths
 without touching the DB commit.
@@ -29,13 +29,13 @@ pytestmark = pytest.mark.unit
 
 
 class _FakeSession:
-    def __init__(self, search_space: Any) -> None:
-        self._search_space = search_space
+    def __init__(self, workspace: Any) -> None:
+        self._workspace = workspace
         self.added: list[Any] = []
         self.commits = 0
 
     async def get(self, _model: Any, _pk: int) -> Any:
-        return self._search_space
+        return self._workspace
 
     def add(self, obj: Any) -> None:
         self.added.append(obj)
@@ -44,9 +44,9 @@ class _FakeSession:
         self.commits += 1
 
 
-def _service(search_space: Any) -> AutomationService:
+def _service(workspace: Any) -> AutomationService:
     return AutomationService(
-        session=_FakeSession(search_space),
+        session=_FakeSession(workspace),
         auth=AuthContext.session(SimpleNamespace(id="u-1")),
     )
 
@@ -81,7 +81,7 @@ async def test_assert_models_billable_raises_422_on_violation(
 async def test_assert_models_billable_raises_404_when_missing(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
-    """A missing search space is a 404, not a policy error."""
+    """A missing workspace is a 404, not a policy error."""
     monkeypatch.setattr(
         automation_mod, "assert_automation_models_billable", lambda _ss: None
     )
@@ -93,23 +93,23 @@ async def test_assert_models_billable_raises_404_when_missing(
     assert exc_info.value.status_code == 404
 
 
-async def test_assert_models_billable_returns_search_space_when_ok(
+async def test_assert_models_billable_returns_workspace_when_ok(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
-    """When the policy accepts, the loaded search space is returned for reuse."""
+    """When the policy accepts, the loaded workspace is returned for reuse."""
     monkeypatch.setattr(
         automation_mod, "assert_automation_models_billable", lambda _ss: None
     )
 
-    search_space = SimpleNamespace(chat_model_id=-1)
-    service = _service(search_space)
-    assert await service._assert_models_billable(1) is search_space
+    workspace = SimpleNamespace(chat_model_id=-1)
+    service = _service(workspace)
+    assert await service._assert_models_billable(1) is workspace
 
 
-async def test_create_injects_captured_models_from_search_space(
+async def test_create_injects_captured_models_from_workspace(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
-    """create() snapshots the search space's model prefs onto the definition."""
+    """create() snapshots the workspace's model prefs onto the definition."""
     monkeypatch.setattr(
         automation_mod, "assert_automation_models_billable", lambda _ss: None
     )
@@ -124,14 +124,14 @@ async def test_create_injects_captured_models_from_search_space(
 
     monkeypatch.setattr(AutomationService, "_get_with_triggers_or_raise", _return_added)
 
-    search_space = SimpleNamespace(
+    workspace = SimpleNamespace(
         chat_model_id=-1,
         image_gen_model_id=5,
         vision_model_id=-1,
     )
-    service = _service(search_space)
+    service = _service(workspace)
     payload = AutomationCreate(
-        search_space_id=1,
+        workspace_id=1,
         name="A",
         definition=_definition(),
     )
@@ -148,7 +148,7 @@ async def test_create_injects_captured_models_from_search_space(
 async def test_create_treats_unset_prefs_as_auto_zero(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
-    """``None`` search-space prefs are captured as ``0`` (Auto) ids."""
+    """``None`` workspace prefs are captured as ``0`` (Auto) ids."""
     monkeypatch.setattr(
         automation_mod, "assert_automation_models_billable", lambda _ss: None
     )
@@ -163,13 +163,13 @@ async def test_create_treats_unset_prefs_as_auto_zero(
 
     monkeypatch.setattr(AutomationService, "_get_with_triggers_or_raise", _return_added)
 
-    search_space = SimpleNamespace(
+    workspace = SimpleNamespace(
         chat_model_id=None,
         image_gen_model_id=None,
         vision_model_id=None,
     )
-    service = _service(search_space)
-    payload = AutomationCreate(search_space_id=1, name="A", definition=_definition())
+    service = _service(workspace)
+    payload = AutomationCreate(workspace_id=1, name="A", definition=_definition())
 
     automation = await service.create(payload)
 
@@ -185,7 +185,7 @@ async def test_create_honors_selected_models_when_provided(
 ) -> None:
     """When the payload carries ``definition.models`` they are validated + kept.
 
-    The search-space snapshot path is bypassed entirely (no
+    The workspace snapshot path is bypassed entirely (no
     ``assert_automation_models_billable`` call).
     """
 
@@ -217,7 +217,7 @@ async def test_create_honors_selected_models_when_provided(
 
     service = _service(SimpleNamespace(chat_model_id=-99))
     payload = AutomationCreate(
-        search_space_id=1,
+        workspace_id=1,
         name="A",
         definition=_definition(
             models=AutomationModels(
@@ -257,7 +257,7 @@ async def test_create_rejects_unbillable_selected_models(
 
     service = _service(SimpleNamespace(chat_model_id=-3))
     payload = AutomationCreate(
-        search_space_id=1,
+        workspace_id=1,
         name="A",
         definition=_definition(
             models=AutomationModels(
@@ -284,7 +284,7 @@ async def test_update_preserves_captured_models(
         "vision_model_id": -1,
     }
     existing = SimpleNamespace(
-        search_space_id=1,
+        workspace_id=1,
         definition={"name": "A", "plan": [], "models": captured},
         version=3,
     )
@@ -315,7 +315,7 @@ async def test_update_honors_changed_models_when_valid(
 ) -> None:
     """A definition edit with a *changed* models block validates + keeps it."""
     existing = SimpleNamespace(
-        search_space_id=1,
+        workspace_id=1,
         definition={
             "name": "A",
             "plan": [],
@@ -376,7 +376,7 @@ async def test_update_rejects_changed_unbillable_models(
 ) -> None:
     """A *changed* non-billable models block is rejected with HTTP 422."""
     existing = SimpleNamespace(
-        search_space_id=1,
+        workspace_id=1,
         definition={
             "name": "A",
             "plan": [],
@@ -438,7 +438,7 @@ async def test_update_keeps_unchanged_models_without_revalidation(
         "vision_model_id": -1,
     }
     existing = SimpleNamespace(
-        search_space_id=1,
+        workspace_id=1,
         definition={"name": "A", "plan": [], "models": captured},
         version=3,
     )
@@ -477,7 +477,7 @@ async def test_model_eligibility_authorizes_and_returns_payload(
     authorized: dict[str, Any] = {}
 
     async def _fake_check_permission(_session, _user, ss_id, permission, _msg):
-        authorized["search_space_id"] = ss_id
+        authorized["workspace_id"] = ss_id
         authorized["permission"] = permission
 
     monkeypatch.setattr(automation_mod, "check_permission", _fake_check_permission)
@@ -488,8 +488,8 @@ async def test_model_eligibility_authorizes_and_returns_payload(
     )
 
     service = _service(SimpleNamespace(chat_model_id=-2))
-    result = await service.model_eligibility(search_space_id=5)
+    result = await service.model_eligibility(workspace_id=5)
 
     assert result == {"allowed": False, "violations": [{"kind": "image"}]}
-    assert authorized["search_space_id"] == 5
+    assert authorized["workspace_id"] == 5
     assert authorized["permission"] == "automations:read"

surfsense_backend/tests/unit/automations/services/test_model_policy.py

@@ -24,8 +24,8 @@ from app.automations.services.model_policy import (
 pytestmark = pytest.mark.unit
 
 
-def _search_space(*, llm: int | None, image: int | None, vision: int | None):
-    """Minimal stand-in for the ``SearchSpace`` ORM row the policy reads."""
+def _workspace(*, llm: int | None, image: int | None, vision: int | None):
+    """Minimal stand-in for the ``Workspace`` ORM row the policy reads."""
     return SimpleNamespace(
         chat_model_id=llm,
         image_gen_model_id=image,
@@ -95,15 +95,15 @@ def test_unknown_global_id_is_blocked(kind: str, patched_globals) -> None:
 
 def test_eligibility_all_billable(patched_globals) -> None:
     """Premium LLM + BYOK image + premium vision → allowed, no violations."""
-    search_space = _search_space(llm=-1, image=5, vision=-1)
-    result = get_automation_model_eligibility(search_space)
+    workspace = _workspace(llm=-1, image=5, vision=-1)
+    result = get_automation_model_eligibility(workspace)
     assert result == {"allowed": True, "violations": []}
 
 
 def test_eligibility_reports_each_violation(patched_globals) -> None:
     """A free LLM, Auto image, and free vision each produce a violation."""
-    search_space = _search_space(llm=-2, image=0, vision=-2)
-    result = get_automation_model_eligibility(search_space)
+    workspace = _workspace(llm=-2, image=0, vision=-2)
+    result = get_automation_model_eligibility(workspace)
 
     assert result["allowed"] is False
     kinds = {v["kind"] for v in result["violations"]}
@@ -115,9 +115,9 @@ def test_eligibility_reports_each_violation(patched_globals) -> None:
 
 def test_assert_raises_with_violations(patched_globals) -> None:
     """``assert_automation_models_billable`` raises when any slot is blocked."""
-    search_space = _search_space(llm=0, image=5, vision=-1)
+    workspace = _workspace(llm=0, image=5, vision=-1)
     with pytest.raises(AutomationModelPolicyError) as exc_info:
-        assert_automation_models_billable(search_space)
+        assert_automation_models_billable(workspace)
 
     assert len(exc_info.value.violations) == 1
     assert exc_info.value.violations[0]["kind"] == "chat"
@@ -125,8 +125,8 @@ def test_assert_raises_with_violations(patched_globals) -> None:
 
 def test_assert_passes_when_all_billable(patched_globals) -> None:
     """No exception when every slot is premium or BYOK."""
-    search_space = _search_space(llm=3, image=-1, vision=4)
-    assert assert_automation_models_billable(search_space) is None
+    workspace = _workspace(llm=3, image=-1, vision=4)
+    assert assert_automation_models_billable(workspace) is None
 
 
 # --- ID-based core (used by the runtime backstop against captured snapshots) ---
@@ -170,9 +170,9 @@ def test_assert_models_billable_passes(patched_globals) -> None:
     )
 
 
-def test_search_space_wrapper_delegates_to_core(patched_globals) -> None:
-    """The search-space wrapper produces the same result as the ID core."""
-    search_space = _search_space(llm=-2, image=0, vision=-2)
-    assert get_automation_model_eligibility(search_space) == get_model_eligibility(
+def test_workspace_wrapper_delegates_to_core(patched_globals) -> None:
+    """The workspace wrapper produces the same result as the ID core."""
+    workspace = _workspace(llm=-2, image=0, vision=-2)
+    assert get_automation_model_eligibility(workspace) == get_model_eligibility(
         chat_model_id=-2, image_gen_model_id=0, vision_model_id=-2
     )

surfsense_backend/tests/unit/automations/templating/test_context.py

@@ -25,7 +25,7 @@ def test_build_run_context_exposes_run_inputs_and_steps_namespaces() -> None:
         automation_id=7,
         automation_name="Weekly digest",
         automation_version=3,
-        search_space_id=1,
+        workspace_id=1,
         creator_id=creator,
         trigger_id=11,
         trigger_type="schedule",
@@ -41,7 +41,7 @@ def test_build_run_context_exposes_run_inputs_and_steps_namespaces() -> None:
             "automation_id": 7,
             "automation_name": "Weekly digest",
             "automation_version": 3,
-            "search_space_id": 1,
+            "workspace_id": 1,
             "creator_id": creator,
             "trigger_id": 11,
             "trigger_type": "schedule",

surfsense_backend/tests/unit/automations/triggers/builtin/event/test_filter.py

@@ -21,9 +21,9 @@ def test_scalar_value_is_implicit_equality() -> None:
 
 
 def test_multiple_fields_are_anded() -> None:
-    flt = {"document_type": "FILE", "search_space_id": 7}
-    assert matches(flt, {"document_type": "FILE", "search_space_id": 7}) is True
-    assert matches(flt, {"document_type": "FILE", "search_space_id": 9}) is False
+    flt = {"document_type": "FILE", "workspace_id": 7}
+    assert matches(flt, {"document_type": "FILE", "workspace_id": 7}) is True
+    assert matches(flt, {"document_type": "FILE", "workspace_id": 9}) is False
 
 
 def test_gt_operator_compares_greater_than() -> None:
@@ -97,12 +97,12 @@ def test_missing_field_never_matches_and_never_raises() -> None:
 
 def test_logical_operators_compose_with_fields() -> None:
     flt = {
-        "search_space_id": 7,
+        "workspace_id": 7,
         "$or": [{"document_type": "FILE"}, {"document_type": "WEBPAGE"}],
     }
-    assert matches(flt, {"search_space_id": 7, "document_type": "FILE"}) is True
-    assert matches(flt, {"search_space_id": 9, "document_type": "FILE"}) is False
-    assert matches(flt, {"search_space_id": 7, "document_type": "SLACK"}) is False
+    assert matches(flt, {"workspace_id": 7, "document_type": "FILE"}) is True
+    assert matches(flt, {"workspace_id": 9, "document_type": "FILE"}) is False
+    assert matches(flt, {"workspace_id": 7, "document_type": "SLACK"}) is False
 
 
 def test_unknown_field_operator_raises_filter_error() -> None:

surfsense_backend/tests/unit/automations/triggers/builtin/event/test_inputs.py

@@ -14,7 +14,7 @@ def test_runtime_inputs_flatten_payload_with_event_metadata() -> None:
     event = Event(
         event_type="document.indexed",
         payload={"document_id": 42, "document_type": "FILE"},
-        search_space_id=7,
+        workspace_id=7,
     )
 
     inputs = event_runtime_inputs(event)

surfsense_backend/tests/unit/automations/triggers/builtin/event/test_match.py

@@ -11,7 +11,7 @@ pytestmark = pytest.mark.unit
 
 
 def _event(event_type: str = "document.indexed", **payload) -> Event:
-    return Event(event_type=event_type, payload=payload, search_space_id=7)
+    return Event(event_type=event_type, payload=payload, workspace_id=7)
 
 
 def test_matches_when_event_type_equal_and_filter_passes() -> None:

surfsense_backend/tests/unit/capabilities/access/test_agent_tools.py

@@ -0,0 +1,138 @@
+"""The agent door (05): generate one LangChain tool per registry verb."""
+
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+from pydantic import BaseModel, Field
+
+from app.capabilities.core.types import BillingUnit, Capability
+from app.services.web_crawl_credit_service import InsufficientCreditsError
+
+pytestmark = pytest.mark.asyncio
+
+
+class _EchoInput(BaseModel):
+    text: str = Field(description="The text to echo back.")
+
+
+class _EchoOutput(BaseModel):
+    echoed: str
+
+    @property
+    def billable_units(self) -> int:
+        return 1
+
+
+def _capability(
+    *, name: str, output: _EchoOutput, unit=BillingUnit.WEB_CRAWL
+) -> Capability:
+    async def _executor(payload: _EchoInput) -> _EchoOutput:
+        _executor.seen = payload
+        return output
+
+    cap = Capability(
+        name=name,
+        description=f"{name} does a thing.",
+        input_schema=_EchoInput,
+        output_schema=_EchoOutput,
+        executor=_executor,
+        billing_unit=unit,
+    )
+    cap.executor.seen = None  # type: ignore[attr-defined]
+    return cap
+
+
+class _FakeSessionCtx:
+    async def __aenter__(self):
+        return SimpleNamespace()
+
+    async def __aexit__(self, *exc):
+        return False
+
+
+@pytest.fixture
+def isolate(monkeypatch):
+    """Stub the billing session + charge/gate so tools never hit the DB."""
+    from app.capabilities.core.access import agent as mod
+
+    monkeypatch.setattr(mod, "async_session_maker", lambda: _FakeSessionCtx())
+    charge = AsyncMock()
+    gate = AsyncMock()
+    monkeypatch.setattr(mod, "charge_capability", charge)
+    monkeypatch.setattr(mod, "gate_capability", gate)
+    return SimpleNamespace(module=mod, charge=charge, gate=gate)
+
+
+def _verb_tool(tools, name: str):
+    """Pick one capability tool out of the list (readers are appended after)."""
+    return next(t for t in tools if t.name == name)
+
+
+async def test_registry_becomes_one_tool_per_verb_plus_readers(isolate):
+    caps = [
+        _capability(name="web.scrape", output=_EchoOutput(echoed="a")),
+        _capability(name="web.discover", output=_EchoOutput(echoed="b"), unit=None),
+    ]
+
+    tools = isolate.module.build_capability_tools(workspace_id=7, capabilities=caps)
+
+    by_name = {t.name: t for t in tools}
+    # One tool per verb, plus the two shared run-reader tools.
+    assert set(by_name) == {"web_scrape", "web_discover", "read_run", "search_run"}
+    assert by_name["web_scrape"].description == "web.scrape does a thing."
+    assert by_name["web_scrape"].args_schema is _EchoInput
+
+
+async def test_input_field_docs_reach_the_model(isolate):
+    """Per-field descriptions must surface in the tool's args schema (LLM context)."""
+    cap = _capability(name="web.scrape", output=_EchoOutput(echoed="a"))
+    tools = isolate.module.build_capability_tools(workspace_id=7, capabilities=[cap])
+    tool = _verb_tool(tools, "web_scrape")
+
+    assert tool.args["text"]["description"] == "The text to echo back."
+
+
+async def test_tool_runs_executor_and_returns_serialized_output(isolate):
+    cap = _capability(name="web.scrape", output=_EchoOutput(echoed="hi there"))
+    tools = isolate.module.build_capability_tools(workspace_id=7, capabilities=[cap])
+    tool = _verb_tool(tools, "web_scrape")
+
+    result = await tool.ainvoke({"text": "ping"})
+
+    # Fake session makes record_run fail -> no run_id key, plain serialized output.
+    assert result == {"echoed": "hi there"}
+    assert cap.executor.seen.text == "ping"
+
+
+async def test_tool_charges_owner(isolate):
+    output = _EchoOutput(echoed="hi")
+    cap = _capability(name="web.scrape", output=output)
+    tools = isolate.module.build_capability_tools(workspace_id=7, capabilities=[cap])
+    tool = _verb_tool(tools, "web_scrape")
+
+    await tool.ainvoke({"text": "ping"})
+
+    isolate.charge.assert_awaited_once()
+    (charged_output, unit, ctx), _ = isolate.charge.call_args
+    assert charged_output is output
+    assert unit is BillingUnit.WEB_CRAWL
+    assert ctx.workspace_id == 7
+
+
+async def test_over_budget_returns_friendly_message(isolate):
+    cap = _capability(name="web.scrape", output=_EchoOutput(echoed="hi"))
+    isolate.gate.side_effect = InsufficientCreditsError(
+        message="This run would exceed your available credit.",
+        balance_micros=0,
+        required_micros=1_000_000,
+    )
+    tools = isolate.module.build_capability_tools(workspace_id=7, capabilities=[cap])
+    tool = _verb_tool(tools, "web_scrape")
+
+    result = await tool.ainvoke({"text": "ping"})
+
+    assert isinstance(result, str)
+    assert "credit" in result.lower()
+    assert cap.executor.seen is None
+    isolate.charge.assert_not_awaited()

surfsense_backend/tests/unit/capabilities/access/test_rate_limit.py

@@ -0,0 +1,37 @@
+"""Per-workspace rate limit: a secondary guard behind the credit meter-gate (05)."""
+
+import pytest
+from fastapi import HTTPException
+from starlette.requests import Request
+
+from app.capabilities.core.access import rate_limit
+
+
+def _request(workspace_id: int) -> Request:
+    return Request({"type": "http", "path_params": {"workspace_id": workspace_id}})
+
+
+@pytest.mark.asyncio
+async def test_passes_at_the_limit(monkeypatch):
+    monkeypatch.setattr(
+        rate_limit, "_incr", lambda *a, **k: rate_limit.CAPABILITY_RATE_LIMIT_PER_MINUTE
+    )
+    await rate_limit.enforce_capability_rate_limit(_request(1))
+
+
+@pytest.mark.asyncio
+async def test_blocks_over_the_limit(monkeypatch):
+    monkeypatch.setattr(
+        rate_limit,
+        "_incr",
+        lambda *a, **k: rate_limit.CAPABILITY_RATE_LIMIT_PER_MINUTE + 1,
+    )
+    with pytest.raises(HTTPException) as exc:
+        await rate_limit.enforce_capability_rate_limit(_request(1))
+    assert exc.value.status_code == 429
+
+
+def test_memory_fallback_counts_within_window():
+    rate_limit._memory.clear()
+    assert rate_limit._incr_memory("k", window_seconds=60) == 1
+    assert rate_limit._incr_memory("k", window_seconds=60) == 2

surfsense_backend/tests/unit/capabilities/access/test_rest_router.py

@@ -0,0 +1,505 @@
+"""The REST door generator turns registry verbs into typed POST routes (05)."""
+
+from types import SimpleNamespace
+
+import pytest
+from fastapi import FastAPI
+from httpx import ASGITransport, AsyncClient
+from pydantic import BaseModel
+
+from app.capabilities.core.types import Capability
+from app.db import get_async_session
+from app.users import get_auth_context
+
+
+class _EchoInput(BaseModel):
+    value: str
+
+
+class _EchoOutput(BaseModel):
+    echo: str
+
+
+async def _echo_executor(payload: _EchoInput) -> _EchoOutput:
+    return _EchoOutput(echo=payload.value)
+
+
+_ECHO = Capability(
+    name="test.echo",
+    description="Echo the input back for tests.",
+    input_schema=_EchoInput,
+    output_schema=_EchoOutput,
+    executor=_echo_executor,
+    billing_unit=None,
+)
+
+
+def _build_app(capabilities, monkeypatch) -> FastAPI:
+    """Mount the generated door with auth/workspace/session/rate-limit stubbed."""
+    from app.capabilities.core.access import rest
+    from app.capabilities.core.access.rate_limit import enforce_capability_rate_limit
+
+    monkeypatch.setattr(rest, "check_workspace_access", _noop_async, raising=True)
+    monkeypatch.setattr(rest, "_record_rest_run", _fake_record, raising=True)
+
+    app = FastAPI()
+    app.include_router(rest.build_capabilities_router(capabilities), prefix="/api/v1")
+    app.dependency_overrides[get_auth_context] = lambda: SimpleNamespace(user=None)
+    app.dependency_overrides[enforce_capability_rate_limit] = _allow
+
+    async def _session():
+        yield SimpleNamespace()
+
+    app.dependency_overrides[get_async_session] = _session
+    return app
+
+
+async def _noop_async(*args, **kwargs) -> None:
+    return None
+
+
+async def _fake_record(**kwargs) -> str:
+    """Stand-in for the DB-backed recorder so unit tests never touch a database."""
+    return "test-run-id"
+
+
+async def _allow() -> None:
+    return None
+
+
+def _client(app: FastAPI) -> AsyncClient:
+    return AsyncClient(transport=ASGITransport(app=app), base_url="http://test")
+
+
+@pytest.mark.asyncio
+async def test_verb_is_exposed_as_typed_post_route(monkeypatch):
+    app = _build_app([_ECHO], monkeypatch)
+    async with _client(app) as client:
+        resp = await client.post(
+            "/api/v1/workspaces/7/scrapers/test/echo",
+            json={"value": "hi"},
+        )
+    assert resp.status_code == 200
+    assert resp.json() == {"echo": "hi"}
+    assert resp.headers["X-Run-Id"] == "run_test-run-id"
+
+
+@pytest.mark.asyncio
+async def test_input_is_validated_against_the_verb_schema(monkeypatch):
+    app = _build_app([_ECHO], monkeypatch)
+    async with _client(app) as client:
+        resp = await client.post(
+            "/api/v1/workspaces/7/scrapers/test/echo",
+            json={"wrong": "field"},
+        )
+    assert resp.status_code == 422
+
+
+def test_registered_verbs_appear_on_rest():
+    """A verb in the registry shows up as a route with no per-verb wiring."""
+    import app.capabilities.web  # noqa: F401  (registers web.* at import)
+    from app.capabilities.core.access import rest
+
+    router = rest.build_capabilities_router()
+    paths = {route.path for route in router.routes}
+    assert "/workspaces/{workspace_id}/scrapers/web/crawl" in paths
+
+
+@pytest.mark.asyncio
+async def test_capabilities_endpoint_lists_verbs_with_input_schema(monkeypatch):
+    """The playground reads verb identity + input JSON schema from one GET."""
+    app = _build_app([_ECHO], monkeypatch)
+    async with _client(app) as client:
+        resp = await client.get("/api/v1/workspaces/7/scrapers/capabilities")
+    assert resp.status_code == 200
+    body = resp.json()
+    assert len(body) == 1
+    entry = body[0]
+    assert entry["name"] == "test.echo"
+    assert entry["description"] == "Echo the input back for tests."
+    # The schemas are the pydantic models' JSON schemas: the form renders the
+    # input schema, the API reference docs render both.
+    assert "value" in entry["input_schema"]["properties"]
+    assert "properties" in entry["output_schema"]
+
+
+@pytest.mark.asyncio
+async def test_capabilities_endpoint_exposes_live_pricing(monkeypatch):
+    """Billed verbs report their per-item rate; free verbs report an empty list."""
+    from app.capabilities.core.types import BillingUnit
+    from app.config import config
+
+    monkeypatch.setattr(config, "PLATFORM_SCRAPE_BILLING_ENABLED", True)
+    monkeypatch.setattr(config, "YOUTUBE_MICROS_PER_VIDEO", 2500)
+
+    billed = Capability(
+        name="test.billed",
+        description="Billed verb for tests.",
+        input_schema=_EchoInput,
+        output_schema=_EchoOutput,
+        executor=_echo_executor,
+        billing_unit=BillingUnit.YOUTUBE_VIDEO,
+    )
+
+    app = _build_app([_ECHO, billed], monkeypatch)
+    async with _client(app) as client:
+        resp = await client.get("/api/v1/workspaces/7/scrapers/capabilities")
+    assert resp.status_code == 200
+    by_name = {entry["name"]: entry for entry in resp.json()}
+    assert by_name["test.echo"]["pricing"] == []
+    assert by_name["test.billed"]["pricing"] == [
+        {"unit": "video", "micros_per_unit": 2500}
+    ]
+
+    # Rates are read live: a config retune shows up without a router rebuild.
+    monkeypatch.setattr(config, "PLATFORM_SCRAPE_BILLING_ENABLED", False)
+    async with _client(app) as client:
+        resp = await client.get("/api/v1/workspaces/7/scrapers/capabilities")
+    assert resp.json()[1]["pricing"] == []
+
+
+@pytest.mark.asyncio
+async def test_over_budget_is_blocked_before_the_executor(monkeypatch):
+    from app.capabilities.core.access import rest
+    from app.services.web_crawl_credit_service import InsufficientCreditsError
+
+    async def _raise(*args, **kwargs):
+        raise InsufficientCreditsError(
+            message="over budget", balance_micros=0, required_micros=1000
+        )
+
+    monkeypatch.setattr(rest, "gate_capability", _raise, raising=True)
+
+    app = _build_app([_ECHO], monkeypatch)
+    async with _client(app) as client:
+        resp = await client.post(
+            "/api/v1/workspaces/7/scrapers/test/echo",
+            json={"value": "hi"},
+        )
+    assert resp.status_code == 402
+
+
+@pytest.mark.asyncio
+async def test_rate_limit_blocks_the_workspace(monkeypatch):
+    """The generated route enforces the per-workspace limit (429)."""
+    from app.capabilities.core.access import rate_limit, rest
+
+    monkeypatch.setattr(rest, "check_workspace_access", _noop_async, raising=True)
+    monkeypatch.setattr(
+        rate_limit,
+        "_incr",
+        lambda *a, **k: rate_limit.CAPABILITY_RATE_LIMIT_PER_MINUTE + 1,
+    )
+
+    app = FastAPI()
+    app.include_router(rest.build_capabilities_router([_ECHO]), prefix="/api/v1")
+    app.dependency_overrides[get_auth_context] = lambda: SimpleNamespace(user=None)
+
+    async def _session():
+        yield SimpleNamespace()
+
+    app.dependency_overrides[get_async_session] = _session
+
+    async with _client(app) as client:
+        resp = await client.post(
+            "/api/v1/workspaces/7/scrapers/test/echo",
+            json={"value": "hi"},
+        )
+    assert resp.status_code == 429
+
+
+def _register_surfsense_handler(app: FastAPI) -> None:
+    """Minimal stand-in for the app's global SurfSenseError handler."""
+    from starlette.responses import JSONResponse
+
+    from app.exceptions import SurfSenseError
+
+    async def _handler(_request, exc: SurfSenseError):
+        return JSONResponse(
+            status_code=exc.status_code,
+            content={"code": exc.code, "message": exc.message},
+        )
+
+    app.add_exception_handler(SurfSenseError, _handler)
+
+
+@pytest.mark.asyncio
+async def test_executor_fault_becomes_502(monkeypatch):
+    """Any non-SurfSense executor error is surfaced as a clean 502, not a 500."""
+
+    async def _boom(_payload: _EchoInput) -> _EchoOutput:
+        raise RuntimeError("upstream provider exploded")
+
+    boom = Capability(
+        name="test.boom",
+        description="Always fails for tests.",
+        input_schema=_EchoInput,
+        output_schema=_EchoOutput,
+        executor=_boom,
+        billing_unit=None,
+    )
+
+    app = _build_app([boom], monkeypatch)
+    _register_surfsense_handler(app)
+    async with _client(app) as client:
+        resp = await client.post(
+            "/api/v1/workspaces/7/scrapers/test/boom",
+            json={"value": "hi"},
+        )
+    assert resp.status_code == 502
+    assert resp.json()["code"] == "CAPABILITY_UPSTREAM_ERROR"
+
+
+@pytest.mark.asyncio
+async def test_surfsense_error_passes_through(monkeypatch):
+    """Intentional, status-carrying errors (e.g. a 403 wall) are not remapped."""
+    from app.exceptions import ForbiddenError
+
+    async def _forbidden(_payload: _EchoInput) -> _EchoOutput:
+        raise ForbiddenError("sign in required", code="GOOGLE_SIGNIN_REQUIRED")
+
+    forbidden = Capability(
+        name="test.forbidden",
+        description="Raises a domain 403 for tests.",
+        input_schema=_EchoInput,
+        output_schema=_EchoOutput,
+        executor=_forbidden,
+        billing_unit=None,
+    )
+
+    app = _build_app([forbidden], monkeypatch)
+    _register_surfsense_handler(app)
+    async with _client(app) as client:
+        resp = await client.post(
+            "/api/v1/workspaces/7/scrapers/test/forbidden",
+            json={"value": "hi"},
+        )
+    assert resp.status_code == 403
+    assert resp.json()["code"] == "GOOGLE_SIGNIN_REQUIRED"
+
+
+def _fake_run_row(**overrides):
+    from datetime import UTC, datetime
+    from uuid import uuid4
+
+    defaults = {
+        "id": uuid4(),
+        "capability": "test.echo",
+        "origin": "api",
+        "status": "success",
+        "item_count": 2,
+        "char_count": 42,
+        "duration_ms": 10,
+        "cost_micros": None,
+        "error": None,
+        "created_at": datetime.now(UTC),
+        "thread_id": None,
+        "input": {"value": "hi"},
+        "output_text": '{"echo": "hi"}',
+        "progress": None,
+    }
+    defaults.update(overrides)
+    return SimpleNamespace(**defaults)
+
+
+def _build_app_with_rows(monkeypatch, rows):
+    """App whose fake session answers select() with the given Run-like rows."""
+    from app.capabilities.core.access import rest
+
+    monkeypatch.setattr(rest, "check_workspace_access", _noop_async, raising=True)
+
+    class _Result:
+        def scalars(self):
+            return self
+
+        def all(self):
+            return rows
+
+        def scalar_one_or_none(self):
+            return rows[0] if rows else None
+
+    class _Session:
+        async def execute(self, stmt):
+            return _Result()
+
+    app = FastAPI()
+    app.include_router(rest.build_capabilities_router([]), prefix="/api/v1")
+    app.dependency_overrides[get_auth_context] = lambda: SimpleNamespace(user=None)
+
+    async def _session():
+        yield _Session()
+
+    app.dependency_overrides[get_async_session] = _session
+    return app
+
+
+@pytest.mark.asyncio
+async def test_runs_list_returns_metadata_without_output(monkeypatch):
+    row = _fake_run_row()
+    app = _build_app_with_rows(monkeypatch, [row])
+    async with _client(app) as client:
+        resp = await client.get("/api/v1/workspaces/7/scrapers/runs")
+    assert resp.status_code == 200
+    [item] = resp.json()
+    assert item["id"] == f"run_{row.id}"
+    assert item["capability"] == "test.echo"
+    assert "output_text" not in item  # list is metadata-only
+
+
+@pytest.mark.asyncio
+async def test_run_detail_includes_output(monkeypatch):
+    row = _fake_run_row()
+    app = _build_app_with_rows(monkeypatch, [row])
+    async with _client(app) as client:
+        resp = await client.get(f"/api/v1/workspaces/7/scrapers/runs/run_{row.id}")
+    assert resp.status_code == 200
+    body = resp.json()
+    assert body["output_text"] == '{"echo": "hi"}'
+    assert body["input"] == {"value": "hi"}
+
+
+@pytest.mark.asyncio
+async def test_run_detail_404s(monkeypatch):
+    app = _build_app_with_rows(monkeypatch, [])
+    async with _client(app) as client:
+        missing = await client.get(
+            "/api/v1/workspaces/7/scrapers/runs/run_00000000-0000-0000-0000-000000000000"
+        )
+        malformed = await client.get("/api/v1/workspaces/7/scrapers/runs/garbage")
+    assert missing.status_code == 404
+    assert malformed.status_code == 404  # bad UUID must not become a 500
+
+
+@pytest.mark.asyncio
+async def test_success_charges_once(monkeypatch):
+    from unittest.mock import AsyncMock
+
+    from app.capabilities.core.access import rest
+
+    charge = AsyncMock()
+    monkeypatch.setattr(rest, "charge_capability", charge, raising=True)
+
+    app = _build_app([_ECHO], monkeypatch)
+    async with _client(app) as client:
+        resp = await client.post(
+            "/api/v1/workspaces/7/scrapers/test/echo",
+            json={"value": "hi"},
+        )
+    assert resp.status_code == 200
+    charge.assert_awaited_once()
+    (output, unit, ctx), _ = charge.call_args
+    assert isinstance(output, _EchoOutput)
+    assert unit is None
+    assert ctx.workspace_id == 7
+
+
+@pytest.mark.asyncio
+async def test_async_mode_returns_202_and_pending_run(monkeypatch):
+    """``?mode=async`` inserts a pending run and returns its id without blocking."""
+    from unittest.mock import AsyncMock
+
+    from app.capabilities.core.access import rest
+
+    monkeypatch.setattr(
+        rest, "create_pending_run", AsyncMock(return_value="async-id"), raising=True
+    )
+    # Don't actually run the scrape in the background during this unit test.
+    monkeypatch.setattr(rest, "_execute_async_run", AsyncMock(), raising=True)
+
+    app = _build_app([_ECHO], monkeypatch)
+    async with _client(app) as client:
+        resp = await client.post(
+            "/api/v1/workspaces/7/scrapers/test/echo?mode=async",
+            json={"value": "hi"},
+        )
+    assert resp.status_code == 202
+    assert resp.json() == {"run_id": "run_async-id", "status": "running"}
+
+
+@pytest.mark.asyncio
+async def test_run_events_replays_buffer_then_finishes(monkeypatch):
+    """The SSE endpoint replays buffered events and closes on ``run.finished``."""
+    from app.capabilities.core.events import run_event_bus
+
+    row = _fake_run_row(status="running")
+    raw = str(row.id)
+    run_event_bus.publish(
+        raw, {"type": "run.progress", "phase": "scraping", "current": 1}
+    )
+    run_event_bus.publish(
+        raw, {"type": "run.finished", "status": "success", "item_count": 2}
+    )
+
+    app = _build_app_with_rows(monkeypatch, [row])
+    try:
+        async with _client(app) as client:
+            resp = await client.get(
+                f"/api/v1/workspaces/7/scrapers/runs/run_{raw}/events"
+            )
+        assert resp.status_code == 200
+        body = resp.text
+        assert '"type": "run.progress"' in body
+        assert '"type": "run.finished"' in body
+        assert '"status": "success"' in body
+    finally:
+        run_event_bus.close(raw)
+
+
+@pytest.mark.asyncio
+async def test_cancel_finalizes_running_run(monkeypatch):
+    """Cancel signals the task, finalizes as ``cancelled``, and emits a terminal."""
+    import asyncio
+    from unittest.mock import AsyncMock
+
+    from app.capabilities.core.access import rest
+    from app.capabilities.core.events import run_event_bus
+
+    finalize = AsyncMock(return_value=True)
+    monkeypatch.setattr(rest, "finalize_run", finalize, raising=True)
+
+    row = _fake_run_row(status="running")
+    raw = str(row.id)
+    task = asyncio.create_task(asyncio.sleep(60))
+    run_event_bus.register_task(raw, task)
+
+    app = _build_app_with_rows(monkeypatch, [row])
+    try:
+        async with _client(app) as client:
+            resp = await client.post(
+                f"/api/v1/workspaces/7/scrapers/runs/run_{raw}/cancel"
+            )
+        assert resp.status_code == 200
+        assert resp.json() == {"run_id": f"run_{raw}", "status": "cancelled"}
+        finalize.assert_awaited_once()
+        assert finalize.await_args.kwargs["status"] == "cancelled"
+    finally:
+        task.cancel()
+        await asyncio.gather(task, return_exceptions=True)
+        run_event_bus.close(raw)
+
+
+@pytest.mark.asyncio
+async def test_cancel_conflicts_when_not_running(monkeypatch):
+    """Cancelling a terminal run is a 409, not a silent overwrite."""
+    row = _fake_run_row(status="success")
+    app = _build_app_with_rows(monkeypatch, [row])
+    async with _client(app) as client:
+        resp = await client.post(
+            f"/api/v1/workspaces/7/scrapers/runs/run_{row.id}/cancel"
+        )
+    assert resp.status_code == 409
+
+
+def test_emit_progress_is_a_noop_without_context():
+    """Scraper code can call ``emit_progress`` freely; unset context = no-op."""
+    from app.capabilities.core.progress import emit_progress, progress_scope
+
+    # No active reporter -> returns without raising, records nothing.
+    emit_progress("phase", "message", current=1, total=2, unit="item")
+
+    # Inside a scope, coarse events are buffered for persistence.
+    with progress_scope() as reporter:
+        emit_progress("starting", "go")
+        emit_progress("done", current=5, unit="item")
+    assert len(reporter.coarse) == 2
+    assert reporter.coarse[0]["phase"] == "starting"

surfsense_backend/tests/unit/capabilities/google_maps/reviews/test_executor.py

@@ -0,0 +1,87 @@
+"""``google_maps.reviews`` executor: verb input → actor input mapping → typed items.
+
+Boundary mocked: the proprietary scraper (injected fake). NOT mocked: the verb's
+own payload→GoogleMapsReviewsInput mapping and the dict→ReviewItem wrapping.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from app.capabilities.google_maps.reviews.executor import build_reviews_executor
+from app.capabilities.google_maps.reviews.schemas import ReviewsInput, ReviewsOutput
+from app.exceptions import ForbiddenError
+from app.proprietary.platforms.google_maps import GoogleMapsReviewsInput
+from app.proprietary.platforms.google_maps.scraper import SignInRequiredError
+
+pytestmark = pytest.mark.unit
+
+
+class _FakeScraper:
+    def __init__(self, items: list[dict]):
+        self._items = items
+        self.calls: list[GoogleMapsReviewsInput] = []
+
+    async def __call__(self, actor_input: GoogleMapsReviewsInput) -> list[dict]:
+        self.calls.append(actor_input)
+        return self._items
+
+
+async def test_maps_urls_to_start_urls_and_wraps_items():
+    scraper = _FakeScraper([{"text": "Great place", "stars": 5.0}])
+    execute = build_reviews_executor(scrape_fn=scraper)
+
+    out = await execute(ReviewsInput(urls=["https://www.google.com/maps/place/x"]))
+
+    assert isinstance(out, ReviewsOutput)
+    assert len(out.items) == 1
+    assert out.items[0].text == "Great place"
+    assert out.items[0].stars == 5.0
+
+    (actor_input,) = scraper.calls
+    assert [u.url for u in actor_input.startUrls] == [
+        "https://www.google.com/maps/place/x"
+    ]
+
+
+async def test_forwards_place_ids_and_options():
+    scraper = _FakeScraper([])
+    execute = build_reviews_executor(scrape_fn=scraper)
+
+    await execute(
+        ReviewsInput(
+            place_ids=["ChIJx"],
+            max_reviews=50,
+            sort_by="highestRanking",
+            language="fr",
+            start_date="2024-01-01",
+        )
+    )
+
+    (actor_input,) = scraper.calls
+    assert actor_input.placeIds == ["ChIJx"]
+    assert actor_input.maxReviews == 50
+    assert actor_input.reviewsSort == "highestRanking"
+    assert actor_input.language == "fr"
+    assert actor_input.reviewsStartDate == "2024-01-01"
+
+
+async def test_sign_in_required_maps_to_forbidden_403():
+    async def _raise(_actor_input):
+        raise SignInRequiredError("wall hit")
+
+    execute = build_reviews_executor(scrape_fn=_raise)
+
+    with pytest.raises(ForbiddenError) as exc_info:
+        await execute(ReviewsInput(place_ids=["ChIJx"]))
+    assert exc_info.value.status_code == 403
+
+
+async def test_other_faults_propagate_for_the_door_to_map():
+    async def _boom(_actor_input):
+        raise RuntimeError("proxy exploded")
+
+    execute = build_reviews_executor(scrape_fn=_boom)
+
+    with pytest.raises(RuntimeError):
+        await execute(ReviewsInput(place_ids=["ChIJx"]))

surfsense_backend/tests/unit/capabilities/google_maps/reviews/test_schemas.py

@@ -0,0 +1,35 @@
+"""``google_maps.reviews`` input guards: a source is required and the batch is bounded."""
+
+from __future__ import annotations
+
+import pytest
+from pydantic import ValidationError
+
+from app.capabilities.google_maps.reviews.schemas import (
+    MAX_MAPS_REVIEW_SOURCES,
+    ReviewsInput,
+)
+
+pytestmark = pytest.mark.unit
+
+
+def test_rejects_input_with_no_source():
+    with pytest.raises(ValidationError):
+        ReviewsInput()
+
+
+def test_accepts_urls_or_place_ids():
+    assert ReviewsInput(urls=["https://maps.google.com/x"]).place_ids == []
+    assert ReviewsInput(place_ids=["ChIJx"]).urls == []
+
+
+def test_max_reviews_defaults_and_is_bounded():
+    assert ReviewsInput(place_ids=["ChIJx"]).max_reviews == 20
+    with pytest.raises(ValidationError):
+        ReviewsInput(place_ids=["ChIJx"], max_reviews=0)
+
+
+def test_rejects_more_sources_than_the_cap():
+    too_many = [f"ChIJ{i}" for i in range(MAX_MAPS_REVIEW_SOURCES + 1)]
+    with pytest.raises(ValidationError):
+        ReviewsInput(place_ids=too_many)

surfsense_backend/tests/unit/capabilities/google_maps/scrape/test_executor.py

@@ -0,0 +1,115 @@
+"""``google_maps.scrape`` executor: verb input → actor input mapping → typed items.
+
+Boundary mocked: the proprietary scraper (injected fake). NOT mocked: the verb's
+own payload→GoogleMapsScrapeInput mapping and the dict→PlaceItem wrapping.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from app.capabilities.google_maps.scrape.executor import build_scrape_executor
+from app.capabilities.google_maps.scrape.schemas import ScrapeInput, ScrapeOutput
+from app.exceptions import ForbiddenError
+from app.proprietary.platforms.google_maps import GoogleMapsScrapeInput
+from app.proprietary.platforms.google_maps.scraper import SignInRequiredError
+
+pytestmark = pytest.mark.unit
+
+
+class _FakeScraper:
+    """Records the actor input it was called with and returns canned items."""
+
+    def __init__(self, items: list[dict]):
+        self._items = items
+        self.calls: list[GoogleMapsScrapeInput] = []
+
+    async def __call__(self, actor_input: GoogleMapsScrapeInput) -> list[dict]:
+        self.calls.append(actor_input)
+        return self._items
+
+
+async def test_maps_queries_and_wraps_items():
+    scraper = _FakeScraper([{"title": "Blue Bottle", "placeId": "abc"}])
+    execute = build_scrape_executor(scrape_fn=scraper)
+
+    out = await execute(ScrapeInput(search_queries=["coffee"], location="Austin"))
+
+    assert isinstance(out, ScrapeOutput)
+    assert len(out.items) == 1
+    assert out.items[0].title == "Blue Bottle"
+    assert out.items[0].placeId == "abc"
+
+    (actor_input,) = scraper.calls
+    assert actor_input.searchStringsArray == ["coffee"]
+    assert actor_input.locationQuery == "Austin"
+
+
+async def test_maps_urls_and_place_ids():
+    scraper = _FakeScraper([])
+    execute = build_scrape_executor(scrape_fn=scraper)
+
+    await execute(
+        ScrapeInput(
+            urls=["https://www.google.com/maps/place/x"],
+            place_ids=["ChIJxxxx"],
+        )
+    )
+
+    (actor_input,) = scraper.calls
+    assert [u.url for u in actor_input.startUrls] == [
+        "https://www.google.com/maps/place/x"
+    ]
+    assert actor_input.placeIds == ["ChIJxxxx"]
+
+
+async def test_max_places_maps_to_per_search_cap():
+    scraper = _FakeScraper([])
+    execute = build_scrape_executor(scrape_fn=scraper)
+
+    await execute(ScrapeInput(search_queries=["x"], max_places=25))
+
+    (actor_input,) = scraper.calls
+    assert actor_input.maxCrawledPlacesPerSearch == 25
+
+
+async def test_forwards_detail_review_and_image_options():
+    scraper = _FakeScraper([])
+    execute = build_scrape_executor(scrape_fn=scraper)
+
+    await execute(
+        ScrapeInput(
+            search_queries=["x"],
+            include_details=True,
+            max_reviews=5,
+            max_images=3,
+            language="fr",
+        )
+    )
+
+    (actor_input,) = scraper.calls
+    assert actor_input.scrapePlaceDetailPage is True
+    assert actor_input.maxReviews == 5
+    assert actor_input.maxImages == 3
+    assert actor_input.language == "fr"
+
+
+async def test_sign_in_required_maps_to_forbidden_403():
+    async def _raise(_actor_input):
+        raise SignInRequiredError("wall hit")
+
+    execute = build_scrape_executor(scrape_fn=_raise)
+
+    with pytest.raises(ForbiddenError) as exc_info:
+        await execute(ScrapeInput(search_queries=["x"]))
+    assert exc_info.value.status_code == 403
+
+
+async def test_other_faults_propagate_for_the_door_to_map():
+    async def _boom(_actor_input):
+        raise RuntimeError("proxy exploded")
+
+    execute = build_scrape_executor(scrape_fn=_boom)
+
+    with pytest.raises(RuntimeError):
+        await execute(ScrapeInput(search_queries=["x"]))

surfsense_backend/tests/unit/capabilities/google_maps/scrape/test_schemas.py

@@ -0,0 +1,36 @@
+"""``google_maps.scrape`` input guards: a source is required and the batch is bounded."""
+
+from __future__ import annotations
+
+import pytest
+from pydantic import ValidationError
+
+from app.capabilities.google_maps.scrape.schemas import (
+    MAX_MAPS_SOURCES,
+    ScrapeInput,
+)
+
+pytestmark = pytest.mark.unit
+
+
+def test_rejects_input_with_no_source():
+    with pytest.raises(ValidationError):
+        ScrapeInput()
+
+
+def test_accepts_any_single_source():
+    assert ScrapeInput(search_queries=["coffee"]).urls == []
+    assert ScrapeInput(urls=["https://maps.google.com/x"]).place_ids == []
+    assert ScrapeInput(place_ids=["ChIJx"]).search_queries == []
+
+
+def test_max_places_defaults_and_is_bounded():
+    assert ScrapeInput(search_queries=["x"]).max_places == 10
+    with pytest.raises(ValidationError):
+        ScrapeInput(search_queries=["x"], max_places=0)
+
+
+def test_rejects_more_sources_than_the_cap():
+    too_many = [f"q{i}" for i in range(MAX_MAPS_SOURCES + 1)]
+    with pytest.raises(ValidationError):
+        ScrapeInput(search_queries=too_many)

surfsense_backend/tests/unit/capabilities/google_maps/test_registry.py

@@ -0,0 +1,32 @@
+"""The google_maps namespace registers each verb as one Capability the doors/agent read."""
+
+from __future__ import annotations
+
+import pytest
+
+from app.capabilities import (
+    google_maps,  # noqa: F401  — importing the namespace registers its verbs
+)
+from app.capabilities.core.store import get_capability
+from app.capabilities.google_maps.reviews.schemas import ReviewsInput, ReviewsOutput
+from app.capabilities.google_maps.scrape.schemas import ScrapeInput, ScrapeOutput
+
+pytestmark = pytest.mark.unit
+
+
+def test_google_maps_scrape_is_registered_and_free():
+    cap = get_capability("google_maps.scrape")
+
+    assert cap.name == "google_maps.scrape"
+    assert cap.input_schema is ScrapeInput
+    assert cap.output_schema is ScrapeOutput
+    assert cap.billing_unit is None
+
+
+def test_google_maps_reviews_is_registered_and_free():
+    cap = get_capability("google_maps.reviews")
+
+    assert cap.name == "google_maps.reviews"
+    assert cap.input_schema is ReviewsInput
+    assert cap.output_schema is ReviewsOutput
+    assert cap.billing_unit is None

surfsense_backend/tests/unit/capabilities/reddit/scrape/test_executor.py

@@ -0,0 +1,98 @@
+"""``reddit.scrape`` executor: verb input → actor input mapping → typed items.
+
+Boundary mocked: the proprietary scraper (injected fake). NOT mocked: the verb's
+own payload→RedditScrapeInput mapping and the dict→RedditItem wrapping.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from app.capabilities.reddit.scrape.executor import build_scrape_executor
+from app.capabilities.reddit.scrape.schemas import ScrapeInput, ScrapeOutput
+from app.exceptions import ForbiddenError
+from app.proprietary.platforms.reddit import RedditAccessBlockedError, RedditScrapeInput
+
+pytestmark = pytest.mark.unit
+
+
+class _FakeScraper:
+    """Records the actor input + limit it was called with; returns canned items."""
+
+    def __init__(self, items: list[dict]):
+        self._items = items
+        self.calls: list[tuple[RedditScrapeInput, int | None]] = []
+
+    async def __call__(
+        self, actor_input: RedditScrapeInput, *, limit: int | None = None
+    ) -> list[dict]:
+        self.calls.append((actor_input, limit))
+        return self._items
+
+
+async def test_maps_urls_to_start_urls_and_wraps_items():
+    scraper = _FakeScraper([{"dataType": "post", "id": "abc", "title": "Hello"}])
+    execute = build_scrape_executor(scrape_fn=scraper)
+
+    out = await execute(ScrapeInput(urls=["https://www.reddit.com/r/python/"]))
+
+    assert isinstance(out, ScrapeOutput)
+    assert len(out.items) == 1
+    assert out.items[0].id == "abc"
+    assert out.items[0].title == "Hello"
+    assert out.items[0].dataType == "post"
+
+    (actor_input, _limit) = scraper.calls[0]
+    assert [u.url for u in actor_input.startUrls] == [
+        "https://www.reddit.com/r/python/"
+    ]
+    assert actor_input.searches == []
+
+
+async def test_forwards_search_queries_and_community():
+    scraper = _FakeScraper([])
+    execute = build_scrape_executor(scrape_fn=scraper)
+
+    await execute(ScrapeInput(search_queries=["a", "b"], community="python"))
+
+    (actor_input, _limit) = scraper.calls[0]
+    assert actor_input.searches == ["a", "b"]
+    assert actor_input.searchCommunityName == "python"
+    assert actor_input.startUrls == []
+
+
+async def test_maps_caps_and_passes_limit():
+    scraper = _FakeScraper([])
+    execute = build_scrape_executor(scrape_fn=scraper)
+
+    await execute(
+        ScrapeInput(
+            search_queries=["x"],
+            max_items=25,
+            max_posts=7,
+            max_comments=3,
+            skip_comments=True,
+            sort="top",
+            time_filter="week",
+        )
+    )
+
+    (actor_input, limit) = scraper.calls[0]
+    assert actor_input.maxItems == 25
+    assert actor_input.maxPostCount == 7
+    assert actor_input.maxComments == 3
+    assert actor_input.skipComments is True
+    assert actor_input.sort == "top"
+    assert actor_input.time == "week"
+    # The outer collection limit is the caller's total-item cap.
+    assert limit == 25
+
+
+async def test_access_blocked_maps_to_forbidden():
+    async def _blocked(actor_input: RedditScrapeInput, *, limit: int | None = None):
+        raise RedditAccessBlockedError("all IPs refused")
+
+    execute = build_scrape_executor(scrape_fn=_blocked)
+
+    with pytest.raises(ForbiddenError):
+        await execute(ScrapeInput(search_queries=["x"]))

surfsense_backend/tests/unit/capabilities/reddit/scrape/test_schemas.py

@@ -0,0 +1,51 @@
+"""``reddit.scrape`` input guards: a source is required and the batch is bounded."""
+
+from __future__ import annotations
+
+import pytest
+from pydantic import ValidationError
+
+from app.capabilities.reddit.scrape.schemas import (
+    MAX_REDDIT_ITEMS,
+    MAX_REDDIT_SOURCES,
+    ScrapeInput,
+)
+
+pytestmark = pytest.mark.unit
+
+
+def test_rejects_input_with_no_source():
+    with pytest.raises(ValidationError):
+        ScrapeInput()
+
+
+def test_accepts_urls_only():
+    payload = ScrapeInput(urls=["https://www.reddit.com/r/python/"])
+    assert payload.search_queries == []
+
+
+def test_accepts_search_queries_only():
+    payload = ScrapeInput(search_queries=["notebooklm alternative"])
+    assert payload.urls == []
+
+
+def test_accepts_community_only():
+    payload = ScrapeInput(community="python")
+    assert payload.community == "python"
+
+
+def test_defaults_and_bounds():
+    payload = ScrapeInput(search_queries=["x"])
+    assert payload.max_items == 10
+    assert payload.sort == "new"
+    assert payload.include_nsfw is True
+    with pytest.raises(ValidationError):
+        ScrapeInput(search_queries=["x"], max_items=0)
+    with pytest.raises(ValidationError):
+        ScrapeInput(search_queries=["x"], max_items=MAX_REDDIT_ITEMS + 1)
+
+
+def test_rejects_more_sources_than_the_cap():
+    too_many = [f"https://redd.it/{i}" for i in range(MAX_REDDIT_SOURCES + 1)]
+    with pytest.raises(ValidationError):
+        ScrapeInput(urls=too_many)

surfsense_backend/tests/unit/capabilities/reddit/test_registry.py

@@ -0,0 +1,22 @@
+"""The reddit namespace registers its verb as one Capability the doors/agent read."""
+
+from __future__ import annotations
+
+import pytest
+
+from app.capabilities import (
+    reddit,  # noqa: F401  — importing the namespace registers its verbs
+)
+from app.capabilities.core.store import get_capability
+from app.capabilities.reddit.scrape.schemas import ScrapeInput, ScrapeOutput
+
+pytestmark = pytest.mark.unit
+
+
+def test_reddit_scrape_is_registered_and_free():
+    cap = get_capability("reddit.scrape")
+
+    assert cap.name == "reddit.scrape"
+    assert cap.input_schema is ScrapeInput
+    assert cap.output_schema is ScrapeOutput
+    assert cap.billing_unit is None

surfsense_backend/tests/unit/capabilities/test_billing.py

@@ -0,0 +1,429 @@
+"""Billing charges the workspace owner once per billable success at the executor (03c).
+
+Boundaries mocked: the DB session and the audit helper. NOT mocked: the real
+WebCrawlCreditService debit math and the owner-billed decision.
+"""
+
+from __future__ import annotations
+
+from unittest.mock import AsyncMock, MagicMock
+from uuid import UUID
+
+import pytest
+
+import app.capabilities.core.billing as billing
+from app.capabilities.core.billing import charge_capability, gate_capability
+from app.capabilities.core.types import BillingUnit, CapabilityContext
+from app.capabilities.web.crawl.schemas import CrawlInput, CrawlItem, CrawlOutput
+from app.config import config
+from app.services.web_crawl_credit_service import InsufficientCreditsError
+
+pytestmark = pytest.mark.unit
+
+_WORKSPACE_ID = 1
+_OWNER = UUID("00000000-0000-0000-0000-0000000000bb")
+
+
+class _FakeUser:
+    def __init__(self, balance_micros: int, reserved_micros: int = 0):
+        self.credit_micros_balance = balance_micros
+        self.credit_micros_reserved = reserved_micros
+
+
+def _make_session(owner_id, balance_micros):
+    """Mock session serving owner-resolution and the charge_credits debit."""
+    fake_user = _FakeUser(balance_micros)
+    session = AsyncMock()
+    session.add = MagicMock()
+
+    def _make_result(*_args, **_kwargs):
+        result = MagicMock()
+        result.scalar_one_or_none.return_value = owner_id  # owner resolution
+        result.unique.return_value.scalar_one_or_none.return_value = fake_user  # debit
+        return result
+
+    session.execute = AsyncMock(side_effect=_make_result)
+    return session, fake_user
+
+
+def _output(*statuses: str) -> CrawlOutput:
+    return CrawlOutput(
+        items=[
+            CrawlItem(url=f"https://{i}.com", status=status)
+            for i, status in enumerate(statuses)
+        ]
+    )
+
+
+def _ctx(session) -> CapabilityContext:
+    return CapabilityContext(session=session, workspace_id=_WORKSPACE_ID)
+
+
+@pytest.fixture(autouse=True)
+def _stub_auto_reload(monkeypatch):
+    import app.services.auto_reload_service as ar
+
+    monkeypatch.setattr(ar, "maybe_trigger_auto_reload", AsyncMock())
+
+
+@pytest.fixture
+def record_usage(monkeypatch):
+    rec = AsyncMock(return_value=MagicMock())
+    monkeypatch.setattr(billing, "record_token_usage", rec)
+    return rec
+
+
+async def test_charges_workspace_owner_per_successful_crawl(monkeypatch, record_usage):
+    monkeypatch.setattr(config, "WEB_CRAWL_CREDIT_BILLING_ENABLED", True)
+    monkeypatch.setattr(config, "WEB_CRAWL_MICROS_PER_SUCCESS", 1000)
+    session, user = _make_session(_OWNER, balance_micros=100_000)
+
+    await charge_capability(
+        _output("success", "empty", "success"), BillingUnit.WEB_CRAWL, _ctx(session)
+    )
+
+    # Owner debited 2 * 1000; one web_crawl audit row billed to the OWNER.
+    assert user.credit_micros_balance == 100_000 - 2000
+    record_usage.assert_awaited_once()
+    kwargs = record_usage.await_args.kwargs
+    assert kwargs["usage_type"] == "web_crawl"
+    assert kwargs["user_id"] == _OWNER
+    assert kwargs["workspace_id"] == _WORKSPACE_ID
+    assert kwargs["cost_micros"] == 2000
+
+
+def _output_with_captcha(*statuses: str, attempts: int, solved: int) -> CrawlOutput:
+    out = _output(*statuses)
+    out.captcha_attempts = attempts
+    out.captcha_solved = solved
+    return out
+
+
+async def test_charges_workspace_owner_per_captcha_attempt_even_when_crawl_failed(
+    monkeypatch, record_usage
+):
+    monkeypatch.setattr(config, "WEB_CRAWL_CREDIT_BILLING_ENABLED", True)
+    monkeypatch.setattr(config, "WEB_CRAWL_CAPTCHA_BILLING_ENABLED", True)
+    monkeypatch.setattr(config, "WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE", 3000)
+    session, user = _make_session(_OWNER, balance_micros=100_000)
+
+    # Crawl failed (no billable success) but the solver ran twice — attempts bill.
+    await charge_capability(
+        _output_with_captcha("failed", attempts=2, solved=1),
+        BillingUnit.WEB_CRAWL,
+        _ctx(session),
+    )
+
+    assert user.credit_micros_balance == 100_000 - 2 * 3000
+    record_usage.assert_awaited_once()
+    kwargs = record_usage.await_args.kwargs
+    assert kwargs["usage_type"] == "web_crawl_captcha"
+    assert kwargs["user_id"] == _OWNER
+    assert kwargs["cost_micros"] == 6000
+
+
+async def test_captcha_billing_disabled_does_not_charge_attempts(
+    monkeypatch, record_usage
+):
+    monkeypatch.setattr(config, "WEB_CRAWL_CREDIT_BILLING_ENABLED", True)
+    monkeypatch.setattr(config, "WEB_CRAWL_CAPTCHA_BILLING_ENABLED", False)
+    session, user = _make_session(_OWNER, balance_micros=100_000)
+
+    await charge_capability(
+        _output_with_captcha("failed", attempts=2, solved=1),
+        BillingUnit.WEB_CRAWL,
+        _ctx(session),
+    )
+
+    record_usage.assert_not_awaited()
+    assert user.credit_micros_balance == 100_000
+
+
+async def test_no_successful_rows_is_free(monkeypatch, record_usage):
+    monkeypatch.setattr(config, "WEB_CRAWL_CREDIT_BILLING_ENABLED", True)
+    session, user = _make_session(_OWNER, balance_micros=100_000)
+
+    await charge_capability(
+        _output("empty", "failed"), BillingUnit.WEB_CRAWL, _ctx(session)
+    )
+
+    record_usage.assert_not_awaited()
+    assert user.credit_micros_balance == 100_000
+
+
+async def test_disabled_is_noop(monkeypatch, record_usage):
+    monkeypatch.setattr(config, "WEB_CRAWL_CREDIT_BILLING_ENABLED", False)
+    session, user = _make_session(_OWNER, balance_micros=100_000)
+
+    await charge_capability(
+        _output("success", "success"), BillingUnit.WEB_CRAWL, _ctx(session)
+    )
+
+    record_usage.assert_not_awaited()
+    session.execute.assert_not_called()
+    assert user.credit_micros_balance == 100_000
+
+
+async def test_free_verb_without_a_unit_is_noop(monkeypatch, record_usage):
+    monkeypatch.setattr(config, "WEB_CRAWL_CREDIT_BILLING_ENABLED", True)
+    session, user = _make_session(_OWNER, balance_micros=100_000)
+
+    await charge_capability(_output("success", "success"), None, _ctx(session))
+
+    record_usage.assert_not_awaited()
+    session.execute.assert_not_called()
+    assert user.credit_micros_balance == 100_000
+
+
+def _gate_session(owner_id, balance_micros):
+    """Mock session serving owner-resolution and the spendable-balance read."""
+    session = AsyncMock()
+
+    def _make_result(*_args, **_kwargs):
+        result = MagicMock()
+        result.scalar_one_or_none.return_value = owner_id  # owner resolution
+        result.first.return_value = (balance_micros, 0)  # balance, reserved
+        return result
+
+    session.execute = AsyncMock(side_effect=_make_result)
+    return session
+
+
+async def test_gate_blocks_when_worst_case_exceeds_balance(monkeypatch):
+    monkeypatch.setattr(config, "WEB_CRAWL_CREDIT_BILLING_ENABLED", True)
+    monkeypatch.setattr(config, "WEB_CRAWL_MICROS_PER_SUCCESS", 1000)
+    session = _gate_session(_OWNER, balance_micros=1500)  # affords 1 crawl, not 2
+
+    with pytest.raises(InsufficientCreditsError):
+        await gate_capability(
+            CrawlInput(startUrls=["https://a.com", "https://b.com"]),
+            BillingUnit.WEB_CRAWL,
+            _ctx(session),
+        )
+
+
+async def test_gate_passes_when_balance_covers_worst_case(monkeypatch):
+    monkeypatch.setattr(config, "WEB_CRAWL_CREDIT_BILLING_ENABLED", True)
+    monkeypatch.setattr(config, "WEB_CRAWL_MICROS_PER_SUCCESS", 1000)
+    session = _gate_session(_OWNER, balance_micros=100_000)
+
+    await gate_capability(
+        CrawlInput(startUrls=["https://a.com", "https://b.com"]),
+        BillingUnit.WEB_CRAWL,
+        _ctx(session),
+    )
+
+
+async def test_gate_is_noop_when_disabled(monkeypatch):
+    monkeypatch.setattr(config, "WEB_CRAWL_CREDIT_BILLING_ENABLED", False)
+    session = _gate_session(_OWNER, balance_micros=0)
+
+    await gate_capability(
+        CrawlInput(startUrls=["https://a.com"]), BillingUnit.WEB_CRAWL, _ctx(session)
+    )
+
+
+async def test_gate_reserves_worst_case_captcha_when_solving_enabled(monkeypatch):
+    monkeypatch.setattr(config, "WEB_CRAWL_CREDIT_BILLING_ENABLED", False)
+    monkeypatch.setattr(config, "WEB_CRAWL_CAPTCHA_BILLING_ENABLED", True)
+    monkeypatch.setattr(config, "WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE", 3000)
+    monkeypatch.setattr(config, "CAPTCHA_MAX_ATTEMPTS_PER_URL", 3)
+    monkeypatch.setattr(billing, "captcha_enabled", lambda: True)
+    session = _gate_session(_OWNER, balance_micros=5000)  # < 1 url * 3 * 3000
+
+    with pytest.raises(InsufficientCreditsError):
+        await gate_capability(
+            CrawlInput(startUrls=["https://a.com"]),
+            BillingUnit.WEB_CRAWL,
+            _ctx(session),
+        )
+
+
+async def test_gate_does_not_reserve_captcha_when_solving_disabled(monkeypatch):
+    monkeypatch.setattr(config, "WEB_CRAWL_CREDIT_BILLING_ENABLED", False)
+    monkeypatch.setattr(config, "WEB_CRAWL_CAPTCHA_BILLING_ENABLED", True)
+    monkeypatch.setattr(billing, "captcha_enabled", lambda: False)
+    session = _gate_session(_OWNER, balance_micros=0)
+
+    # Solving off → attempts can never happen → nothing to reserve → passes.
+    await gate_capability(
+        CrawlInput(startUrls=["https://a.com"]), BillingUnit.WEB_CRAWL, _ctx(session)
+    )
+
+
+async def test_gate_is_noop_for_free_verb(monkeypatch):
+    monkeypatch.setattr(config, "WEB_CRAWL_CREDIT_BILLING_ENABLED", True)
+    session = _gate_session(_OWNER, balance_micros=0)
+
+    await gate_capability(CrawlInput(startUrls=["https://a.com"]), None, _ctx(session))
+
+    session.execute.assert_not_called()
+
+
+# ===================================================================
+# Platform scraper per-item billing (Reddit / Search / Maps / YouTube)
+# ===================================================================
+
+
+class _FakePlatformOutput:
+    """Stand-in for a verb output: only the billing-read properties matter."""
+
+    def __init__(self, items: int, attached_review_count: int = 0):
+        self._items = items
+        self._reviews = attached_review_count
+
+    @property
+    def billable_units(self) -> int:
+        return self._items
+
+    @property
+    def attached_review_count(self) -> int:
+        return self._reviews
+
+
+class _FakePlatformInput:
+    """Stand-in for a verb input reporting its worst-case unit counts."""
+
+    def __init__(self, estimated_units: int, estimated_review_units: int = 0):
+        self._units = estimated_units
+        self._review_units = estimated_review_units
+
+    @property
+    def estimated_units(self) -> int:
+        return self._units
+
+    @property
+    def estimated_review_units(self) -> int:
+        return self._review_units
+
+
+@pytest.fixture
+def _enable_platform_billing(monkeypatch):
+    monkeypatch.setattr(config, "PLATFORM_SCRAPE_BILLING_ENABLED", True)
+
+
+async def test_platform_charges_owner_per_item(
+    monkeypatch, record_usage, _enable_platform_billing
+):
+    monkeypatch.setattr(config, "REDDIT_SCRAPE_MICROS_PER_ITEM", 3500)
+    session, user = _make_session(_OWNER, balance_micros=1_000_000)
+
+    charged = await charge_capability(
+        _FakePlatformOutput(3), BillingUnit.REDDIT_ITEM, _ctx(session)
+    )
+
+    assert charged == 3 * 3500
+    assert user.credit_micros_balance == 1_000_000 - 3 * 3500
+    record_usage.assert_awaited_once()
+    kwargs = record_usage.await_args.kwargs
+    assert kwargs["usage_type"] == "reddit_item"
+    assert kwargs["user_id"] == _OWNER
+    assert kwargs["workspace_id"] == _WORKSPACE_ID
+    assert kwargs["cost_micros"] == 3 * 3500
+
+
+async def test_platform_maps_scrape_dual_meters_places_and_reviews(
+    monkeypatch, record_usage, _enable_platform_billing
+):
+    monkeypatch.setattr(config, "GOOGLE_MAPS_MICROS_PER_PLACE", 5000)
+    monkeypatch.setattr(config, "GOOGLE_MAPS_MICROS_PER_REVIEW", 2000)
+    session, user = _make_session(_OWNER, balance_micros=1_000_000)
+
+    # 2 places + 10 attached reviews -> 2*5000 + 10*2000 = 30000.
+    charged = await charge_capability(
+        _FakePlatformOutput(2, attached_review_count=10),
+        BillingUnit.GOOGLE_MAPS_PLACE,
+        _ctx(session),
+    )
+
+    assert charged == 2 * 5000 + 10 * 2000
+    assert user.credit_micros_balance == 1_000_000 - 30_000
+    assert record_usage.await_count == 2
+    usage_types = {c.kwargs["usage_type"] for c in record_usage.await_args_list}
+    assert usage_types == {"google_maps_place", "google_maps_review"}
+
+
+async def test_platform_charge_disabled_is_noop(monkeypatch, record_usage):
+    monkeypatch.setattr(config, "PLATFORM_SCRAPE_BILLING_ENABLED", False)
+    monkeypatch.setattr(config, "REDDIT_SCRAPE_MICROS_PER_ITEM", 3500)
+    session, user = _make_session(_OWNER, balance_micros=1_000_000)
+
+    charged = await charge_capability(
+        _FakePlatformOutput(3), BillingUnit.REDDIT_ITEM, _ctx(session)
+    )
+
+    assert charged == 0
+    record_usage.assert_not_awaited()
+    session.execute.assert_not_called()
+    assert user.credit_micros_balance == 1_000_000
+
+
+async def test_platform_no_items_is_free(
+    monkeypatch, record_usage, _enable_platform_billing
+):
+    monkeypatch.setattr(config, "YOUTUBE_MICROS_PER_COMMENT", 3500)
+    session, user = _make_session(_OWNER, balance_micros=1_000_000)
+
+    charged = await charge_capability(
+        _FakePlatformOutput(0), BillingUnit.YOUTUBE_COMMENT, _ctx(session)
+    )
+
+    assert charged == 0
+    record_usage.assert_not_awaited()
+    assert user.credit_micros_balance == 1_000_000
+
+
+async def test_platform_gate_blocks_when_worst_case_exceeds_balance(
+    monkeypatch, _enable_platform_billing
+):
+    monkeypatch.setattr(config, "GOOGLE_SEARCH_MICROS_PER_SERP", 5500)
+    session = _gate_session(_OWNER, balance_micros=6000)  # affords 1 SERP, not 2
+
+    with pytest.raises(InsufficientCreditsError):
+        await gate_capability(
+            _FakePlatformInput(estimated_units=2),
+            BillingUnit.GOOGLE_SEARCH_SERP,
+            _ctx(session),
+        )
+
+
+async def test_platform_gate_maps_reserves_places_plus_reviews(
+    monkeypatch, _enable_platform_billing
+):
+    monkeypatch.setattr(config, "GOOGLE_MAPS_MICROS_PER_PLACE", 5000)
+    monkeypatch.setattr(config, "GOOGLE_MAPS_MICROS_PER_REVIEW", 2000)
+    # 1 place (5000) + 10 worst-case reviews (20000) = 25000 required.
+    session = _gate_session(_OWNER, balance_micros=20_000)
+
+    with pytest.raises(InsufficientCreditsError):
+        await gate_capability(
+            _FakePlatformInput(estimated_units=1, estimated_review_units=10),
+            BillingUnit.GOOGLE_MAPS_PLACE,
+            _ctx(session),
+        )
+
+
+async def test_platform_gate_passes_when_affordable(
+    monkeypatch, _enable_platform_billing
+):
+    monkeypatch.setattr(config, "GOOGLE_SEARCH_MICROS_PER_SERP", 5500)
+    session = _gate_session(_OWNER, balance_micros=1_000_000)
+
+    await gate_capability(
+        _FakePlatformInput(estimated_units=2),
+        BillingUnit.GOOGLE_SEARCH_SERP,
+        _ctx(session),
+    )
+
+
+async def test_platform_gate_disabled_is_noop(monkeypatch):
+    monkeypatch.setattr(config, "PLATFORM_SCRAPE_BILLING_ENABLED", False)
+    session = _gate_session(_OWNER, balance_micros=0)
+
+    await gate_capability(
+        _FakePlatformInput(estimated_units=1000),
+        BillingUnit.REDDIT_ITEM,
+        _ctx(session),
+    )
+
+    session.execute.assert_not_called()

surfsense_backend/tests/unit/capabilities/test_registry.py

@@ -0,0 +1,23 @@
+"""The registry exposes each verb as one Capability entry the doors/agent read from."""
+
+from __future__ import annotations
+
+import pytest
+
+from app.capabilities import (
+    web,  # noqa: F401  — importing the namespace registers its verbs
+)
+from app.capabilities.core.store import get_capability
+from app.capabilities.core.types import BillingUnit
+from app.capabilities.web.crawl.schemas import CrawlInput, CrawlOutput
+
+pytestmark = pytest.mark.unit
+
+
+def test_web_crawl_is_registered_with_its_schemas_and_billing_unit():
+    cap = get_capability("web.crawl")
+
+    assert cap.name == "web.crawl"
+    assert cap.input_schema is CrawlInput
+    assert cap.output_schema is CrawlOutput
+    assert cap.billing_unit is BillingUnit.WEB_CRAWL

surfsense_backend/tests/unit/capabilities/test_run_truncation.py

@@ -0,0 +1,346 @@
+"""Tool-boundary truncation + run read-tool behavior (no DB).
+
+Covers the pure pieces of the DB-backed run log: JSONL serialization, the
+char-budgeted preview (including the single-oversized-item case and the
+storage-failure degrade), and the ``read_run``/``search_run`` tools' paging,
+search, ReDoS fallback, and workspace scoping — all with a fake session so the
+unit suite never touches a database.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import json
+
+import pytest
+from pydantic import BaseModel
+
+from app.agents.chat.multi_agent_chat.subagents.shared import run_reader
+from app.capabilities.core.access.agent import _build_preview
+from app.capabilities.core.runs import (
+    RUN_OUTPUT_CHAR_CAP,
+    SerializedOutput,
+    serialize_output,
+)
+
+pytestmark = pytest.mark.unit
+
+
+class _Item(BaseModel):
+    id: int
+    name: str
+    note: str | None = None
+
+
+class _Output(BaseModel):
+    items: list[_Item]
+
+
+class _Scalar(BaseModel):
+    value: str
+
+
+def test_serialize_output_is_jsonl_and_excludes_none():
+    out = _Output(items=[_Item(id=1, name="a"), _Item(id=2, name="b", note="x")])
+    result = serialize_output(out)
+
+    lines = result.text.split("\n")
+    assert result.item_count == 2
+    assert len(lines) == 2
+    # exclude_none: the first item has no "note" key
+    assert "note" not in json.loads(lines[0])
+    assert json.loads(lines[1])["note"] == "x"
+    assert result.char_count == len(result.text)
+
+
+def test_serialize_output_without_items_is_single_line():
+    result = serialize_output(_Scalar(value="hi"))
+    assert result.item_count == 1
+    assert json.loads(result.text) == {"value": "hi"}
+
+
+def test_preview_is_char_budgeted_and_references_run():
+    # Many small items whose total blows the cap.
+    per_item = "y" * 500
+    items = [f'{{"i": {i}, "v": "{per_item}"}}' for i in range(500)]
+    body = "\n".join(items)
+    serialized = SerializedOutput(
+        text=body, item_count=len(items), char_count=len(body)
+    )
+
+    preview = _build_preview(serialized, run_id="abc")
+
+    assert len(preview) < serialized.char_count
+    assert "run_abc" in preview
+    assert "read_run" in preview
+    # Only a prefix of items is shown.
+    assert preview.count('"i":') < len(items)
+
+
+def test_preview_handles_single_oversized_item():
+    huge = "z" * (RUN_OUTPUT_CHAR_CAP * 2)
+    serialized = SerializedOutput(text=huge, item_count=1, char_count=len(huge))
+
+    preview = _build_preview(serialized, run_id="big")
+
+    # Still returns a clipped head rather than nothing.
+    assert "z" in preview
+    assert "run_big" in preview
+    assert len(preview) < serialized.char_count
+
+
+def test_preview_degrades_when_storage_failed():
+    body = "\n".join(f'{{"i": {i}}}' for i in range(200))
+    serialized = SerializedOutput(text=body, item_count=200, char_count=len(body))
+
+    preview = _build_preview(serialized, run_id=None)
+
+    assert "storage error" in preview
+    assert "run_" not in preview
+
+
+# --- read tools -----------------------------------------------------------
+
+
+class _FakeResult:
+    def __init__(self, value):
+        self._value = value
+
+    def scalar_one_or_none(self):
+        return self._value
+
+
+class _FakeSession:
+    def __init__(self, value, calls):
+        self._value = value
+        self._calls = calls
+
+    async def execute(self, stmt):
+        self._calls.append(str(stmt))
+        return _FakeResult(self._value)
+
+
+def _patch_session(monkeypatch, value, calls):
+    @contextlib.asynccontextmanager
+    async def _maker():
+        yield _FakeSession(value, calls)
+
+    monkeypatch.setattr(run_reader, "shielded_async_session", _maker)
+
+
+def _tools():
+    read_run, search_run, _export_run = run_reader.build_run_reader_tools(
+        workspace_id=7
+    )
+    return read_run, search_run
+
+
+_BODY = "\n".join(f'{{"i": {i}, "name": "item_{i}"}}' for i in range(10))
+
+
+@pytest.mark.asyncio
+async def test_read_run_paginates(monkeypatch):
+    calls: list[str] = []
+    _patch_session(monkeypatch, _BODY, calls)
+    read_run, _ = _tools()
+
+    out = await read_run.ainvoke(
+        {
+            "ref": "run_" + "0" * 8 + "-0000-0000-0000-000000000000",
+            "offset": 2,
+            "limit": 3,
+        }
+    )
+    assert "item_2" in out and "item_3" in out and "item_4" in out
+    assert "item_0" not in out and "item_5" not in out
+    # Scoped by workspace_id in the query.
+    assert "workspace_id" in calls[0]
+
+
+@pytest.mark.asyncio
+async def test_read_run_char_offset_pages_inside_one_huge_item(monkeypatch):
+    """A single item bigger than the cap is fully reachable via char_offset."""
+    huge_line = "A" * RUN_OUTPUT_CHAR_CAP + "MARKER" + "B" * 1000
+    _patch_session(monkeypatch, huge_line, [])
+    read_run, _ = _tools()
+    ref = "run_" + "0" * 8 + "-0000-0000-0000-000000000000"
+
+    first = await read_run.ainvoke({"ref": ref, "offset": 0, "limit": 1})
+    assert "MARKER" not in first  # clipped at the cap
+    assert f"char_offset={RUN_OUTPUT_CHAR_CAP}" in first  # continuation hint
+
+    second = await read_run.ainvoke(
+        {"ref": ref, "offset": 0, "limit": 1, "char_offset": RUN_OUTPUT_CHAR_CAP}
+    )
+    assert "MARKER" in second
+    assert "truncated" not in second  # remainder fits
+
+    past_end = await read_run.ainvoke(
+        {"ref": ref, "offset": 0, "limit": 1, "char_offset": len(huge_line) + 5}
+    )
+    assert "No content at char_offset" in past_end
+
+
+@pytest.mark.asyncio
+async def test_search_run_excerpts_huge_matched_line(monkeypatch):
+    """A match inside a huge line returns a window around it, not the whole line."""
+    huge_line = "x" * 100_000 + "NEEDLE" + "y" * 100_000
+    _patch_session(monkeypatch, huge_line, [])
+    _, search_run = _tools()
+
+    out = await search_run.ainvoke(
+        {"ref": "run_" + "0" * 8 + "-0000-0000-0000-000000000000", "pattern": "NEEDLE"}
+    )
+    assert "NEEDLE" in out
+    assert "match at char 100000" in out
+    assert len(out) < 2000  # excerpt, not the 200k line
+
+
+@pytest.mark.asyncio
+async def test_read_run_rejects_bad_ref(monkeypatch):
+    _patch_session(monkeypatch, _BODY, [])
+    read_run, _ = _tools()
+    out = await read_run.ainvoke({"ref": "not-a-ref"})
+    assert "not a valid run reference" in out
+
+
+@pytest.mark.asyncio
+async def test_read_run_not_found(monkeypatch):
+    _patch_session(monkeypatch, None, [])
+    read_run, _ = _tools()
+    out = await read_run.ainvoke(
+        {"ref": "run_" + "0" * 8 + "-0000-0000-0000-000000000000"}
+    )
+    assert "not found" in out
+
+
+@pytest.mark.asyncio
+async def test_search_run_matches(monkeypatch):
+    _patch_session(monkeypatch, _BODY, [])
+    _, search_run = _tools()
+    out = await search_run.ainvoke(
+        {
+            "ref": "spill_" + "0" * 8 + "-0000-0000-0000-000000000000",
+            "pattern": "item_7",
+        }
+    )
+    assert "item_7" in out
+    assert "item_1" not in out.split("item_7")[0]
+
+
+# --- export_run ------------------------------------------------------------
+
+
+_CRAWL_BODY = "\n".join(
+    [
+        json.dumps(
+            {
+                "url": "https://x.com/team/",
+                "status": "success",
+                "links": [
+                    {
+                        "url": "https://x.com/author/jane/",
+                        "text": "Jane Doe",
+                        "context": "Jane Doe General Partner",
+                        "kind": "internal",
+                    },
+                    {
+                        "url": "https://x.com/author/bob/",
+                        "text": "Bob Roe",
+                        "context": "Bob Roe Operations",
+                        "kind": "internal",
+                    },
+                    # Duplicate of Jane (nav + card) — must dedupe.
+                    {
+                        "url": "https://x.com/author/jane/",
+                        "text": "Jane Doe",
+                        "context": "Jane Doe General Partner",
+                        "kind": "internal",
+                    },
+                    {
+                        "url": "https://x.com/about/",
+                        "text": "About",
+                        "kind": "internal",
+                    },
+                ],
+            }
+        ),
+        json.dumps({"url": "https://x.com/jobs/", "status": "failed", "links": []}),
+        "not json — skipped",
+    ]
+)
+
+
+def test_rows_from_body_links_explode_and_items():
+    links = run_reader._rows_from_body(_CRAWL_BODY, "links")
+    assert len(links) == 4
+    assert links[0]["page"] == "https://x.com/team/"
+    assert links[0]["text"] == "Jane Doe"
+
+    items = run_reader._rows_from_body(_CRAWL_BODY, "items")
+    assert [i["url"] for i in items] == ["https://x.com/team/", "https://x.com/jobs/"]
+
+
+def test_rows_to_csv_dedupes_and_orders_columns():
+    records = run_reader._rows_from_body(_CRAWL_BODY, "links")
+    csv_text, count = run_reader._rows_to_csv(records, ["page", "url", "text"])
+    lines = csv_text.strip().split("\n")
+    assert lines[0] == "page,url,text"
+    assert count == 3  # 4 records - 1 duplicate
+    assert len(lines) == 4  # header + 3 rows
+    assert "Jane Doe" in lines[1]
+
+
+@pytest.mark.asyncio
+async def test_export_run_filters_and_saves(monkeypatch):
+    _patch_session(monkeypatch, _CRAWL_BODY, [])
+    saved: dict = {}
+
+    async def _fake_save(*, virtual_path, content, workspace_id):
+        saved["path"] = virtual_path
+        saved["content"] = content
+        saved["workspace_id"] = workspace_id
+        return 42, "/documents/exports/team.csv"
+
+    monkeypatch.setattr(run_reader, "_save_export_document", _fake_save)
+    _, _, export_run = run_reader.build_run_reader_tools(workspace_id=7)
+
+    out = await export_run.ainvoke(
+        {
+            "ref": "run_" + "0" * 8 + "-0000-0000-0000-000000000000",
+            "path": "exports/team.csv",
+            "rows": "links",
+            "include_pattern": "/author/",
+        }
+    )
+    assert "Exported 2 rows" in out  # Jane + Bob; About filtered; dupe deduped
+    assert "/documents/exports/team.csv" in out
+    assert "document id 42" in out
+    assert saved["workspace_id"] == 7
+    assert "About" not in saved["content"]
+    assert "Bob Roe" in saved["content"]
+
+
+@pytest.mark.asyncio
+async def test_export_run_empty_filter_is_error(monkeypatch):
+    _patch_session(monkeypatch, _CRAWL_BODY, [])
+    _, _, export_run = run_reader.build_run_reader_tools(workspace_id=7)
+    out = await export_run.ainvoke(
+        {
+            "ref": "run_" + "0" * 8 + "-0000-0000-0000-000000000000",
+            "path": "exports/none.csv",
+            "include_pattern": "no-such-thing-anywhere",
+        }
+    )
+    assert out.startswith("Error: no rows to export")
+
+
+@pytest.mark.asyncio
+async def test_search_run_falls_back_on_bad_regex(monkeypatch):
+    _patch_session(monkeypatch, _BODY, [])
+    _, search_run = _tools()
+    # "(" is an invalid regex -> substring fallback, must not raise.
+    out = await search_run.ainvoke(
+        {"ref": "run_" + "0" * 8 + "-0000-0000-0000-000000000000", "pattern": "("}
+    )
+    assert "matched" in out or "No lines" in out

surfsense_backend/tests/unit/capabilities/web/crawl/test_executor.py

@@ -0,0 +1,166 @@
+"""``web.crawl`` executor behavior: CrawlPage list → typed CrawlOutput items.
+
+Boundary mocked: the crawler engine (fake ``crawl_url`` + link graph). NOT
+mocked: the executor's page→item mapping, truncation, and captcha rollup.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from app.capabilities.web.crawl.executor import build_crawl_executor
+from app.capabilities.web.crawl.schemas import CrawlInput, CrawlOutput
+from app.proprietary.web_crawler import CrawlOutcome, CrawlOutcomeStatus
+
+pytestmark = pytest.mark.unit
+
+_SUCCESS = CrawlOutcomeStatus.SUCCESS
+
+
+class _FakeEngine:
+    def __init__(self, graph: dict[str, tuple[CrawlOutcomeStatus, list[str]]]):
+        self._graph = graph
+        self.calls: list[str] = []
+
+    async def crawl_url(self, url: str) -> CrawlOutcome:
+        self.calls.append(url)
+        status, links = self._graph[url]
+        if status is _SUCCESS:
+            return CrawlOutcome(
+                status=_SUCCESS,
+                result={
+                    "content": f"C:{url}",
+                    "metadata": {"title": url},
+                    "links": links,
+                },
+            )
+        return CrawlOutcome(status=status, error="boom")
+
+
+async def test_single_url_depth_zero_returns_one_item() -> None:
+    engine = _FakeEngine({"https://e.com/": (_SUCCESS, ["https://e.com/a"])})
+    execute = build_crawl_executor(engine=engine)
+
+    out = await execute(CrawlInput(startUrls=["https://e.com/"]))
+
+    assert isinstance(out, CrawlOutput)
+    assert len(out.items) == 1
+    item = out.items[0]
+    assert item.url == "https://e.com/"
+    assert item.status == "success"
+    assert item.markdown == "C:https://e.com/"
+    assert item.metadata == {"title": "https://e.com/"}
+    assert item.crawl is not None
+    assert item.crawl.depth == 0
+    assert item.crawl.referrerUrl is None
+
+
+async def test_spider_collects_multiple_pages_with_provenance() -> None:
+    engine = _FakeEngine(
+        {
+            "https://e.com/": (_SUCCESS, ["https://e.com/a"]),
+            "https://e.com/a": (_SUCCESS, []),
+        }
+    )
+    execute = build_crawl_executor(engine=engine)
+
+    out = await execute(
+        CrawlInput(startUrls=["https://e.com/"], maxCrawlDepth=1, maxCrawlPages=10)
+    )
+
+    by_url = {item.url: item for item in out.items}
+    assert set(by_url) == {"https://e.com/", "https://e.com/a"}
+    assert by_url["https://e.com/a"].crawl.referrerUrl == "https://e.com/"
+
+
+async def test_content_is_truncated_to_max_length() -> None:
+    engine = _FakeEngine({"https://e.com/": (_SUCCESS, [])})
+    execute = build_crawl_executor(engine=engine)
+
+    out = await execute(CrawlInput(startUrls=["https://e.com/"], maxLength=3))
+
+    assert out.items[0].markdown == "C:h"
+
+
+async def test_failed_page_has_no_markdown_but_keeps_error() -> None:
+    engine = _FakeEngine({"https://e.com/": (CrawlOutcomeStatus.FAILED, [])})
+    execute = build_crawl_executor(engine=engine)
+
+    out = await execute(CrawlInput(startUrls=["https://e.com/"]))
+
+    item = out.items[0]
+    assert item.status == "failed"
+    assert item.markdown is None
+    assert item.error == "boom"
+
+
+async def test_aggregated_contacts_carry_provenance_and_site_wide_flag() -> None:
+    footer = "https://linkedin.com/company/e"
+    person = "https://linkedin.com/in/jane"
+
+    class _ContactsEngine:
+        async def crawl_url(self, url: str) -> CrawlOutcome:
+            socials = [footer] + ([person] if url.endswith("/about") else [])
+            links = (
+                ["https://e.com/about", "https://e.com/blog"]
+                if url == "https://e.com/"
+                else []
+            )
+            return CrawlOutcome(
+                status=_SUCCESS,
+                result={
+                    "content": "ok",
+                    "metadata": {},
+                    "links": links,
+                    "contacts": {"emails": [], "phones": [], "socials": socials},
+                },
+            )
+
+    execute = build_crawl_executor(engine=_ContactsEngine())
+    out = await execute(
+        CrawlInput(startUrls=["https://e.com/"], maxCrawlDepth=1, maxCrawlPages=10)
+    )
+
+    by_value = {ref.value: ref for ref in out.contacts.socials}
+    assert by_value[footer].siteWide  # on all 3 pages -> boilerplate
+    assert by_value[footer].pageCount == 3
+    assert not by_value[person].siteWide  # only on /about -> page-local entity
+    assert by_value[person].pages == ["https://e.com/about"]
+
+
+async def test_single_page_crawl_marks_contacts_site_wide() -> None:
+    class _OnePageEngine:
+        async def crawl_url(self, url: str) -> CrawlOutcome:
+            return CrawlOutcome(
+                status=_SUCCESS,
+                result={
+                    "content": "ok",
+                    "metadata": {},
+                    "links": [],
+                    "contacts": {"emails": ["a@e.com"], "phones": [], "socials": []},
+                },
+            )
+
+    execute = build_crawl_executor(engine=_OnePageEngine())
+    out = await execute(CrawlInput(startUrls=["https://e.com/"]))
+
+    assert out.contacts.emails[0].siteWide  # one page: no signal to split on
+
+
+async def test_captcha_telemetry_is_rolled_up_for_billing() -> None:
+    class _CaptchaEngine:
+        async def crawl_url(self, url: str) -> CrawlOutcome:
+            return CrawlOutcome(
+                status=_SUCCESS,
+                result={"content": "ok", "metadata": {}, "links": []},
+                captcha_attempts=2,
+                captcha_solved=True,
+            )
+
+    execute = build_crawl_executor(engine=_CaptchaEngine())
+
+    out = await execute(CrawlInput(startUrls=["https://e.com/"]))
+
+    assert out.captcha_attempts == 2
+    assert out.captcha_solved == 1
+    assert out.billable_units == 1

surfsense_backend/tests/unit/capabilities/web/crawl/test_schemas.py

@@ -0,0 +1,69 @@
+"""``web.crawl`` I/O contract: camelCase surface, bounds, and billing counters."""
+
+from __future__ import annotations
+
+import pytest
+from pydantic import ValidationError
+
+from app.capabilities.web.crawl.schemas import (
+    CrawlInput,
+    CrawlItem,
+    CrawlMeta,
+    CrawlOutput,
+)
+
+pytestmark = pytest.mark.unit
+
+
+def test_requires_at_least_one_start_url() -> None:
+    with pytest.raises(ValidationError):
+        CrawlInput(startUrls=[])
+
+
+def test_camelcase_fields_and_defaults() -> None:
+    model = CrawlInput(startUrls=["https://e.com"])
+    assert model.startUrls == ["https://e.com"]
+    assert model.maxCrawlDepth == 0
+    assert model.maxCrawlPages == 10
+    assert model.maxLength == 50_000
+
+
+def test_depth_and_page_bounds_are_enforced() -> None:
+    with pytest.raises(ValidationError):
+        CrawlInput(startUrls=["https://e.com"], maxCrawlDepth=-1)
+    with pytest.raises(ValidationError):
+        CrawlInput(startUrls=["https://e.com"], maxCrawlDepth=99)
+    with pytest.raises(ValidationError):
+        CrawlInput(startUrls=["https://e.com"], maxCrawlPages=0)
+
+
+def test_estimated_units_for_single_url_is_seed_count() -> None:
+    model = CrawlInput(startUrls=["https://a.com", "https://b.com"], maxCrawlDepth=0)
+    assert model.estimated_units == 2
+
+
+def test_estimated_units_for_spider_is_max_pages() -> None:
+    model = CrawlInput(startUrls=["https://a.com"], maxCrawlDepth=2, maxCrawlPages=25)
+    assert model.estimated_units == 25
+
+
+def test_billable_units_counts_only_successes() -> None:
+    out = CrawlOutput(
+        items=[
+            CrawlItem(
+                url="a", status="success", crawl=CrawlMeta(loadedUrl="a", depth=0)
+            ),
+            CrawlItem(url="b", status="empty", crawl=CrawlMeta(loadedUrl="b", depth=1)),
+            CrawlItem(
+                url="c", status="failed", crawl=CrawlMeta(loadedUrl="c", depth=1)
+            ),
+        ]
+    )
+    assert out.billable_units == 1
+
+
+def test_captcha_counters_are_excluded_from_the_wire_shape() -> None:
+    out = CrawlOutput(items=[], captcha_attempts=3, captcha_solved=1)
+    dumped = out.model_dump()
+    assert "captcha_attempts" not in dumped
+    assert "captcha_solved" not in dumped

surfsense_backend/tests/unit/capabilities/youtube/comments/test_executor.py

@@ -0,0 +1,55 @@
+"""``youtube.comments`` executor: verb input → actor input mapping → typed items."""
+
+from __future__ import annotations
+
+import pytest
+
+from app.capabilities.youtube.comments.executor import build_comments_executor
+from app.capabilities.youtube.comments.schemas import CommentsInput, CommentsOutput
+from app.proprietary.platforms.youtube import YouTubeCommentsInput
+
+pytestmark = pytest.mark.unit
+
+
+class _FakeScraper:
+    def __init__(self, items: list[dict]):
+        self._items = items
+        self.calls: list[YouTubeCommentsInput] = []
+
+    async def __call__(self, actor_input: YouTubeCommentsInput) -> list[dict]:
+        self.calls.append(actor_input)
+        return self._items
+
+
+async def test_maps_urls_and_wraps_comment_items():
+    scraper = _FakeScraper([{"cid": "c1", "comment": "nice", "author": "@a"}])
+    execute = build_comments_executor(scrape_fn=scraper)
+
+    out = await execute(CommentsInput(urls=["https://www.youtube.com/watch?v=abc"]))
+
+    assert isinstance(out, CommentsOutput)
+    assert len(out.items) == 1
+    assert out.items[0].cid == "c1"
+    assert out.items[0].comment == "nice"
+
+    (actor_input,) = scraper.calls
+    assert [u.url for u in actor_input.startUrls] == [
+        "https://www.youtube.com/watch?v=abc"
+    ]
+
+
+async def test_forwards_max_comments_and_sort():
+    scraper = _FakeScraper([])
+    execute = build_comments_executor(scrape_fn=scraper)
+
+    await execute(
+        CommentsInput(
+            urls=["https://youtu.be/abc"],
+            max_comments=50,
+            sort_by="TOP_COMMENTS",
+        )
+    )
+
+    (actor_input,) = scraper.calls
+    assert actor_input.maxComments == 50
+    assert actor_input.sortCommentsBy == "TOP_COMMENTS"

surfsense_backend/tests/unit/capabilities/youtube/comments/test_schemas.py

@@ -0,0 +1,35 @@
+"""``youtube.comments`` input guards: URLs required and batch/count bounded."""
+
+from __future__ import annotations
+
+import pytest
+from pydantic import ValidationError
+
+from app.capabilities.youtube.comments.schemas import (
+    MAX_COMMENT_VIDEOS,
+    CommentsInput,
+)
+
+pytestmark = pytest.mark.unit
+
+
+def test_rejects_empty_url_batch():
+    with pytest.raises(ValidationError):
+        CommentsInput(urls=[])
+
+
+def test_rejects_batch_over_the_cap():
+    too_many = [f"https://youtu.be/{i}" for i in range(MAX_COMMENT_VIDEOS + 1)]
+    with pytest.raises(ValidationError):
+        CommentsInput(urls=too_many)
+
+
+def test_defaults_max_comments_and_newest_first():
+    payload = CommentsInput(urls=["https://youtu.be/abc"])
+    assert payload.max_comments == 20
+    assert payload.sort_by == "NEWEST_FIRST"
+
+
+def test_rejects_zero_max_comments():
+    with pytest.raises(ValidationError):
+        CommentsInput(urls=["https://youtu.be/abc"], max_comments=0)

surfsense_backend/tests/unit/capabilities/youtube/scrape/test_executor.py

@@ -0,0 +1,87 @@
+"""``youtube.scrape`` executor: verb input → actor input mapping → typed items.
+
+Boundary mocked: the proprietary scraper (injected fake). NOT mocked: the verb's
+own payload→YouTubeScrapeInput mapping and the dict→VideoItem wrapping.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from app.capabilities.youtube.scrape.executor import build_scrape_executor
+from app.capabilities.youtube.scrape.schemas import ScrapeInput, ScrapeOutput
+from app.proprietary.platforms.youtube import YouTubeScrapeInput
+
+pytestmark = pytest.mark.unit
+
+
+class _FakeScraper:
+    """Records the actor input it was called with and returns canned items."""
+
+    def __init__(self, items: list[dict]):
+        self._items = items
+        self.calls: list[YouTubeScrapeInput] = []
+
+    async def __call__(self, actor_input: YouTubeScrapeInput) -> list[dict]:
+        self.calls.append(actor_input)
+        return self._items
+
+
+async def test_maps_urls_to_start_urls_and_wraps_items():
+    scraper = _FakeScraper([{"id": "abc", "title": "Hello"}])
+    execute = build_scrape_executor(scrape_fn=scraper)
+
+    out = await execute(ScrapeInput(urls=["https://www.youtube.com/watch?v=abc"]))
+
+    assert isinstance(out, ScrapeOutput)
+    assert len(out.items) == 1
+    assert out.items[0].id == "abc"
+    assert out.items[0].title == "Hello"
+
+    (actor_input,) = scraper.calls
+    assert [u.url for u in actor_input.startUrls] == [
+        "https://www.youtube.com/watch?v=abc"
+    ]
+    assert actor_input.searchQueries == []
+
+
+async def test_forwards_search_queries():
+    scraper = _FakeScraper([])
+    execute = build_scrape_executor(scrape_fn=scraper)
+
+    await execute(ScrapeInput(search_queries=["python", "rust"]))
+
+    (actor_input,) = scraper.calls
+    assert actor_input.searchQueries == ["python", "rust"]
+    assert actor_input.startUrls == []
+
+
+async def test_max_results_caps_every_content_type():
+    scraper = _FakeScraper([])
+    execute = build_scrape_executor(scrape_fn=scraper)
+
+    await execute(ScrapeInput(search_queries=["x"], max_results=7))
+
+    (actor_input,) = scraper.calls
+    # Videos, shorts, and streams must all inherit the caller's cap, otherwise a
+    # channel scrape would silently return only plain videos (actor default 0).
+    assert actor_input.maxResults == 7
+    assert actor_input.maxResultsShorts == 7
+    assert actor_input.maxResultStreams == 7
+
+
+async def test_forwards_subtitle_options():
+    scraper = _FakeScraper([])
+    execute = build_scrape_executor(scrape_fn=scraper)
+
+    await execute(
+        ScrapeInput(
+            urls=["https://youtu.be/abc"],
+            download_subtitles=True,
+            subtitles_language="fr",
+        )
+    )
+
+    (actor_input,) = scraper.calls
+    assert actor_input.downloadSubtitles is True
+    assert actor_input.subtitlesLanguage == "fr"

surfsense_backend/tests/unit/capabilities/youtube/scrape/test_schemas.py

@@ -0,0 +1,40 @@
+"""``youtube.scrape`` input guards: a source is required and the batch is bounded."""
+
+from __future__ import annotations
+
+import pytest
+from pydantic import ValidationError
+
+from app.capabilities.youtube.scrape.schemas import (
+    MAX_YOUTUBE_SOURCES,
+    ScrapeInput,
+)
+
+pytestmark = pytest.mark.unit
+
+
+def test_rejects_input_with_neither_urls_nor_queries():
+    with pytest.raises(ValidationError):
+        ScrapeInput()
+
+
+def test_accepts_urls_only():
+    payload = ScrapeInput(urls=["https://www.youtube.com/watch?v=abc"])
+    assert payload.search_queries == []
+
+
+def test_accepts_search_queries_only():
+    payload = ScrapeInput(search_queries=["python tutorial"])
+    assert payload.urls == []
+
+
+def test_max_results_defaults_and_is_bounded():
+    assert ScrapeInput(search_queries=["x"]).max_results == 10
+    with pytest.raises(ValidationError):
+        ScrapeInput(search_queries=["x"], max_results=0)
+
+
+def test_rejects_more_sources_than_the_cap():
+    too_many = [f"https://youtu.be/{i}" for i in range(MAX_YOUTUBE_SOURCES + 1)]
+    with pytest.raises(ValidationError):
+        ScrapeInput(urls=too_many)

surfsense_backend/tests/unit/capabilities/youtube/test_registry.py

@@ -0,0 +1,32 @@
+"""The youtube namespace registers each verb as one Capability the doors/agent read."""
+
+from __future__ import annotations
+
+import pytest
+
+from app.capabilities import (
+    youtube,  # noqa: F401  — importing the namespace registers its verbs
+)
+from app.capabilities.core.store import get_capability
+from app.capabilities.youtube.comments.schemas import CommentsInput, CommentsOutput
+from app.capabilities.youtube.scrape.schemas import ScrapeInput, ScrapeOutput
+
+pytestmark = pytest.mark.unit
+
+
+def test_youtube_scrape_is_registered_and_free():
+    cap = get_capability("youtube.scrape")
+
+    assert cap.name == "youtube.scrape"
+    assert cap.input_schema is ScrapeInput
+    assert cap.output_schema is ScrapeOutput
+    assert cap.billing_unit is None
+
+
+def test_youtube_comments_is_registered_and_free():
+    cap = get_capability("youtube.comments")
+
+    assert cap.name == "youtube.comments"
+    assert cap.input_schema is CommentsInput
+    assert cap.output_schema is CommentsOutput
+    assert cap.billing_unit is None

surfsense_backend/tests/unit/connector_indexers/test_confluence_parallel.py

@@ -69,7 +69,7 @@ async def test_build_connector_doc_produces_correct_fields():
         page,
         markdown,
         connector_id=_CONNECTOR_ID,
-        search_space_id=_SEARCH_SPACE_ID,
+        workspace_id=_SEARCH_SPACE_ID,
         user_id=_USER_ID,
     )
 
@@ -77,7 +77,7 @@ async def test_build_connector_doc_produces_correct_fields():
     assert doc.unique_id == "abc-123"
     assert doc.document_type == DocumentType.CONFLUENCE_CONNECTOR
     assert doc.source_markdown == markdown
-    assert doc.search_space_id == _SEARCH_SPACE_ID
+    assert doc.workspace_id == _SEARCH_SPACE_ID
     assert doc.connector_id == _CONNECTOR_ID
     assert doc.created_by_id == _USER_ID
     assert doc.metadata["page_id"] == "abc-123"
@@ -181,7 +181,7 @@ async def _run_index(mocks, **overrides):
     return await index_confluence_pages(
         session=mocks["session"],
         connector_id=overrides.get("connector_id", _CONNECTOR_ID),
-        search_space_id=overrides.get("search_space_id", _SEARCH_SPACE_ID),
+        workspace_id=overrides.get("workspace_id", _SEARCH_SPACE_ID),
         user_id=overrides.get("user_id", _USER_ID),
         start_date=overrides.get("start_date", "2025-01-01"),
         end_date=overrides.get("end_date", "2025-12-31"),

surfsense_backend/tests/unit/connector_indexers/test_dropbox_parallel.py

@@ -69,7 +69,7 @@ async def test_single_file_returns_one_connector_document(
         mock_dropbox_client,
         [_make_file_dict("f1", "test.txt")],
         connector_id=_CONNECTOR_ID,
-        search_space_id=_SEARCH_SPACE_ID,
+        workspace_id=_SEARCH_SPACE_ID,
         user_id=_USER_ID,
     )
 
@@ -94,7 +94,7 @@ async def test_multiple_files_all_produce_documents(
         mock_dropbox_client,
         files,
         connector_id=_CONNECTOR_ID,
-        search_space_id=_SEARCH_SPACE_ID,
+        workspace_id=_SEARCH_SPACE_ID,
         user_id=_USER_ID,
     )
 
@@ -121,7 +121,7 @@ async def test_one_download_exception_does_not_block_others(
         mock_dropbox_client,
         files,
         connector_id=_CONNECTOR_ID,
-        search_space_id=_SEARCH_SPACE_ID,
+        workspace_id=_SEARCH_SPACE_ID,
         user_id=_USER_ID,
     )
 
@@ -147,7 +147,7 @@ async def test_etl_error_counts_as_download_failure(
         mock_dropbox_client,
         files,
         connector_id=_CONNECTOR_ID,
-        search_space_id=_SEARCH_SPACE_ID,
+        workspace_id=_SEARCH_SPACE_ID,
         user_id=_USER_ID,
     )
 
@@ -185,7 +185,7 @@ async def test_concurrency_bounded_by_semaphore(
         mock_dropbox_client,
         files,
         connector_id=_CONNECTOR_ID,
-        search_space_id=_SEARCH_SPACE_ID,
+        workspace_id=_SEARCH_SPACE_ID,
         user_id=_USER_ID,
         max_concurrency=2,
     )
@@ -224,7 +224,7 @@ async def test_heartbeat_fires_during_parallel_downloads(
         mock_dropbox_client,
         files,
         connector_id=_CONNECTOR_ID,
-        search_space_id=_SEARCH_SPACE_ID,
+        workspace_id=_SEARCH_SPACE_ID,
         user_id=_USER_ID,
         on_heartbeat=_on_heartbeat,
     )
@@ -257,7 +257,7 @@ def full_scan_mocks(mock_dropbox_client, monkeypatch):
 
     monkeypatch.setattr("app.config.config.ETL_SERVICE", "LLAMACLOUD")
 
-    async def _fake_skip(session, file, search_space_id):
+    async def _fake_skip(session, file, workspace_id):
         from app.connectors.dropbox.file_types import should_skip_file as _skip
 
         item_skip, unsup_ext = _skip(file)
@@ -389,7 +389,7 @@ def selected_files_mocks(mock_dropbox_client, monkeypatch):
 
     skip_results: dict[str, tuple[bool, str | None]] = {}
 
-    async def _fake_skip(session, file, search_space_id):
+    async def _fake_skip(session, file, workspace_id):
         return skip_results.get(file["id"], (False, None))
 
     monkeypatch.setattr(_mod, "_should_skip_file", _fake_skip)
@@ -430,7 +430,7 @@ async def _run_selected(mocks, file_tuples):
         mocks["session"],
         file_tuples,
         connector_id=_CONNECTOR_ID,
-        search_space_id=_SEARCH_SPACE_ID,
+        workspace_id=_SEARCH_SPACE_ID,
         user_id=_USER_ID,
     )
 
@@ -547,7 +547,7 @@ async def test_delta_sync_deletions_call_remove_document(monkeypatch):
 
     remove_calls: list[str] = []
 
-    async def _fake_remove(session, file_id, search_space_id):
+    async def _fake_remove(session, file_id, workspace_id):
         remove_calls.append(file_id)
 
     monkeypatch.setattr(_mod, "_remove_document", _fake_remove)
@@ -641,7 +641,7 @@ async def test_delta_sync_mix_deletions_and_upserts(monkeypatch):
 
     remove_calls: list[str] = []
 
-    async def _fake_remove(session, file_id, search_space_id):
+    async def _fake_remove(session, file_id, workspace_id):
         remove_calls.append(file_id)
 
     monkeypatch.setattr(_mod, "_remove_document", _fake_remove)

surfsense_backend/tests/unit/connector_indexers/test_etl_credits.py

@@ -201,7 +201,7 @@ async def _run_gdrive_selected(mocks, file_ids):
         mocks["session"],
         file_ids,
         connector_id=_CONNECTOR_ID,
-        search_space_id=_SEARCH_SPACE_ID,
+        workspace_id=_SEARCH_SPACE_ID,
         user_id=_USER_ID,
     )
 
@@ -542,7 +542,7 @@ async def _run_onedrive_selected(mocks, file_ids):
         mocks["session"],
         file_ids,
         connector_id=_CONNECTOR_ID,
-        search_space_id=_SEARCH_SPACE_ID,
+        workspace_id=_SEARCH_SPACE_ID,
         user_id=_USER_ID,
     )
 
@@ -641,7 +641,7 @@ async def _run_dropbox_selected(mocks, file_paths):
         mocks["session"],
         file_paths,
         connector_id=_CONNECTOR_ID,
-        search_space_id=_SEARCH_SPACE_ID,
+        workspace_id=_SEARCH_SPACE_ID,
         user_id=_USER_ID,
     )
 

surfsense_backend/tests/unit/connector_indexers/test_google_drive_parallel.py

@@ -64,7 +64,7 @@ async def test_single_file_returns_one_connector_document(
         mock_drive_client,
         [_make_file_dict("f1", "test.txt")],
         connector_id=_CONNECTOR_ID,
-        search_space_id=_SEARCH_SPACE_ID,
+        workspace_id=_SEARCH_SPACE_ID,
         user_id=_USER_ID,
     )
 
@@ -88,7 +88,7 @@ async def test_multiple_files_all_produce_documents(
         mock_drive_client,
         files,
         connector_id=_CONNECTOR_ID,
-        search_space_id=_SEARCH_SPACE_ID,
+        workspace_id=_SEARCH_SPACE_ID,
         user_id=_USER_ID,
     )
 
@@ -115,7 +115,7 @@ async def test_one_download_exception_does_not_block_others(
         mock_drive_client,
         files,
         connector_id=_CONNECTOR_ID,
-        search_space_id=_SEARCH_SPACE_ID,
+        workspace_id=_SEARCH_SPACE_ID,
         user_id=_USER_ID,
     )
 
@@ -141,7 +141,7 @@ async def test_etl_error_counts_as_download_failure(
         mock_drive_client,
         files,
         connector_id=_CONNECTOR_ID,
-        search_space_id=_SEARCH_SPACE_ID,
+        workspace_id=_SEARCH_SPACE_ID,
         user_id=_USER_ID,
     )
 
@@ -180,7 +180,7 @@ async def test_concurrency_bounded_by_semaphore(
         mock_drive_client,
         files,
         connector_id=_CONNECTOR_ID,
-        search_space_id=_SEARCH_SPACE_ID,
+        workspace_id=_SEARCH_SPACE_ID,
         user_id=_USER_ID,
         max_concurrency=2,
     )
@@ -219,7 +219,7 @@ async def test_heartbeat_fires_during_parallel_downloads(
         mock_drive_client,
         files,
         connector_id=_CONNECTOR_ID,
-        search_space_id=_SEARCH_SPACE_ID,
+        workspace_id=_SEARCH_SPACE_ID,
         user_id=_USER_ID,
         on_heartbeat=_on_heartbeat,
     )
@@ -284,7 +284,7 @@ def full_scan_mocks(mock_drive_client, monkeypatch):
 
     skip_results: dict[str, tuple[bool, str | None]] = {}
 
-    async def _fake_skip(session, file, search_space_id):
+    async def _fake_skip(session, file, workspace_id):
         return skip_results.get(file["id"], (False, None))
 
     monkeypatch.setattr(_mod, "_should_skip_file", _fake_skip)
@@ -458,7 +458,7 @@ async def test_delta_sync_removals_serial_rest_parallel(monkeypatch):
 
     remove_calls: list[str] = []
 
-    async def _fake_remove(session, file_id, search_space_id):
+    async def _fake_remove(session, file_id, workspace_id):
         remove_calls.append(file_id)
 
     monkeypatch.setattr(_mod, "_remove_document", _fake_remove)
@@ -532,7 +532,7 @@ def selected_files_mocks(mock_drive_client, monkeypatch):
 
     skip_results: dict[str, tuple[bool, str | None]] = {}
 
-    async def _fake_skip(session, file, search_space_id):
+    async def _fake_skip(session, file, workspace_id):
         return skip_results.get(file["id"], (False, None))
 
     monkeypatch.setattr(_mod, "_should_skip_file", _fake_skip)
@@ -563,7 +563,7 @@ async def _run_selected(mocks, file_ids):
         mocks["session"],
         file_ids,
         connector_id=_CONNECTOR_ID,
-        search_space_id=_SEARCH_SPACE_ID,
+        workspace_id=_SEARCH_SPACE_ID,
         user_id=_USER_ID,
     )
 

surfsense_backend/tests/unit/connector_indexers/test_linear_parallel.py

@@ -68,7 +68,7 @@ async def test_build_connector_doc_produces_correct_fields():
         formatted,
         markdown,
         connector_id=_CONNECTOR_ID,
-        search_space_id=_SEARCH_SPACE_ID,
+        workspace_id=_SEARCH_SPACE_ID,
         user_id=_USER_ID,
     )
 
@@ -76,7 +76,7 @@ async def test_build_connector_doc_produces_correct_fields():
     assert doc.unique_id == "abc-123"
     assert doc.document_type == DocumentType.LINEAR_CONNECTOR
     assert doc.source_markdown == markdown
-    assert doc.search_space_id == _SEARCH_SPACE_ID
+    assert doc.workspace_id == _SEARCH_SPACE_ID
     assert doc.connector_id == _CONNECTOR_ID
     assert doc.created_by_id == _USER_ID
     assert doc.metadata["issue_id"] == "abc-123"
@@ -196,7 +196,7 @@ async def _run_index(mocks, **overrides):
     return await index_linear_issues(
         session=mocks["session"],
         connector_id=overrides.get("connector_id", _CONNECTOR_ID),
-        search_space_id=overrides.get("search_space_id", _SEARCH_SPACE_ID),
+        workspace_id=overrides.get("workspace_id", _SEARCH_SPACE_ID),
         user_id=overrides.get("user_id", _USER_ID),
         start_date=overrides.get("start_date", "2025-01-01"),
         end_date=overrides.get("end_date", "2025-12-31"),

surfsense_backend/tests/unit/connector_indexers/test_notion_parallel.py

@@ -39,7 +39,7 @@ async def test_build_connector_doc_produces_correct_fields():
         page,
         markdown,
         connector_id=_CONNECTOR_ID,
-        search_space_id=_SEARCH_SPACE_ID,
+        workspace_id=_SEARCH_SPACE_ID,
         user_id=_USER_ID,
     )
 
@@ -47,7 +47,7 @@ async def test_build_connector_doc_produces_correct_fields():
     assert doc.unique_id == "abc-123"
     assert doc.document_type == DocumentType.NOTION_CONNECTOR
     assert doc.source_markdown == markdown
-    assert doc.search_space_id == _SEARCH_SPACE_ID
+    assert doc.workspace_id == _SEARCH_SPACE_ID
     assert doc.connector_id == _CONNECTOR_ID
     assert doc.created_by_id == _USER_ID
     assert doc.metadata["page_title"] == "My Notion Page"
@@ -159,7 +159,7 @@ async def _run_index(mocks, **overrides):
     return await index_notion_pages(
         session=mocks["session"],
         connector_id=overrides.get("connector_id", _CONNECTOR_ID),
-        search_space_id=overrides.get("search_space_id", _SEARCH_SPACE_ID),
+        workspace_id=overrides.get("workspace_id", _SEARCH_SPACE_ID),
         user_id=overrides.get("user_id", _USER_ID),
         start_date=overrides.get("start_date", "2025-01-01"),
         end_date=overrides.get("end_date", "2025-12-31"),

surfsense_backend/tests/unit/connector_indexers/test_onedrive_parallel.py

@@ -63,7 +63,7 @@ async def test_single_file_returns_one_connector_document(
         mock_onedrive_client,
         [_make_file_dict("f1", "test.txt")],
         connector_id=_CONNECTOR_ID,
-        search_space_id=_SEARCH_SPACE_ID,
+        workspace_id=_SEARCH_SPACE_ID,
         user_id=_USER_ID,
     )
 
@@ -88,7 +88,7 @@ async def test_multiple_files_all_produce_documents(
         mock_onedrive_client,
         files,
         connector_id=_CONNECTOR_ID,
-        search_space_id=_SEARCH_SPACE_ID,
+        workspace_id=_SEARCH_SPACE_ID,
         user_id=_USER_ID,
     )
 
@@ -115,7 +115,7 @@ async def test_one_download_exception_does_not_block_others(
         mock_onedrive_client,
         files,
         connector_id=_CONNECTOR_ID,
-        search_space_id=_SEARCH_SPACE_ID,
+        workspace_id=_SEARCH_SPACE_ID,
         user_id=_USER_ID,
     )
 
@@ -141,7 +141,7 @@ async def test_etl_error_counts_as_download_failure(
         mock_onedrive_client,
         files,
         connector_id=_CONNECTOR_ID,
-        search_space_id=_SEARCH_SPACE_ID,
+        workspace_id=_SEARCH_SPACE_ID,
         user_id=_USER_ID,
     )
 
@@ -179,7 +179,7 @@ async def test_concurrency_bounded_by_semaphore(
         mock_onedrive_client,
         files,
         connector_id=_CONNECTOR_ID,
-        search_space_id=_SEARCH_SPACE_ID,
+        workspace_id=_SEARCH_SPACE_ID,
         user_id=_USER_ID,
         max_concurrency=2,
     )
@@ -218,7 +218,7 @@ async def test_heartbeat_fires_during_parallel_downloads(
         mock_onedrive_client,
         files,
         connector_id=_CONNECTOR_ID,
-        search_space_id=_SEARCH_SPACE_ID,
+        workspace_id=_SEARCH_SPACE_ID,
         user_id=_USER_ID,
         on_heartbeat=_on_heartbeat,
     )

surfsense_backend/tests/unit/event_bus/test_bus.py

@@ -13,7 +13,7 @@ pytestmark = pytest.mark.unit
 
 
 def _event() -> Event:
-    return Event(event_type="x.happened", payload={"k": "v"}, search_space_id=1)
+    return Event(event_type="x.happened", payload={"k": "v"}, workspace_id=1)
 
 
 async def _noop(_event: Event) -> None:
@@ -152,13 +152,13 @@ async def test_publish_builds_a_stamped_event_and_fans_it_out() -> None:
         received.append(event)
 
     bus.subscribe(handler)
-    await bus.publish("document.indexed", {"document_id": 42}, search_space_id=7)
+    await bus.publish("document.indexed", {"document_id": 42}, workspace_id=7)
 
     assert len(received) == 1
     event = received[0]
     assert event.event_type == "document.indexed"
     assert event.payload == {"document_id": 42}
-    assert event.search_space_id == 7
+    assert event.workspace_id == 7
     # Engine-stamped identity/time on the way through.
     assert event.event_id
     assert event.occurred_at
@@ -172,10 +172,10 @@ async def test_publish_defaults_payload_to_empty_dict() -> None:
         received.append(event)
 
     bus.subscribe(handler)
-    await bus.publish("x.happened", search_space_id=1)
+    await bus.publish("x.happened", workspace_id=1)
 
     assert received[0].payload == {}
 
 
 async def test_publish_with_no_subscribers_is_a_noop() -> None:
-    await EventBus().publish("x.happened", search_space_id=1)  # must not raise
+    await EventBus().publish("x.happened", workspace_id=1)  # must not raise

surfsense_backend/tests/unit/event_bus/test_entered_folder_predicate.py

@@ -14,7 +14,7 @@ pytestmark = pytest.mark.unit
 def _call(**overrides: Any) -> dict[str, Any] | None:
     defaults: dict[str, Any] = {
         "document_id": 1,
-        "search_space_id": 10,
+        "workspace_id": 10,
         "new_folder_id": 7,
         "previous_folder_id": None,
         "folder_id_changed": True,
@@ -33,7 +33,7 @@ def test_folder_set_ready_fires() -> None:
 
     assert result is not None
     assert result["event_type"] == "document.entered_folder"
-    assert result["search_space_id"] == 10
+    assert result["workspace_id"] == 10
     assert result["payload"]["folder_id"] == 7
     assert result["payload"]["previous_folder_id"] is None
 

surfsense_backend/tests/unit/event_bus/test_event.py

@@ -16,17 +16,17 @@ def test_event_carries_caller_supplied_facts() -> None:
     event = Event(
         event_type="document.indexed",
         payload={"document_id": 42, "content_type": "pdf"},
-        search_space_id=7,
+        workspace_id=7,
     )
 
     assert event.event_type == "document.indexed"
     assert event.payload == {"document_id": 42, "content_type": "pdf"}
-    assert event.search_space_id == 7
+    assert event.workspace_id == 7
 
 
 def test_event_stamps_identity_and_time_when_not_supplied() -> None:
     """Engine stamps id + time so subscribers can dedup/order."""
-    event = Event(event_type="x.happened", payload={}, search_space_id=1)
+    event = Event(event_type="x.happened", payload={}, workspace_id=1)
 
     assert event.event_id
     assert isinstance(event.occurred_at, datetime)
@@ -34,8 +34,8 @@ def test_event_stamps_identity_and_time_when_not_supplied() -> None:
 
 def test_event_ids_are_unique_per_instance() -> None:
     """Two events published with identical content are still distinct facts."""
-    first = Event(event_type="x.happened", payload={}, search_space_id=1)
-    second = Event(event_type="x.happened", payload={}, search_space_id=1)
+    first = Event(event_type="x.happened", payload={}, workspace_id=1)
+    second = Event(event_type="x.happened", payload={}, workspace_id=1)
 
     assert first.event_id != second.event_id
 
@@ -45,7 +45,7 @@ def test_event_survives_json_round_trip() -> None:
     original = Event(
         event_type="podcast.generated",
         payload={"podcast_id": 9, "duration_s": 123.5},
-        search_space_id=3,
+        workspace_id=3,
     )
 
     restored = Event.model_validate_json(original.model_dump_json())

surfsense_backend/tests/unit/gateway/test_webhook_routes.py

@@ -330,10 +330,10 @@ async def test_discord_gateway_install_returns_oauth_url(monkeypatch, mocker):
         "http://localhost:8000/api/v1/gateway/discord/callback",
     )
     monkeypatch.setattr(routes.config, "SECRET_KEY", "test-secret")
-    monkeypatch.setattr(routes, "check_search_space_access", mocker.AsyncMock())
+    monkeypatch.setattr(routes, "check_workspace_access", mocker.AsyncMock())
 
     response = await routes.install_discord_gateway(
-        search_space_id=123,
+        workspace_id=123,
         auth=AuthContext.session(
             SimpleNamespace(id="00000000-0000-0000-0000-000000000001")
         ),

surfsense_backend/tests/unit/indexing_pipeline/test_connector_document.py

@@ -14,7 +14,7 @@ def test_valid_document_created_with_required_fields():
         source_markdown="## Task\n\nSome content.",
         unique_id="task-1",
         document_type=DocumentType.CLICKUP_CONNECTOR,
-        search_space_id=1,
+        workspace_id=1,
         connector_id=42,
         created_by_id="00000000-0000-0000-0000-000000000001",
     )
@@ -32,7 +32,7 @@ def test_omitting_created_by_id_raises():
             source_markdown="## Content",
             unique_id="task-1",
             document_type=DocumentType.CLICKUP_CONNECTOR,
-            search_space_id=1,
+            workspace_id=1,
             connector_id=42,
         )
 
@@ -45,7 +45,7 @@ def test_empty_source_markdown_raises():
             source_markdown="",
             unique_id="task-1",
             document_type=DocumentType.CLICKUP_CONNECTOR,
-            search_space_id=1,
+            workspace_id=1,
         )
 
 
@@ -57,7 +57,7 @@ def test_whitespace_only_source_markdown_raises():
             source_markdown="   \n\t  ",
             unique_id="task-1",
             document_type=DocumentType.CLICKUP_CONNECTOR,
-            search_space_id=1,
+            workspace_id=1,
         )
 
 
@@ -69,7 +69,7 @@ def test_empty_title_raises():
             source_markdown="## Content",
             unique_id="task-1",
             document_type=DocumentType.CLICKUP_CONNECTOR,
-            search_space_id=1,
+            workspace_id=1,
         )
 
 
@@ -81,21 +81,21 @@ def test_empty_created_by_id_raises():
             source_markdown="## Content",
             unique_id="task-1",
             document_type=DocumentType.CLICKUP_CONNECTOR,
-            search_space_id=1,
+            workspace_id=1,
             connector_id=42,
             created_by_id="",
         )
 
 
-def test_zero_search_space_id_raises():
-    """search_space_id of zero raises a validation error."""
+def test_zero_workspace_id_raises():
+    """workspace_id of zero raises a validation error."""
     with pytest.raises(ValidationError):
         ConnectorDocument(
             title="Task",
             source_markdown="## Content",
             unique_id="task-1",
             document_type=DocumentType.CLICKUP_CONNECTOR,
-            search_space_id=0,
+            workspace_id=0,
             connector_id=42,
             created_by_id="00000000-0000-0000-0000-000000000001",
         )
@@ -109,5 +109,5 @@ def test_empty_unique_id_raises():
             source_markdown="## Content",
             unique_id="",
             document_type=DocumentType.CLICKUP_CONNECTOR,
-            search_space_id=1,
+            workspace_id=1,
         )

surfsense_backend/tests/unit/indexing_pipeline/test_create_placeholder_documents.py

@@ -27,7 +27,7 @@ def _make_placeholder(**overrides) -> PlaceholderInfo:
         "title": "Test Doc",
         "document_type": DocumentType.GOOGLE_DRIVE_FILE,
         "unique_id": "file-001",
-        "search_space_id": 1,
+        "workspace_id": 1,
         "connector_id": 42,
         "created_by_id": "00000000-0000-0000-0000-000000000001",
     }
@@ -36,9 +36,7 @@ def _make_placeholder(**overrides) -> PlaceholderInfo:
 
 
 def _uid_hash(p: PlaceholderInfo) -> str:
-    return compute_identifier_hash(
-        p.document_type.value, p.unique_id, p.search_space_id
-    )
+    return compute_identifier_hash(p.document_type.value, p.unique_id, p.workspace_id)
 
 
 def _session_with_existing_hashes(existing: set[str] | None = None):
@@ -82,7 +80,7 @@ async def test_creates_documents_with_pending_status_and_commits():
     assert doc.document_type == DocumentType.GOOGLE_DRIVE_FILE
     assert doc.content == "Pending..."
     assert DocumentStatus.is_state(doc.status, DocumentStatus.PENDING)
-    assert doc.search_space_id == 1
+    assert doc.workspace_id == 1
     assert doc.connector_id == 42
 
     session.commit.assert_awaited_once()

surfsense_backend/tests/unit/indexing_pipeline/test_document_hashing.py

@@ -19,12 +19,12 @@ def test_different_unique_id_produces_different_hash(make_connector_document):
     )
 
 
-def test_different_search_space_produces_different_identifier_hash(
+def test_different_workspace_produces_different_identifier_hash(
     make_connector_document,
 ):
-    """Same document in different search spaces produces different identifier hashes."""
-    doc_a = make_connector_document(search_space_id=1)
-    doc_b = make_connector_document(search_space_id=2)
+    """Same document in different workspaces produces different identifier hashes."""
+    doc_a = make_connector_document(workspace_id=1)
+    doc_b = make_connector_document(workspace_id=2)
     assert compute_unique_identifier_hash(doc_a) != compute_unique_identifier_hash(
         doc_b
     )
@@ -42,18 +42,18 @@ def test_different_document_type_produces_different_identifier_hash(
 
 
 def test_same_content_same_space_produces_same_content_hash(make_connector_document):
-    """Identical content in the same search space always produces the same content hash."""
-    doc_a = make_connector_document(source_markdown="Hello world", search_space_id=1)
-    doc_b = make_connector_document(source_markdown="Hello world", search_space_id=1)
+    """Identical content in the same workspace always produces the same content hash."""
+    doc_a = make_connector_document(source_markdown="Hello world", workspace_id=1)
+    doc_b = make_connector_document(source_markdown="Hello world", workspace_id=1)
     assert compute_content_hash(doc_a) == compute_content_hash(doc_b)
 
 
 def test_same_content_different_space_produces_different_content_hash(
     make_connector_document,
 ):
-    """Identical content in different search spaces produces different content hashes."""
-    doc_a = make_connector_document(source_markdown="Hello world", search_space_id=1)
-    doc_b = make_connector_document(source_markdown="Hello world", search_space_id=2)
+    """Identical content in different workspaces produces different content hashes."""
+    doc_a = make_connector_document(source_markdown="Hello world", workspace_id=1)
+    doc_b = make_connector_document(source_markdown="Hello world", workspace_id=2)
     assert compute_content_hash(doc_a) != compute_content_hash(doc_b)
 
 
@@ -69,7 +69,7 @@ def test_compute_identifier_hash_matches_connector_doc_hash(make_connector_docum
     doc = make_connector_document(
         document_type=DocumentType.GOOGLE_GMAIL_CONNECTOR,
         unique_id="msg-123",
-        search_space_id=5,
+        workspace_id=5,
     )
     raw_hash = compute_identifier_hash("GOOGLE_GMAIL_CONNECTOR", "msg-123", 5)
     assert raw_hash == compute_unique_identifier_hash(doc)

surfsense_backend/tests/unit/indexing_pipeline/test_index_batch.py

@@ -24,12 +24,12 @@ async def test_calls_prepare_then_index_per_document(pipeline, make_connector_do
     doc1 = make_connector_document(
         document_type=DocumentType.GOOGLE_GMAIL_CONNECTOR,
         unique_id="msg-1",
-        search_space_id=1,
+        workspace_id=1,
     )
     doc2 = make_connector_document(
         document_type=DocumentType.GOOGLE_GMAIL_CONNECTOR,
         unique_id="msg-2",
-        search_space_id=1,
+        workspace_id=1,
     )
 
     orm1 = MagicMock(spec=Document)
@@ -63,7 +63,7 @@ async def test_skips_document_without_matching_connector_doc(
     doc1 = make_connector_document(
         document_type=DocumentType.GOOGLE_GMAIL_CONNECTOR,
         unique_id="msg-1",
-        search_space_id=1,
+        workspace_id=1,
     )
 
     orphan_orm = MagicMock(spec=Document)

surfsense_backend/tests/unit/indexing_pipeline/test_index_batch_parallel.py

@@ -82,7 +82,7 @@ async def test_index_calls_embed_and_chunk_via_to_thread(
     connector_doc = make_connector_document(
         document_type=DocumentType.GOOGLE_GMAIL_CONNECTOR,
         unique_id="msg-1",
-        search_space_id=1,
+        workspace_id=1,
     )
     document = MagicMock(spec=Document)
     document.id = 1
@@ -140,7 +140,7 @@ async def test_non_code_documents_use_hybrid_chunker(
     connector_doc = make_connector_document(
         document_type=DocumentType.GOOGLE_GMAIL_CONNECTOR,
         unique_id="msg-1",
-        search_space_id=1,
+        workspace_id=1,
         should_use_code_chunker=False,
     )
     document = MagicMock(spec=Document)
@@ -184,7 +184,7 @@ async def test_batch_parallel_indexes_all_documents(
         make_connector_document(
             document_type=DocumentType.GOOGLE_GMAIL_CONNECTOR,
             unique_id=f"msg-{i}",
-            search_space_id=1,
+            workspace_id=1,
         )
         for i in range(3)
     ]
@@ -222,7 +222,7 @@ async def test_batch_parallel_one_failure_does_not_affect_others(
         make_connector_document(
             document_type=DocumentType.GOOGLE_GMAIL_CONNECTOR,
             unique_id=f"msg-{i}",
-            search_space_id=1,
+            workspace_id=1,
         )
         for i in range(3)
     ]

surfsense_backend/tests/unit/indexing_pipeline/test_migrate_legacy_docs.py

@@ -42,7 +42,7 @@ async def test_updates_hash_and_type_for_legacy_document(
     doc = make_connector_document(
         document_type=DocumentType.GOOGLE_GMAIL_CONNECTOR,
         unique_id="msg-abc",
-        search_space_id=1,
+        workspace_id=1,
     )
 
     legacy_hash = compute_identifier_hash("COMPOSIO_GMAIL_CONNECTOR", "msg-abc", 1)
@@ -70,7 +70,7 @@ async def test_noop_when_no_legacy_document_exists(
     doc = make_connector_document(
         document_type=DocumentType.GOOGLE_GMAIL_CONNECTOR,
         unique_id="msg-xyz",
-        search_space_id=1,
+        workspace_id=1,
     )
 
     result_mock = MagicMock()
@@ -89,7 +89,7 @@ async def test_skips_non_google_doc_types(
     doc = make_connector_document(
         document_type=DocumentType.SLACK_CONNECTOR,
         unique_id="slack-123",
-        search_space_id=1,
+        workspace_id=1,
     )
 
     await pipeline.migrate_legacy_docs([doc])
@@ -111,7 +111,7 @@ async def test_handles_all_three_google_types(
         doc = make_connector_document(
             document_type=native_type,
             unique_id="id-1",
-            search_space_id=1,
+            workspace_id=1,
         )
 
         existing = MagicMock(spec=Document)

surfsense_backend/tests/unit/indexing_pipeline/test_prepare_placeholder_dedup.py

@@ -32,7 +32,7 @@ def _make_connector_doc(**overrides) -> ConnectorDocument:
         "source_markdown": "## Some new content",
         "unique_id": "file-001",
         "document_type": DocumentType.GOOGLE_DRIVE_FILE,
-        "search_space_id": 1,
+        "workspace_id": 1,
         "connector_id": 42,
         "created_by_id": "00000000-0000-0000-0000-000000000001",
     }

surfsense_backend/tests/unit/middleware/test_b_filesystem_rm_rmdir_cloud.py

@@ -39,11 +39,11 @@ pytestmark = pytest.mark.unit
 
 def _make_middleware(mode: FilesystemMode = FilesystemMode.CLOUD):
     selection = FilesystemSelection(mode=mode)
-    resolver = build_backend_resolver(selection, search_space_id=1)
+    resolver = build_backend_resolver(selection, workspace_id=1)
     return build_filesystem_mw(
         backend_resolver=resolver,
         filesystem_mode=mode,
-        search_space_id=1,
+        workspace_id=1,
         user_id="00000000-0000-0000-0000-000000000001",
         thread_id=1,
     )
@@ -290,7 +290,7 @@ class TestKBPostgresBackendDeleteFilter:
 
     def _make_backend(self, state: dict[str, Any]) -> KBPostgresBackend:
         runtime = SimpleNamespace(state=state)
-        return KBPostgresBackend(search_space_id=1, runtime=runtime)
+        return KBPostgresBackend(workspace_id=1, runtime=runtime)
 
     def test_pending_filesystem_view_returns_deleted_paths(self):
         backend = self._make_backend(

surfsense_backend/tests/unit/middleware/test_filesystem_backends.py

@@ -38,16 +38,16 @@ def test_backend_resolver_returns_multi_root_backend_for_single_root(tmp_path: P
 def test_backend_resolver_uses_cloud_mode_by_default():
     resolver = build_backend_resolver(FilesystemSelection())
     backend = resolver(_RuntimeStub())
-    # When no search_space_id is provided we fall back to StateBackend so
+    # When no workspace_id is provided we fall back to StateBackend so
     # sub-agents / tests without DB access still work.
     assert backend.__class__.__name__ == "StateBackend"
 
 
-def test_backend_resolver_uses_kb_postgres_in_cloud_with_search_space():
-    resolver = build_backend_resolver(FilesystemSelection(), search_space_id=42)
+def test_backend_resolver_uses_kb_postgres_in_cloud_with_workspace():
+    resolver = build_backend_resolver(FilesystemSelection(), workspace_id=42)
     backend = resolver(_RuntimeStub())
     assert backend.__class__.__name__ == "KBPostgresBackend"
-    assert backend.search_space_id == 42
+    assert backend.workspace_id == 42
 
 
 def test_backend_resolver_returns_multi_root_backend_for_multiple_roots(tmp_path: Path):

surfsense_backend/tests/unit/middleware/test_kb_persistence_filesystem_parity.py

@@ -92,7 +92,7 @@ async def test_create_document_allows_identical_content_at_different_paths() ->
         session,  # type: ignore[arg-type]
         virtual_path="/documents/a/notes.md",
         content=content,
-        search_space_id=42,
+        workspace_id=42,
         created_by_id="user-1",
     )
     assert isinstance(first, Document)
@@ -104,7 +104,7 @@ async def test_create_document_allows_identical_content_at_different_paths() ->
         session,  # type: ignore[arg-type]
         virtual_path="/documents/b/notes-copy.md",
         content=content,
-        search_space_id=42,
+        workspace_id=42,
         created_by_id="user-1",
     )
     assert isinstance(second, Document)
@@ -121,7 +121,7 @@ async def test_create_document_still_rejects_path_collision() -> None:
     """Path uniqueness remains the hard invariant.
 
     If ``unique_identifier_hash`` already points at an existing row in
-    the same search space, the create call must raise ``ValueError``
+    the same workspace, the create call must raise ``ValueError``
     with a clear message — matching the behavior the commit loop relies
     on to upsert via the existing-row code path.
     """
@@ -137,7 +137,7 @@ async def test_create_document_still_rejects_path_collision() -> None:
             session,  # type: ignore[arg-type]
             virtual_path="/documents/notes.md",
             content="anything",
-            search_space_id=42,
+            workspace_id=42,
             created_by_id="user-1",
         )
 
@@ -160,7 +160,7 @@ async def test_create_document_does_not_query_for_content_hash_collision(
         session,  # type: ignore[arg-type]
         virtual_path="/documents/notes.md",
         content="hello",
-        search_space_id=42,
+        workspace_id=42,
         created_by_id="user-1",
     )
     # Path-collision SELECT only. No content_hash SELECT.

surfsense_backend/tests/unit/middleware/test_kb_persistence_revisions.py

@@ -217,7 +217,7 @@ async def test_pre_write_snapshot_defers_dispatch_when_list_provided(
         session,  # type: ignore[arg-type]
         doc=doc,
         action_id=42,
-        search_space_id=1,
+        workspace_id=1,
         turn_id="t-1",
         deferred_dispatches=deferred,
     )
@@ -262,7 +262,7 @@ async def test_pre_write_snapshot_dispatches_inline_when_list_omitted(
         session,  # type: ignore[arg-type]
         doc=doc,
         action_id=88,
-        search_space_id=1,
+        workspace_id=1,
         turn_id="t-1",
         # No deferred_dispatches arg — fall back to inline dispatch.
     )
@@ -302,7 +302,7 @@ async def test_pre_mkdir_snapshot_defers_dispatch_when_list_provided(
         session,  # type: ignore[arg-type]
         folder=folder,
         action_id=55,
-        search_space_id=1,
+        workspace_id=1,
         turn_id="t-1",
         deferred_dispatches=deferred,
     )

surfsense_backend/tests/unit/middleware/test_kb_postgres_read.py

@@ -28,7 +28,7 @@ pytestmark = pytest.mark.unit
 
 
 def _backend(state: dict) -> KBPostgresBackend:
-    return KBPostgresBackend(search_space_id=1, runtime=SimpleNamespace(state=state))
+    return KBPostgresBackend(workspace_id=1, runtime=SimpleNamespace(state=state))
 
 
 def test_render_full_document_uses_full_view_and_registers() -> None:

surfsense_backend/tests/unit/middleware/test_knowledge_tree.py

@@ -101,7 +101,7 @@ class TestFormatTreeRendering:
         docs = [_Row(**spec) for spec in doc_specs]
 
         mw = KnowledgeTreeMiddleware(
-            search_space_id=1,
+            workspace_id=1,
             filesystem_mode=None,  # type: ignore[arg-type]
         )
         return mw._format_tree(index, docs)

surfsense_backend/tests/unit/notifications/api/test_transform.py

@@ -53,7 +53,7 @@ def _notification(**overrides) -> Notification:
     defaults = {
         "id": 1,
         "user_id": uuid.uuid4(),
-        "search_space_id": 3,
+        "workspace_id": 3,
         "type": "document_processing",
         "title": "Title",
         "message": "Message",

surfsense_backend/tests/unit/notifications/service/messages/test_document_processing.py

@@ -10,7 +10,7 @@ pytestmark = pytest.mark.unit
 
 
 def test_operation_id_encodes_type_and_space():
-    """The operation id embeds the document type and search space id."""
+    """The operation id embeds the document type and workspace id."""
     op = msg.operation_id("FILE", "report.pdf", 9)
     assert op.startswith("doc_FILE_9_")
 

surfsense_backend/tests/unit/notifications/service/messages/test_insufficient_credits.py

@@ -9,8 +9,8 @@ from app.notifications.service.messages import insufficient_credits as msg
 pytestmark = pytest.mark.unit
 
 
-def test_operation_id_encodes_search_space():
-    """The operation id embeds the search space id."""
+def test_operation_id_encodes_workspace():
+    """The operation id embeds the workspace id."""
     assert msg.operation_id("doc.pdf", 9).startswith("insufficient_credits_9_")
 
 

surfsense_backend/tests/unit/observability/test_helpers.py

@@ -26,7 +26,6 @@ def _disable_otel(monkeypatch: pytest.MonkeyPatch):
         ("delete_folder_documents_background", "delete"),
         ("delete_search_space_background", "delete"),
         ("process_extension_document", "process"),
-        ("process_youtube_video", "process"),
         ("process_file_upload", "process"),
         ("process_file_upload_with_document", "process"),
         ("process_circleback_meeting", "process"),
@@ -36,7 +35,6 @@ def _disable_otel(monkeypatch: pytest.MonkeyPatch):
         ("cleanup_stale_indexing_notifications", "cleanup"),
         ("reconcile_pending_stripe_credit_purchases", "reconcile"),
         ("check_periodic_schedules", "check"),
-        ("ai_sort_search_space", "ai"),
         ("index_notion_pages", "index"),
         ("index_github_repos", "index"),
         ("index_google_drive_files", "index"),

surfsense_backend/tests/unit/observability/test_otel.py

@@ -166,11 +166,11 @@ class TestMetricHelpers:
             model="gpt-4o",
             provider="openai",
         )
-        metrics.record_tool_call_duration(3.0, tool_name="web_search")
-        metrics.record_tool_call_error(tool_name="web_search")
+        metrics.record_tool_call_duration(3.0, tool_name="scrape_webpage")
+        metrics.record_tool_call_error(tool_name="scrape_webpage")
         metrics.record_kb_search_duration(
             4.0,
-            search_space_id=1,
+            workspace_id=1,
             surface="documents",
         )
         metrics.record_compaction_run(reason="auto")
@@ -250,7 +250,7 @@ class TestNoopSpansWhenDisabled:
         helpers = [
             otel.tool_call_span("write_file", input_size=42),
             otel.model_call_span(model_id="openai:gpt-4o", provider="openai"),
-            otel.kb_search_span(search_space_id=1, query_chars=99),
+            otel.kb_search_span(workspace_id=1, query_chars=99),
             otel.kb_persist_span(document_type="NOTE", document_id=7),
             otel.compaction_span(reason="overflow", messages_in=120),
             otel.interrupt_span(interrupt_type="permission_ask"),
@@ -278,3 +278,30 @@ class TestEnabledIntegration:
             sp.set_attribute("tool.truncated", False)
         with otel.model_call_span(model_id="m", provider="p") as sp:
             sp.set_attribute("retry.count", 3)
+
+
+class TestPackageVersionResilience:
+    """A version-tag lookup must never crash the request path (e.g. subagents).
+
+    An editable/dynamic install can have distribution metadata with no
+    ``Version`` field, which raises ``KeyError`` deep inside importlib.metadata.
+    ``_package_version`` must swallow that and every other lookup failure.
+    """
+
+    def test_missing_version_key_falls_back(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        def _raise_key_error(_name: str) -> str:
+            raise KeyError("Version")
+
+        monkeypatch.setattr(metrics.metadata, "version", _raise_key_error)
+        assert metrics._package_version() == "unknown"
+
+    def test_package_not_found_falls_back(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        def _raise_not_found(_name: str) -> str:
+            raise metrics.metadata.PackageNotFoundError("surf-new-backend")
+
+        monkeypatch.setattr(metrics.metadata, "version", _raise_not_found)
+        assert metrics._package_version() == "unknown"

surfsense_backend/tests/unit/observability/test_retriever_otel.py

@@ -48,14 +48,14 @@ async def test_retriever_wrapper_records_one_span_and_metric(monkeypatch) -> Non
             self,
             query_text: str,
             top_k: int,
-            search_space_id: int,
+            workspace_id: int,
         ) -> list[str]:
-            del query_text, top_k, search_space_id
+            del query_text, top_k, workspace_id
             return ["doc-1", "doc-2"]
 
     result = await Retriever().search("hello", 3, 42)
 
     assert result == ["doc-1", "doc-2"]
     assert len(calls) == 1
-    assert calls[0]["search_space_id"] == 42
+    assert calls[0]["workspace_id"] == 42
     assert calls[0]["surface"] == "documents"

surfsense_backend/tests/unit/platforms/__init__.py

@@ -0,0 +1 @@
+"""Platform-native scraper packages live here (one subpackage per platform)."""

surfsense_backend/tests/unit/platforms/google_maps/test_parsers.py

@@ -0,0 +1,175 @@
+"""Offline parser tests against a captured real place ``darray`` fixture.
+
+The fixture is the ``jd[6]`` array from a live ``/maps/preview/place`` RPC
+response for "Kim's Island" (the restaurant used in the Apify output example),
+regenerated by ``scripts/e2e_google_maps_scraper.py``. These tests pin the
+array-index paths so a Google structure shift is caught offline.
+"""
+
+import json
+from pathlib import Path
+
+import pytest
+
+from app.proprietary.platforms.google_maps.fetch import strip_xssi
+from app.proprietary.platforms.google_maps.parsers import (
+    brace_match_json,
+    dig,
+    parse_place,
+)
+
+_FIXTURE = Path(__file__).parent / "fixtures" / "place_darray.json"
+
+
+@pytest.fixture
+def darray() -> list:
+    return json.loads(_FIXTURE.read_text(encoding="utf-8"))
+
+
+def test_parse_place_core_fields(darray):
+    place = parse_place(darray)
+    assert place["title"] == "Kim's Island"
+    assert place["placeId"] == "ChIJJQz5EZzKw4kRCZ95UajbyGw"
+    assert place["fid"] == "0x89c3ca9c11f90c25:0x6cc8dba851799f09"
+    assert place["categoryName"] == "Chinese restaurant"
+    assert "Chinese restaurant" in place["categories"]
+    assert place["totalScore"] == 4.5
+    assert place["phone"] == "(718) 356-5168"
+    assert place["website"] == "http://kimsislandsi.com/"
+    assert place["plusCode"].startswith("GQ62+8M")
+
+
+def test_parse_place_address_components(darray):
+    place = parse_place(darray)
+    assert place["neighborhood"] == "Tottenville"
+    assert place["street"] == "175 Main St"
+    assert place["city"] == "Staten Island"
+    assert place["postalCode"] == "10307"
+    assert place["state"] == "New York"
+    assert place["countryCode"] == "US"
+
+
+def test_parse_place_location_and_hours(darray):
+    place = parse_place(darray)
+    assert place["location"]["lat"] == pytest.approx(40.5107736)
+    assert place["location"]["lng"] == pytest.approx(-74.2482624)
+    hours = place["openingHours"]
+    assert hours and all("day" in h and "hours" in h for h in hours)
+
+
+def test_parse_place_detail_extras(darray):
+    place = parse_place(darray)
+    assert place["kgmid"] == "/g/1tmgdcj8"
+    # cid = decimal of the fid's second hex half
+    assert place["cid"] == str(int("0x6cc8dba851799f09", 16))
+    info = place["additionalInfo"]
+    assert "Accessibility" in info
+    assert {"Wheelchair accessible entrance": True} in info["Accessibility"]
+    # every option is a single {name: bool} pair
+    for entries in info.values():
+        for entry in entries:
+            assert len(entry) == 1
+            assert all(isinstance(v, bool) for v in entry.values())
+
+
+def test_parse_place_session_gated_fields(darray):
+    # These fields only appear when the RPC is sent with an NID session
+    # cookie + the full-page pb selector; the fixture pins that payload.
+    place = parse_place(darray)
+    dist = place["reviewsDistribution"]
+    assert set(dist) == {"oneStar", "twoStar", "threeStar", "fourStar", "fiveStar"}
+    assert place["reviewsCount"] == sum(dist.values())
+    assert place["imagesCount"] > 0
+    assert "All" in place["imageCategories"]
+    assert all(u.startswith("https://") for u in place["imageUrls"])
+    tags = place["reviewsTags"]
+    assert tags and all(
+        isinstance(t["title"], str) and isinstance(t["count"], int) for t in tags
+    )
+    assert len(place["additionalInfo"]) >= 5  # full sections, not just Accessibility
+
+
+def test_parse_place_hotel_fields():
+    # Fixture: live darray for The Plaza (NYC), captured via the detail RPC.
+    fixture = Path(__file__).parent / "fixtures" / "hotel_darray.json"
+    place = parse_place(json.loads(fixture.read_text(encoding="utf-8")))
+    assert place["title"] == "The Plaza"
+    assert place["hotelStars"] == "5 stars"
+    assert place["checkInDate"] < place["checkOutDate"]  # ISO dates compare
+    assert place["hotelDescription"]
+    similar = place["similarHotelsNearby"]
+    assert len(similar) >= 3
+    assert all(h["title"] and h["fid"] for h in similar)
+    assert any("hotel" in (h.get("description") or "").lower() for h in similar)
+    ads = place["hotelAds"]
+    assert ads and all(a["url"].startswith("https://") for a in ads)
+    assert any(a.get("price", "").startswith("$") for a in ads)
+
+
+def test_hotel_fields_absent_for_non_hotels(darray):
+    # Kim's Island (restaurant) must not grow hotel fields.
+    place = parse_place(darray)
+    for key in ("hotelStars", "checkInDate", "similarHotelsNearby", "hotelAds"):
+        assert key not in place
+
+
+def test_popular_times_shapes():
+    from app.proprietary.platforms.google_maps.parsers import _popular_times
+
+    darray: list = [None] * 100
+    darray[84] = [
+        [[7, [[6, 0, "", "None", "6 AM", "No wait", "6a"], [7, 35]]]],
+        None,
+        None,
+        None,
+        None,
+        None,
+        "A little busy",
+        [22, 76],
+    ]
+    out = _popular_times(darray)
+    assert out["popularTimesHistogram"] == {
+        "Su": [
+            {"hour": 6, "occupancyPercent": 0},
+            {"hour": 7, "occupancyPercent": 35},
+        ]
+    }
+    assert out["popularTimesLiveText"] == "A little busy"
+    assert out["popularTimesLivePercent"] == 76
+    assert _popular_times([None] * 100) == {}
+
+
+def test_parse_place_reservation_links():
+    fixture = Path(__file__).parent / "fixtures" / "search_response.json"
+    from app.proprietary.platforms.google_maps.fetch import _search_darrays
+
+    jd = json.loads(fixture.read_text(encoding="utf-8"))
+    place = parse_place(_search_darrays(jd)[0])
+    # the first search hit in the fixture carries a reservation provider
+    links = place["tableReservationLinks"]
+    assert all(set(link) == {"url", "source"} for link in links)
+    assert place["reserveTableUrl"] == links[0]["url"]
+
+
+def test_parse_place_omits_missing_fields(darray):
+    # parse_place drops keys whose path missed, rather than emitting nulls.
+    place = parse_place(darray)
+    assert all(v is not None for v in place.values())
+
+
+def test_dig_tolerates_ragged_paths():
+    assert dig([1, [2, 3]], 1, 0) == 2
+    assert dig([1, [2, 3]], 5) is None
+    assert dig([1, None], 1, 0) is None
+    assert dig("not a list", 0) is None
+
+
+def test_strip_xssi_and_brace_match_handle_html_wrapper():
+    # Mirrors the real proxy response: text/plain body wrapped in <p>…</p>.
+    raw = ')]}\'\n[1,[2,3],"x"]</p></body></html>'
+    body = strip_xssi(raw)
+    blob = brace_match_json(body, body.index("["))
+    assert json.loads(blob) == [1, [2, 3], "x"]
+
+    wrapped = '<html><body><p>)]}\'\n[["a"]]'
+    assert strip_xssi(wrapped).startswith("[[")

surfsense_backend/tests/unit/platforms/google_maps/test_reviews.py

@@ -0,0 +1,116 @@
+"""Offline tests for the Google Maps reviews parsing + flow helpers.
+
+``fixtures/boq_reviews_page.json`` is a real ``GetLocalBoqProxy`` page (the
+raw review list, ``jd[1][10][2]``) captured by scripts/e2e_google_maps_scraper.py
+step 7. Regenerate it with that script if Google shifts the structure.
+"""
+
+import json
+from datetime import datetime
+from pathlib import Path
+
+import pytest
+
+from app.proprietary.platforms.google_maps.fetch import build_reviews_url
+from app.proprietary.platforms.google_maps.parsers import (
+    parse_review,
+    parse_reviews_page,
+    strip_personal_data,
+)
+from app.proprietary.platforms.google_maps.reviews import _before_cutoff, _keep
+
+_FIXTURE = Path(__file__).parent / "fixtures" / "boq_reviews_page.json"
+
+
+@pytest.fixture
+def reviews_raw() -> list:
+    return json.loads(_FIXTURE.read_text(encoding="utf-8"))
+
+
+def test_parse_reviews_page_full_page(reviews_raw):
+    parsed = parse_reviews_page(reviews_raw)
+    assert len(parsed) == len(reviews_raw) == 10
+    for review in parsed:
+        assert review["name"]
+        assert review["reviewId"]
+        assert 1 <= review["stars"] <= 5
+        assert review["publishedAtDate"].endswith("Z")
+
+
+def test_parse_review_fields(reviews_raw):
+    review = parse_review(reviews_raw[0])
+    assert review["name"] == "Greg"
+    assert review["stars"] == 5
+    assert review["reviewerId"] == "108156147079988470963"
+    assert review["reviewerUrl"].startswith("https://www.google.com/maps/contrib/")
+    assert review["reviewerNumberOfReviews"] == 63
+    assert review["isLocalGuide"] is True
+    assert review["reviewOrigin"] == "Google"
+    assert review["originalLanguage"] == "en"
+    assert review["publishAt"] == "2 months ago"
+    # 1776469524140 ms epoch -> UTC ISO
+    assert review["publishedAtDate"] == "2026-04-17T23:45:24.140Z"
+    assert "spiced" in review["text"]
+    # Guided answers split into context vs per-aspect ratings.
+    assert review["reviewContext"]["Order type"] == "Take out"
+    assert review["reviewDetailedRating"]["Food"] == 5
+
+
+def test_parse_review_owner_response_and_images(reviews_raw):
+    with_reply = [
+        r
+        for r in (parse_review(x) for x in reviews_raw)
+        if r and r.get("responseFromOwnerText")
+    ]
+    assert with_reply, "fixture should contain at least one owner reply"
+    assert with_reply[0]["responseFromOwnerText"]
+
+    with_images = [
+        r
+        for r in (parse_review(x) for x in reviews_raw)
+        if r and r.get("reviewImageUrls")
+    ]
+    assert with_images, "fixture should contain at least one review with photos"
+    assert with_images[0]["reviewImageUrls"][0].startswith("https://")
+
+
+def test_parse_review_rejects_garbage():
+    assert parse_review(None) is None
+    assert parse_review([]) is None
+    assert parse_review([None, 5]) is None  # no author block
+
+
+def test_strip_personal_data(reviews_raw):
+    review = parse_review(reviews_raw[0])
+    strip_personal_data(review)
+    for key in ("name", "reviewerId", "reviewerUrl", "reviewerPhotoUrl"):
+        assert key not in review
+    assert review["reviewId"]  # non-personal fields stay
+    assert review["stars"] == 5
+
+
+def test_before_cutoff():
+    cutoff = datetime(2026, 3, 1)
+    assert _before_cutoff({"publishedAtDate": "2026-02-01T00:00:00.000Z"}, cutoff)
+    assert not _before_cutoff({"publishedAtDate": "2026-04-01T00:00:00.000Z"}, cutoff)
+    assert not _before_cutoff({}, cutoff)  # undated reviews are kept
+
+
+def test_keep_filters():
+    review = {"text": "Great NOODLES here", "reviewOrigin": "Google"}
+    assert _keep(review, filter_string="", origin="all")
+    assert _keep(review, filter_string="noodles", origin="google")
+    assert not _keep(review, filter_string="pizza", origin="all")
+    assert not _keep({"reviewOrigin": "TripAdvisor"}, filter_string="", origin="google")
+
+
+def test_build_reviews_url_shape():
+    fid = "0x89c3ca9c11f90c25:0x6cc8dba851799f09"
+    first = build_reviews_url(fid, sort_code=2)
+    assert "GetLocalBoqProxy" in first
+    assert "hl=en" in first
+    assert fid in json.dumps(first) or "0x89c3ca9c11f90c25" in first
+    # First page requests a page size; later pages carry the token instead.
+    paged = build_reviews_url(fid, sort_code=2, page_token="TOKEN123")
+    assert "TOKEN123" in paged
+    assert paged != first

surfsense_backend/tests/unit/platforms/google_maps/test_search.py

@@ -0,0 +1,150 @@
+"""Offline tests for the map-search flow: response extraction, URL building,
+and the Apify result filters. The fixture is a real ``search?tbm=map`` response
+captured by scripts/e2e_google_maps_scraper.py (step 9, query "pizza new york").
+"""
+
+import json
+from pathlib import Path
+
+import pytest
+
+from app.proprietary.platforms.google_maps.fetch import (
+    _search_darrays,
+    build_search_url,
+)
+from app.proprietary.platforms.google_maps.parsers import parse_place
+from app.proprietary.platforms.google_maps.schemas import GoogleMapsScrapeInput
+from app.proprietary.platforms.google_maps.scraper import (
+    _custom_point,
+    _location_text,
+    _passes_filters,
+)
+
+_FIXTURE = Path(__file__).parent / "fixtures" / "search_response.json"
+
+
+@pytest.fixture(scope="module")
+def search_jd():
+    return json.loads(_FIXTURE.read_text(encoding="utf-8"))
+
+
+def test_search_darrays_extracts_full_page(search_jd):
+    darrays = _search_darrays(search_jd)
+    assert len(darrays) == 20
+    parsed = [parse_place(d) for d in darrays]
+    for fields in parsed:
+        assert fields["title"]
+        assert fields["fid"].startswith("0x")
+        assert fields["placeId"].startswith("ChIJ")
+        assert "location" in fields
+    fids = [f["fid"] for f in parsed]
+    assert len(set(fids)) == 20  # no dupes within a page
+
+
+def test_search_result_has_detail_fields(search_jd):
+    fields = parse_place(_search_darrays(search_jd)[0])
+    # Search entries carry the full darray, so detail fields come through.
+    assert fields["categories"]
+    assert fields["address"]
+    assert fields["totalScore"] > 0
+    assert fields["city"]
+
+
+def test_search_darrays_rejects_garbage():
+    assert _search_darrays(None) == []
+    assert _search_darrays([]) == []
+    assert _search_darrays([[None, None]]) == []
+
+
+def test_build_search_url_shape():
+    url = build_search_url("pizza new york", offset=20, language="de")
+    assert url.startswith("https://www.google.com/search?tbm=map")
+    assert "hl=de" in url
+    assert "q=pizza%20new%20york" in url
+    assert "!8i20" in url  # offset
+    assert "!7i20" in url  # page size
+    # whole-earth viewport when no coordinates given
+    assert "!1d25000000" in url
+
+    geo_url = build_search_url("museum", lat=48.85, lng=2.35, radius_m=5000)
+    assert "!2d2.35" in geo_url and "!3d48.85" in geo_url
+    assert "!1d10000" in geo_url  # radius -> diameter
+
+
+def test_location_text_prefers_location_query():
+    assert (
+        _location_text(GoogleMapsScrapeInput(locationQuery="Berlin, Germany"))
+        == "Berlin, Germany"
+    )
+    assert (
+        _location_text(
+            GoogleMapsScrapeInput(city="Austin", state="TX", countryCode="US")
+        )
+        == "Austin, TX, US"
+    )
+    assert _location_text(GoogleMapsScrapeInput()) is None
+
+
+def test_custom_point():
+    lat, lng, radius = _custom_point(
+        GoogleMapsScrapeInput(
+            customGeolocation={
+                "type": "Point",
+                "coordinates": [2.35, 48.85],
+                "radiusKm": 5,
+            }
+        )
+    )
+    assert (lat, lng, radius) == (48.85, 2.35, 5000)
+    assert _custom_point(GoogleMapsScrapeInput()) == (None, None, None)
+    assert _custom_point(
+        GoogleMapsScrapeInput(customGeolocation={"type": "Polygon"})
+    ) == (None, None, None)
+
+
+def test_passes_filters():
+    fields = {
+        "title": "Joe's Pizza",
+        "categories": ["Pizza restaurant"],
+        "totalScore": 4.4,
+        "website": "https://joes.example",
+    }
+    default = GoogleMapsScrapeInput()
+    assert _passes_filters(fields, "pizza", default)
+
+    assert not _passes_filters(
+        fields, "pizza", GoogleMapsScrapeInput(searchMatching="only_exact")
+    )
+    assert _passes_filters(
+        fields, "pizza", GoogleMapsScrapeInput(searchMatching="only_includes")
+    )
+    assert not _passes_filters(
+        fields, "burger", GoogleMapsScrapeInput(searchMatching="only_includes")
+    )
+
+    assert _passes_filters(
+        fields, "pizza", GoogleMapsScrapeInput(categoryFilterWords=["pizza"])
+    )
+    assert not _passes_filters(
+        fields, "pizza", GoogleMapsScrapeInput(categoryFilterWords=["barber"])
+    )
+
+    assert not _passes_filters(
+        fields, "pizza", GoogleMapsScrapeInput(placeMinimumStars="fourAndHalf")
+    )
+    assert _passes_filters(
+        fields, "pizza", GoogleMapsScrapeInput(placeMinimumStars="four")
+    )
+
+    assert _passes_filters(
+        fields, "pizza", GoogleMapsScrapeInput(website="withWebsite")
+    )
+    assert not _passes_filters(
+        fields, "pizza", GoogleMapsScrapeInput(website="withoutWebsite")
+    )
+
+    closed = {**fields, "permanentlyClosed": True}
+    assert not _passes_filters(
+        closed, "pizza", GoogleMapsScrapeInput(skipClosedPlaces=True)
+    )
+    assert _passes_filters(closed, "pizza", default)

surfsense_backend/tests/unit/platforms/google_maps/test_skeleton.py

@@ -0,0 +1,97 @@
+"""Offline checks for the Google Maps scraper skeleton.
+
+Covers the pure parts: URL classification and Apify-spec schema defaults/
+serialization. The live flows (place / reviews / search) are covered by the
+e2e scripts and the fixture-based parser tests.
+"""
+
+import pytest
+
+from app.proprietary.platforms.google_maps import (
+    GoogleMapsReviewsInput,
+    GoogleMapsScrapeInput,
+    PlaceItem,
+    ReviewItem,
+)
+from app.proprietary.platforms.google_maps.url_resolver import extract_fid, resolve_url
+
+
+@pytest.mark.parametrize(
+    ("url", "kind", "value"),
+    [
+        (
+            "https://www.google.com/maps/place/Kim's+Island/@40.51,-74.24,17z/data=!4m6",
+            "place",
+            "Kim's Island",
+        ),
+        (
+            "https://www.google.com/maps/search/restaurants/@52.5190603,13.388574,13z/",
+            "search",
+            "restaurants",
+        ),
+        ("https://www.google.com/maps/reviews/data=!4m8!14m7", "reviews", None),
+        (
+            "https://www.google.com/maps?cid=7838756667406262025",
+            "cid",
+            "7838756667406262025",
+        ),
+        ("https://goo.gl/maps/abc123", "shortlink", None),
+        ("https://maps.app.goo.gl/xyz", "shortlink", None),
+    ],
+)
+def test_resolve_url(url, kind, value):
+    resolved = resolve_url(url)
+    assert resolved is not None
+    assert resolved.kind == kind
+    if value is not None:
+        assert resolved.value == value
+
+
+def test_resolve_url_rejects_non_maps():
+    assert resolve_url("https://example.com/maps/place/foo") is None
+    assert resolve_url("https://www.google.com/search?q=pizza") is None
+
+
+def test_extract_fid():
+    url = (
+        "https://www.google.com/maps/place/Kim's+Island/@40.51,-74.24,17z/"
+        "data=!4m6!3m5!1s0x89c3ca9c11f90c25:0x6cc8dba851799f09!8m2"
+    )
+    assert extract_fid(url) == "0x89c3ca9c11f90c25:0x6cc8dba851799f09"
+    assert extract_fid("https://www.google.com/maps/place/Foo") is None
+    assert resolve_url(url).fid == "0x89c3ca9c11f90c25:0x6cc8dba851799f09"
+
+
+def test_scrape_input_defaults_match_apify_spec():
+    inp = GoogleMapsScrapeInput()
+    assert inp.language == "en"
+    assert inp.maxCrawledPlacesPerSearch is None  # empty = all places
+    assert inp.searchMatching == "all"
+    assert inp.website == "allPlaces"
+    assert inp.reviewsSort == "newest"
+    assert inp.reviewsOrigin == "all"
+    assert inp.scrapeReviewsPersonalData is True
+    assert inp.maxCompetitorsToAnalyze == 30
+    assert inp.scrapeSocialMediaProfiles.facebooks is False
+    # Unknown fields are accepted (extra="allow") so parity is additive.
+    GoogleMapsScrapeInput(someFutureField=1)
+
+
+def test_reviews_input_defaults_match_apify_spec():
+    inp = GoogleMapsReviewsInput()
+    assert inp.maxReviews == 10_000_000
+    assert inp.reviewsSort == "newest"
+    assert inp.personalData is True
+
+
+def test_output_items_serialize_full_shape():
+    place = PlaceItem(title="Kim's Island", placeId="ChIJx").to_output()
+    assert place["title"] == "Kim's Island"
+    assert place["permanentlyClosed"] is False
+    assert place["categories"] == []
+    assert "reviewsDistribution" in place  # unsourced fields still emitted
+
+    review = ReviewItem(reviewId="abc", stars=5).to_output()
+    assert review["stars"] == 5
+    assert review["reviewImageUrls"] == []
+    assert "responseFromOwnerText" in review

surfsense_backend/tests/unit/platforms/google_search/test_browser_loop.py

@@ -0,0 +1,36 @@
+"""The Windows regression this guards: main.py runs the server on a
+SelectorEventLoop (psycopg needs it), and Selector loops cannot spawn
+subprocesses — so patchright's Chromium launch died with NotImplementedError
+on every render. fetch.py now marshals all browser work onto a dedicated
+subprocess-capable loop; this check proves that marshalling works from a
+Selector main loop without paying for a real browser launch.
+"""
+
+import asyncio
+import sys
+
+import pytest
+
+from app.proprietary.platforms.google_search import fetch
+
+
+def test_browser_loop_can_spawn_subprocess_from_selector_loop():
+    async def spawn():
+        proc = await asyncio.create_subprocess_exec(
+            sys.executable,
+            "-c",
+            "print('ok')",
+            stdout=asyncio.subprocess.PIPE,
+        )
+        out, _ = await proc.communicate()
+        return out
+
+    async def main():
+        if sys.platform == "win32":
+            # The original bug: the server's own loop cannot do this.
+            with pytest.raises(NotImplementedError):
+                await spawn()
+        # The fix: the same work marshalled onto the browser loop succeeds.
+        assert b"ok" in await fetch._in_browser_loop(spawn())
+
+    asyncio.run(main(), loop_factory=asyncio.SelectorEventLoop)

surfsense_backend/tests/unit/platforms/google_search/test_fetch_concurrency.py

@@ -0,0 +1,63 @@
+"""The production regression this guards: several scrape runs render SERPs
+concurrently on the shared browser. When one render failed, `_drop_session`
+closed the browser immediately — under the sibling renders — so every
+in-flight fetch died with TargetClosedError and the runs cascaded into
+"exhausted 24 IPs". The drop must defer the actual close until the last
+in-flight render on that session finishes.
+"""
+
+import asyncio
+
+from app.proprietary.platforms.google_search import fetch
+
+
+class _FakeSession:
+    def __init__(self):
+        self.release = asyncio.Event()
+        self.closed = 0
+
+    async def fetch(self, url, proxy=None):
+        await self.release.wait()
+        return "page"
+
+    async def close(self):
+        self.closed += 1
+
+
+def test_drop_defers_close_until_inflight_renders_finish(monkeypatch):
+    session = _FakeSession()
+
+    async def fake_get_session(mobile):
+        return session
+
+    monkeypatch.setattr(fetch, "_get_session", fake_get_session)
+    monkeypatch.setitem(fetch._sessions, False, session)
+
+    async def main():
+        render = asyncio.create_task(fetch._render_on_loop("u", None, False))
+        await asyncio.sleep(0)  # let the render register as in-flight
+        assert fetch._inflight[session] == 1
+
+        await fetch._drop_session_on_loop(False)
+        assert session.closed == 0, "must not close under an in-flight render"
+        assert session in fetch._doomed
+        assert False not in fetch._sessions  # next fetch relaunches
+
+        session.release.set()
+        assert await render == "page"
+        assert session.closed == 1, "last render out closes the doomed browser"
+        assert session not in fetch._inflight
+        assert session not in fetch._doomed
+
+    asyncio.run(main())
+
+
+def test_drop_closes_immediately_when_idle():
+    session = _FakeSession()
+
+    async def main():
+        fetch._sessions[False] = session
+        await fetch._drop_session_on_loop(False)
+        assert session.closed == 1
+
+    asyncio.run(main())