Merge pull request #1572 from CREDO23/add-google-search-verb

[Feat] CI : Add google_search.scrape verb + subagent

surfsense_backend/app/agents/chat/multi_agent_chat/constants.py

@@ -29,6 +29,7 @@ SUBAGENT_TO_REQUIRED_CONNECTOR_MAP: dict[str, frozenset[str]] = {
     "web_crawler": frozenset(),
     "youtube": frozenset(),
     "google_maps": frozenset(),
+    "google_search": frozenset(),
     "airtable": frozenset({"AIRTABLE_CONNECTOR"}),
     "calendar": frozenset({"GOOGLE_CALENDAR_CONNECTOR"}),
     "clickup": frozenset({"CLICKUP_CONNECTOR"}),

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/google_search/__init__.py

@@ -0,0 +1 @@
+"""``google_search`` builtin subagent: structured Google Search results."""

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/google_search/agent.py

@@ -0,0 +1,43 @@
+"""``google_search`` route: ``SurfSenseSubagentSpec`` builder for deepagents."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from langchain_core.language_models import BaseChatModel
+from langchain_core.tools import BaseTool
+
+from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import (
+    read_md_file,
+)
+from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec
+from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import (
+    pack_subagent,
+)
+
+from .tools.index import NAME, RULESET, load_tools
+
+
+def build_subagent(
+    *,
+    dependencies: dict[str, Any],
+    model: BaseChatModel | None = None,
+    middleware_stack: dict[str, Any] | None = None,
+    mcp_tools: list[BaseTool] | None = None,
+) -> SurfSenseSubagentSpec:
+    tools = [*load_tools(dependencies=dependencies), *(mcp_tools or [])]
+    description = (
+        read_md_file(__package__, "description").strip()
+        or "Searches Google and returns structured results for a query."
+    )
+    system_prompt = read_md_file(__package__, "system_prompt").strip()
+    return pack_subagent(
+        name=NAME,
+        description=description,
+        system_prompt=system_prompt,
+        tools=tools,
+        ruleset=RULESET,
+        dependencies=dependencies,
+        model=model,
+        middleware_stack=middleware_stack,
+    )

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/google_search/description.md

@@ -0,0 +1,2 @@
+Google Search specialist: runs Google searches and returns structured SERP data — organic results (title, URL, description), related queries, people-also-ask, and any AI overview. Searches by plain query (optionally scoped to a country/language or restricted to a single site) or scrapes a Google Search URL as-is, and can page deeper with max_pages_per_query. Also compares fresh search results against earlier findings in this chat.
+Use whenever the task is to find pages or sources on the open web via Google, discover which sites rank for a topic, or gather a list of result URLs to hand off for crawling. Triggers include "search Google for X", "find websites/pages about X", "who ranks for X", "top results for X", "find X's website", and comparisons against earlier search results in this chat. Not for reading a specific page's content (use the web crawling specialist), Google Maps places (use the Google Maps specialist), or YouTube (use the YouTube specialist).

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/google_search/system_prompt.md

@@ -0,0 +1,62 @@
+You are the SurfSense Google Search sub-agent.
+You receive delegated instructions from a supervisor agent and return structured results for supervisor synthesis.
+
+<goal>
+Answer the delegated question from live Google Search data gathered with your verb, comparing against earlier results already in this conversation when the task calls for it.
+</goal>
+
+<available_tools>
+- `google_search_scrape`
+</available_tools>
+
+<playbook>
+- Finding pages on a topic: call `google_search_scrape` with `queries`, scoping with `country_code`/`language_code` when locale matters.
+- Restricting to one website: set `site` (e.g. "example.com") to only return results from that domain.
+- Scraping a specific results page: pass the full Google Search URL in `queries`.
+- Need more results: raise `max_pages_per_query` to page beyond the first page.
+- Batch multiple search terms into one call rather than many single-term calls.
+- Handing URLs off for crawling: return the organic result URLs so the supervisor can route them to the web crawling specialist.
+- Comparison requests: pull the current results, compare against prior values already in this conversation's earlier tool results, and report concrete deltas (added, removed, moved up/down).
+</playbook>
+
+<tool_policy>
+- Use only tools in `<available_tools>`.
+- Report only results present in the tool output. Never invent titles, URLs, snippets, or rankings.
+</tool_policy>
+
+<out_of_scope>
+- Do not read or extract a specific page's content — return the URLs for the web crawling specialist.
+- Do not generate deliverables or perform connector mutations; return findings for the supervisor to act on.
+- Google Maps places belong to the Google Maps specialist; YouTube belongs to the YouTube specialist.
+</out_of_scope>
+
+<safety>
+- Report uncertainty explicitly when evidence is incomplete or conflicting.
+- Never present unverified claims as facts.
+</safety>
+
+<failure_policy>
+- Underspecified request — no usable query or URL — return `status=blocked` with the missing fields.
+- Tool failure: return `status=error` with a concise recovery `next_step`.
+- No useful evidence: return `status=blocked` with a narrower query or the scope you still need.
+</failure_policy>
+
+<output_contract>
+Return **only** one JSON object (no markdown/prose):
+{
+  "status": "success" | "partial" | "blocked" | "error",
+  "action_summary": string,
+  "evidence": {
+    "findings": string[],
+    "sources": string[],
+    "confidence": "high" | "medium" | "low"
+  },
+  "next_step": string | null,
+  "missing_fields": string[] | null,
+  "assumptions": string[] | null
+}
+<include snippet="output_contract_base"/>
+Route-specific rules:
+- `evidence.findings`: max 10 entries, each a single sentence stating one distinct result or delta. Do not paste raw payloads.
+- `evidence.sources`: max 10 URLs, one per finding when applicable. List each URL once.
+</output_contract>

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/google_search/tools/index.py

@@ -0,0 +1,27 @@
+"""``google_search`` sub-agent tools: the Google Search scrape capability verb."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from langchain_core.tools import BaseTool
+
+from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset
+from app.capabilities.core.access.agent import build_capability_tools
+from app.capabilities.google_search.scrape.definition import GOOGLE_SEARCH_SCRAPE
+
+NAME = "google_search"
+
+RULESET = Ruleset(origin=NAME, rules=[])
+
+_CI_VERBS = [GOOGLE_SEARCH_SCRAPE]
+
+
+def load_tools(
+    *, dependencies: dict[str, Any] | None = None, **kwargs: Any
+) -> list[BaseTool]:
+    d = {**(dependencies or {}), **kwargs}
+    return build_capability_tools(
+        workspace_id=d.get("workspace_id"),
+        capabilities=_CI_VERBS,
+    )

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py

@@ -18,6 +18,9 @@ from app.agents.chat.multi_agent_chat.subagents.builtins.deliverables.agent impo
 from app.agents.chat.multi_agent_chat.subagents.builtins.google_maps.agent import (
     build_subagent as build_google_maps_subagent,
 )
+from app.agents.chat.multi_agent_chat.subagents.builtins.google_search.agent import (
+    build_subagent as build_google_search_subagent,
+)
 from app.agents.chat.multi_agent_chat.subagents.builtins.knowledge_base.agent import (
     build_subagent as build_knowledge_base_subagent,
 )
@@ -106,6 +109,7 @@ SUBAGENT_BUILDERS_BY_NAME: dict[str, SubagentBuilder] = {
     "gmail": build_gmail_subagent,
     "google_drive": build_google_drive_subagent,
     "google_maps": build_google_maps_subagent,
+    "google_search": build_google_search_subagent,
     "jira": build_jira_subagent,
     "knowledge_base": build_knowledge_base_subagent,
     "linear": build_linear_subagent,

surfsense_backend/app/capabilities/google_search/__init__.py

@@ -0,0 +1,5 @@
+"""``google_search.*`` namespace: platform-native Google Search data verbs."""
+
+from __future__ import annotations
+
+from app.capabilities.google_search.scrape import definition as _scrape  # noqa: F401

surfsense_backend/app/capabilities/google_search/scrape/__init__.py

@@ -0,0 +1,3 @@
+"""``google_search.scrape`` verb: search terms / Google Search URLs → SERP items."""
+
+from __future__ import annotations

surfsense_backend/app/capabilities/google_search/scrape/definition.py

@@ -0,0 +1,24 @@
+"""``google_search.scrape`` capability registration (free — see 04-capabilities open item)."""
+
+from __future__ import annotations
+
+from app.capabilities.core import Capability, register_capability
+from app.capabilities.google_search.scrape.executor import build_scrape_executor
+from app.capabilities.google_search.scrape.schemas import ScrapeInput, ScrapeOutput
+
+GOOGLE_SEARCH_SCRAPE = Capability(
+    name="google_search.scrape",
+    description=(
+        "Search Google and return structured results. Give it search terms "
+        "(optionally scoped by country/language or to a single site) or full "
+        "Google Search URLs, and it returns SERP items — organic results "
+        "(title, url, description), related queries, people-also-ask, and any "
+        "AI overview. Use max_pages_per_query to page deeper."
+    ),
+    input_schema=ScrapeInput,
+    output_schema=ScrapeOutput,
+    executor=build_scrape_executor(),
+    billing_unit=None,
+)
+
+register_capability(GOOGLE_SEARCH_SCRAPE)

surfsense_backend/app/capabilities/google_search/scrape/executor.py

@@ -0,0 +1,41 @@
+"""``google_search.scrape`` executor: verb input → scraper → SERP items."""
+
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable
+
+from app.capabilities.core import Executor
+from app.capabilities.google_search.scrape.schemas import (
+    MAX_PAGES_PER_QUERY,
+    MAX_SEARCH_QUERIES,
+    ScrapeInput,
+    ScrapeOutput,
+)
+from app.proprietary.platforms.google_search import (
+    GoogleSearchScrapeInput,
+    scrape_serps,
+)
+
+ScrapeFn = Callable[..., Awaitable[list[dict]]]
+
+# Hard ceiling on SERP pages returned per call (protects the run regardless of
+# how queries * max_pages_per_query multiply out).
+_MAX_SERP_ITEMS = MAX_SEARCH_QUERIES * MAX_PAGES_PER_QUERY
+
+
+def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor:
+    """Bind the executor to a scraper fn (defaults to the proprietary actor)."""
+    scrape_fn = scrape_fn or scrape_serps
+
+    async def execute(payload: ScrapeInput) -> ScrapeOutput:
+        actor_input = GoogleSearchScrapeInput(
+            queries="\n".join(payload.queries),
+            maxPagesPerQuery=payload.max_pages_per_query,
+            countryCode=payload.country_code,
+            languageCode=payload.language_code,
+            site=payload.site,
+        )
+        items = await scrape_fn(actor_input, limit=_MAX_SERP_ITEMS)
+        return ScrapeOutput(items=items)
+
+    return execute

surfsense_backend/app/capabilities/google_search/scrape/schemas.py

@@ -0,0 +1,55 @@
+"""``google_search.scrape`` I/O contracts.
+
+A lean, agent-friendly surface over ``GoogleSearchScrapeInput``
+(``app/proprietary/platforms/google_search``). The executor maps this to the
+full scraper input; the scraper's ``SerpItem`` is reused verbatim as the output
+element.
+"""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, Field
+
+from app.proprietary.platforms.google_search import SerpItem
+
+MAX_SEARCH_QUERIES = 20
+"""Per-call cap on queries: bounds a synchronous request's fan-out."""
+
+MAX_PAGES_PER_QUERY = 10
+"""Deepest result-page pagination a single query will follow."""
+
+
+class ScrapeInput(BaseModel):
+    queries: list[str] = Field(
+        min_length=1,
+        max_length=MAX_SEARCH_QUERIES,
+        description=(
+            "Search terms (e.g. 'wedding photographers denver') or full Google "
+            "Search URLs. Each term is searched; each URL is scraped as-is."
+        ),
+    )
+    max_pages_per_query: int = Field(
+        default=1,
+        ge=1,
+        le=MAX_PAGES_PER_QUERY,
+        description="Result pages to fetch per query (1 = first page only).",
+    )
+    country_code: str | None = Field(
+        default=None,
+        description="Two-letter country to search from, e.g. 'us', 'fr'.",
+    )
+    language_code: str = Field(
+        default="",
+        description="Result language code, e.g. 'en', 'fr' (blank = Google default).",
+    )
+    site: str | None = Field(
+        default=None,
+        description="Restrict results to a single domain, e.g. 'example.com'.",
+    )
+
+
+class ScrapeOutput(BaseModel):
+    items: list[SerpItem] = Field(
+        default_factory=list,
+        description="One item per fetched SERP page, in the scraper's emission order.",
+    )

surfsense_backend/app/routes/__init__.py

@@ -2,6 +2,7 @@ from fastapi import APIRouter, Depends
 
 # Import verb namespaces for their registration side effects before the door builds.
 import app.capabilities.google_maps
+import app.capabilities.google_search
 import app.capabilities.web
 import app.capabilities.youtube  # noqa: F401
 from app.automations.api import router as automations_router