feat(search): standalone web search via Serper.dev (bring-your-own-key)

Rescued from the stale perf/audit-fixes branch and ported cleanly onto
current main. OSS surfaces can now search without the hosted webclaw API
when the caller supplies their own Serper.dev key (free at serper.dev).

- webclaw-fetch::search() — calls Serper.dev directly (plain wreq client;
  a JSON API needs no fingerprinting) and, with scrape=true, fetches +
  extracts the top result pages concurrently (bounded) via the caller's
  FetchClient. parse_serper_organic() is pure and unit-tested.
- MCP `search` tool: local-first — uses SERPER_API_KEY when set, else
  falls back to the hosted webclaw API. Adds country/lang/scrape params.
- OSS REST server: POST /v1/search, gated on SERPER_API_KEY (501 when
  unset, with a setup hint). Adds ApiError::NotImplemented.
- CLI: `webclaw search <query> [--serper-key|SERPER_API_KEY] [--num]
  [--country] [--lang] [--scrape] [--format]`.

No new dependencies (reuses futures-util already in the tree). Original
work by the prior author on perf/audit-fixes; this re-applies only the
search slice onto main.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

crates/webclaw-server/src/error.rs

@@ -38,16 +38,24 @@ pub enum ApiError {
 
     #[error("internal: {0}")]
     Internal(String),
+
+    #[error("{0}")]
+    NotImplemented(String),
 }
 
 impl ApiError {
     pub fn bad_request(msg: impl Into<String>) -> Self {
         Self::BadRequest(msg.into())
     }
-    #[allow(dead_code)]
     pub fn internal(msg: impl Into<String>) -> Self {
         Self::Internal(msg.into())
     }
+    /// 501 — a capability the operator hasn't configured (e.g. search
+    /// without `SERPER_API_KEY`). Distinct from `BadRequest` (client's
+    /// fault) and `Internal` (our fault): it's a deployment-config gap.
+    pub fn not_implemented(msg: impl Into<String>) -> Self {
+        Self::NotImplemented(msg.into())
+    }
 
     fn status(&self) -> StatusCode {
         match self {
@@ -57,6 +65,7 @@ impl ApiError {
             Self::Fetch(_) => StatusCode::BAD_GATEWAY,
             Self::Extract(_) | Self::Llm(_) => StatusCode::UNPROCESSABLE_ENTITY,
             Self::Internal(_) => StatusCode::INTERNAL_SERVER_ERROR,
+            Self::NotImplemented(_) => StatusCode::NOT_IMPLEMENTED,
         }
     }
 }

crates/webclaw-server/src/main.rs

@@ -94,6 +94,7 @@ async fn main() -> anyhow::Result<()> {
         )
         .route("/crawl", post(routes::crawl::crawl))
         .route("/map", post(routes::map::map))
+        .route("/search", post(routes::search::search))
         .route("/batch", post(routes::batch::batch))
         .route("/extract", post(routes::extract::extract))
         .route("/extractors", get(routes::structured::list_extractors))

crates/webclaw-server/src/routes/mod.rs

@@ -6,6 +6,11 @@
 //! (anti-bot bypass with stealth Chrome, JS rendering at scale,
 //! per-user auth, billing, async job queues, agent loops) are
 //! intentionally not implemented here. Use api.webclaw.io for those.
+//!
+//! `POST /v1/search` is supported when the operator supplies their own
+//! Serper.dev API key via the `SERPER_API_KEY` env var (free key at
+//! serper.dev). Without it, the route returns 501. This is the
+//! bring-your-own-key path — no hosted webclaw account required.
 
 pub mod batch;
 pub mod brand;
@@ -15,5 +20,6 @@ pub mod extract;
 pub mod health;
 pub mod map;
 pub mod scrape;
+pub mod search;
 pub mod structured;
 pub mod summarize;

crates/webclaw-server/src/routes/search.rs

@@ -0,0 +1,68 @@
+//! POST /v1/search — web search via Serper.dev using the operator's own key.
+//!
+//! Enabled only when the server is started with `SERPER_API_KEY` set
+//! (get a free key at serper.dev). Without it, this route returns 501 so
+//! self-hosters know the capability exists but isn't configured.
+//!
+//! With `scrape: true`, each result page is fetched + extracted to
+//! markdown via the shared [`webclaw_fetch::FetchClient`]. A per-result
+//! fetch failure leaves that result's `content` null; it never fails the
+//! whole search.
+
+use axum::{Json, extract::State};
+use serde::Deserialize;
+use serde_json::{Value, json};
+
+use crate::{error::ApiError, state::AppState};
+
+#[derive(Debug, Deserialize)]
+pub struct SearchRequest {
+    pub query: String,
+    /// Max results to return (default 5, clamped to 1..=10).
+    #[serde(default = "default_num_results")]
+    pub num_results: usize,
+    /// Country code for localization (e.g. "us", "gb", "it").
+    pub country: Option<String>,
+    /// Language code for localization (e.g. "en", "it").
+    pub lang: Option<String>,
+    /// When true, fetch + extract each result page and include its markdown.
+    #[serde(default)]
+    pub scrape: bool,
+}
+
+fn default_num_results() -> usize {
+    5
+}
+
+pub async fn search(
+    State(state): State<AppState>,
+    Json(req): Json<SearchRequest>,
+) -> Result<Json<Value>, ApiError> {
+    if req.query.trim().is_empty() {
+        return Err(ApiError::bad_request("`query` is required"));
+    }
+
+    let serper_key = state.serper_api_key().ok_or_else(|| {
+        ApiError::not_implemented(
+            "search is not configured: start the server with SERPER_API_KEY set \
+             (get a free key at serper.dev)",
+        )
+    })?;
+
+    let opts = webclaw_fetch::SearchOptions {
+        num_results: req.num_results,
+        country: req.country.clone(),
+        lang: req.lang.clone(),
+        scrape: req.scrape,
+    };
+
+    let results = webclaw_fetch::search(state.fetch(), serper_key, &req.query, &opts)
+        .await
+        .map_err(|e| ApiError::internal(format!("search failed: {e}")))?;
+
+    Ok(Json(json!({
+        "query": req.query,
+        "count": results.len(),
+        "results": results,
+    })))
+}

crates/webclaw-server/src/state.rs

@@ -36,6 +36,9 @@ struct Inner {
     pub fetch: Arc<FetchClient>,
     /// Inbound bearer-auth token for this server's own `/v1/*` surface.
     pub api_key: Option<String>,
+    /// Operator's own Serper.dev API key, read from `SERPER_API_KEY`.
+    /// Enables `/v1/search`. Unset = `/v1/search` returns 501.
+    pub serper_api_key: Option<String>,
 }
 
 impl AppState {
@@ -66,10 +69,20 @@ impl AppState {
             fetch = fetch.with_cloud(cloud);
         }
 
+        // Operator's own Serper.dev key enables /v1/search. Empty/unset
+        // leaves search returning 501 with a setup hint.
+        let serper_api_key = std::env::var("SERPER_API_KEY")
+            .ok()
+            .filter(|k| !k.trim().is_empty());
+        if serper_api_key.is_some() {
+            info!("search enabled — using SERPER_API_KEY for /v1/search");
+        }
+
         Ok(Self {
             inner: Arc::new(Inner {
                 fetch: Arc::new(fetch),
                 api_key: inbound_api_key,
+                serper_api_key,
             }),
         })
     }
@@ -81,6 +94,11 @@ impl AppState {
     pub fn api_key(&self) -> Option<&str> {
         self.inner.api_key.as_deref()
     }
+
+    /// Operator's Serper.dev key for `/v1/search`, if configured.
+    pub fn serper_api_key(&self) -> Option<&str> {
+        self.inner.serper_api_key.as_deref()
+    }
 }
 
 /// Resolve the outbound cloud key. Prefers `WEBCLAW_CLOUD_API_KEY`;