apunkt/webclaw
1
0
Fork 0
mirror of https://github.com/0xMassi/webclaw.git synced 2026-06-25 03:08:06 +02:00
Code Issues Projects Releases Packages Wiki Activity

feat(search): standalone web search with your own Serper.dev key

Browse source 
OSS surfaces can now search without the hosted webclaw API. New
webclaw-fetch::search() calls Serper.dev directly with a user-supplied key
and optionally fetches + extracts the result pages. Wired into the CLI
(webclaw search, --serper-key / SERPER_API_KEY), the MCP search tool
(local-first when SERPER_API_KEY is set, cloud fallback otherwise), and the
OSS reference server (POST /v1/search). Adds futures for concurrent result
page scraping.
This commit is contained in:
webclaw 2026-06-06 14:20:03 +02:00
parent b7bd1155c6
commit de899ab3ba
14 changed files with 671 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

10
crates/webclaw-server/src/error.rs
View file

@ -38,6 +38,9 @@ pub enum ApiError {
#[error("internal: {0}")]
Internal(String),
#[error("{0}")]
NotImplemented(String),
}
impl ApiError {
@ -48,6 +51,12 @@ impl ApiError {
pub fn internal(msg: impl Into<String>) -> Self {
Self::Internal(msg.into())
}
/// 501 — a capability the operator hasn't configured (e.g. search
/// without `SERPER_API_KEY`). Distinct from `BadRequest` (client's
/// fault) and `Internal` (our fault): it's a deployment-config gap.
pub fn not_implemented(msg: impl Into<String>) -> Self {
Self::NotImplemented(msg.into())
}
fn status(&self) -> StatusCode {
match self {
@ -57,6 +66,7 @@ impl ApiError {
Self::Fetch(_) => StatusCode::BAD_GATEWAY,
Self::Extract(_) | Self::Llm(_) => StatusCode::UNPROCESSABLE_ENTITY,
Self::Internal(_) => StatusCode::INTERNAL_SERVER_ERROR,
Self::NotImplemented(_) => StatusCode::NOT_IMPLEMENTED,
}
}
}

45
crates/webclaw-server/src/main.rs
View file

@ -123,6 +123,7 @@ fn build_app(state: AppState) -> Router {
)
.route("/crawl", post(routes::crawl::crawl))
.route("/map", post(routes::map::map))
.route("/search", post(routes::search::search))
.route("/batch", post(routes::batch::batch))
.route("/extract", post(routes::extract::extract))
.route("/extractors", get(routes::structured::list_extractors))
@ -289,4 +290,48 @@ mod tests {
"expected unknown-format error, got {body:?}"
);
}
fn post_json(uri: &str, body: &str) -> Request<Body> {
Request::builder()
.method("POST")
.uri(uri)
.header("content-type", "application/json")
.body(Body::from(body.to_owned()))
.expect("request")
}
#[tokio::test]
async fn search_empty_query_is_bad_request() {
// The empty-query guard runs before the key check, so this is
// hermetic regardless of whether SERPER_API_KEY is set.
let app = app_with_key(None).await;
let resp = app
.oneshot(post_json("/v1/search", r#"{"query":" "}"#))
.await
.expect("response");
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
}
#[tokio::test]
async fn search_without_serper_key_is_not_implemented() {
// Only meaningful when the operator hasn't configured a key.
// Skip if the test environment happens to set SERPER_API_KEY so
// we don't make a live Serper call from the test suite.
if std::env::var("SERPER_API_KEY").is_ok_and(|k| !k.trim().is_empty()) {
return;
}
let app = app_with_key(None).await;
let resp = app
.oneshot(post_json("/v1/search", r#"{"query":"rust"}"#))
.await
.expect("response");
assert_eq!(resp.status(), StatusCode::NOT_IMPLEMENTED);
let body = json_body(resp).await;
assert!(
body["error"]
.as_str()
.is_some_and(|e| e.contains("SERPER_API_KEY")),
"expected serper setup hint, got {body:?}"
);
}
}

6
crates/webclaw-server/src/routes/mod.rs
View file

@ -6,6 +6,11 @@
//! (anti-bot bypass with stealth Chrome, JS rendering at scale,
//! per-user auth, billing, async job queues, agent loops) are
//! intentionally not implemented here. Use api.webclaw.io for those.
//!
//! `POST /v1/search` is supported when the operator supplies their own
//! Serper.dev API key via the `SERPER_API_KEY` env var (free key at
//! serper.dev). Without it, the route returns 501. This is the
//! bring-your-own-key path — no hosted webclaw account required.
pub mod batch;
pub mod brand;
@ -15,5 +20,6 @@ pub mod extract;
pub mod health;
pub mod map;
pub mod scrape;
pub mod search;
pub mod structured;
pub mod summarize;

68
crates/webclaw-server/src/routes/search.rs Normal file
View file

@ -0,0 +1,68 @@
//! POST /v1/search — web search via Serper.dev using the operator's own key.
//!
//! Enabled only when the server is started with `SERPER_API_KEY` set
//! (get a free key at serper.dev). Without it, this route returns 501 so
//! self-hosters know the capability exists but isn't configured.
//!
//! With `scrape: true`, each result page is fetched + extracted to
//! markdown via the shared [`webclaw_fetch::FetchClient`]. A per-result
//! fetch failure leaves that result's `content` null; it never fails the
//! whole search.
use axum::{Json, extract::State};
use serde::Deserialize;
use serde_json::{Value, json};
use crate::{error::ApiError, state::AppState};
#[derive(Debug, Deserialize)]
pub struct SearchRequest {
pub query: String,
/// Max results to return (default 5, clamped to 1..=10).
#[serde(default = "default_num_results")]
pub num_results: usize,
/// Country code for localization (e.g. "us", "gb", "it").
pub country: Option<String>,
/// Language code for localization (e.g. "en", "it").
pub lang: Option<String>,
/// When true, fetch + extract each result page and include its markdown.
#[serde(default)]
pub scrape: bool,
}
fn default_num_results() -> usize {
5
}
pub async fn search(
State(state): State<AppState>,
Json(req): Json<SearchRequest>,
) -> Result<Json<Value>, ApiError> {
if req.query.trim().is_empty() {
return Err(ApiError::bad_request("`query` is required"));
}
let serper_key = state.serper_api_key().ok_or_else(|| {
ApiError::not_implemented(
"search is not configured: start the server with SERPER_API_KEY set \
(get a free key at serper.dev)",
)
})?;
let opts = webclaw_fetch::SearchOptions {
num_results: req.num_results,
country: req.country.clone(),
lang: req.lang.clone(),
scrape: req.scrape,
};
let results = webclaw_fetch::search(state.fetch(), serper_key, &req.query, &opts)
.await
.map_err(|e| ApiError::internal(format!("search failed: {e}")))?;
Ok(Json(json!({
"query": req.query,
"count": results.len(),
"results": results,
})))
}

18
crates/webclaw-server/src/state.rs
View file

@ -47,6 +47,9 @@ struct Inner {
pub llm_chain: Arc<ProviderChain>,
/// Inbound bearer-auth token for this server's own `/v1/*` surface.
pub api_key: Option<String>,
/// Operator's own Serper.dev API key, read from `SERPER_API_KEY`.
/// Enables `/v1/search`. Unset = `/v1/search` returns 501.
pub serper_api_key: Option<String>,
}
impl AppState {
@ -82,12 +85,22 @@ impl AppState {
let llm_chain = Arc::new(ProviderChain::default().await);
// Operator's own Serper.dev key enables /v1/search. Empty/unset
// leaves search returning 501 with a setup hint.
let serper_api_key = std::env::var("SERPER_API_KEY")
.ok()
.filter(|k| !k.trim().is_empty());
if serper_api_key.is_some() {
info!("search enabled — using SERPER_API_KEY for /v1/search");
}
Ok(Self {
inner: Arc::new(Inner {
fetch: Arc::new(fetch),
fetch_config: config,
llm_chain,
api_key: inbound_api_key,
serper_api_key,
}),
})
}
@ -112,6 +125,11 @@ impl AppState {
pub fn api_key(&self) -> Option<&str> {
self.inner.api_key.as_deref()
}
/// Operator's Serper.dev key for `/v1/search`, if configured.
pub fn serper_api_key(&self) -> Option<&str> {
self.inner.serper_api_key.as_deref()
}
}
/// Resolve the outbound cloud key. Prefers `WEBCLAW_CLOUD_API_KEY`;