From 06f151c560227d540ed31d7b18cf88dfc7391e39 Mon Sep 17 00:00:00 2001 From: Valerio Date: Wed, 17 Jun 2026 15:10:58 +0200 Subject: [PATCH 01/46] feat(search): standalone web search via Serper.dev (bring-your-own-key) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rescued from the stale perf/audit-fixes branch and ported cleanly onto current main. OSS surfaces can now search without the hosted webclaw API when the caller supplies their own Serper.dev key (free at serper.dev). - webclaw-fetch::search() — calls Serper.dev directly (plain wreq client; a JSON API needs no fingerprinting) and, with scrape=true, fetches + extracts the top result pages concurrently (bounded) via the caller's FetchClient. parse_serper_organic() is pure and unit-tested. - MCP `search` tool: local-first — uses SERPER_API_KEY when set, else falls back to the hosted webclaw API. Adds country/lang/scrape params. - OSS REST server: POST /v1/search, gated on SERPER_API_KEY (501 when unset, with a setup hint). Adds ApiError::NotImplemented. - CLI: `webclaw search [--serper-key|SERPER_API_KEY] [--num] [--country] [--lang] [--scrape] [--format]`. No new dependencies (reuses futures-util already in the tree). Original work by the prior author on perf/audit-fixes; this re-applies only the search slice onto main. Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/webclaw-cli/src/main.rs | 138 +++++++++ crates/webclaw-fetch/src/lib.rs | 2 + crates/webclaw-fetch/src/search.rs | 322 +++++++++++++++++++++ crates/webclaw-mcp/src/server.rs | 52 +++- crates/webclaw-mcp/src/tools.rs | 11 +- crates/webclaw-server/src/error.rs | 11 +- crates/webclaw-server/src/main.rs | 1 + crates/webclaw-server/src/routes/mod.rs | 6 + crates/webclaw-server/src/routes/search.rs | 68 +++++ crates/webclaw-server/src/state.rs | 18 ++ 10 files changed, 622 insertions(+), 7 deletions(-) create mode 100644 crates/webclaw-fetch/src/search.rs create mode 100644 crates/webclaw-server/src/routes/search.rs diff --git a/crates/webclaw-cli/src/main.rs b/crates/webclaw-cli/src/main.rs index 7d82f73..6f59be6 100644 --- a/crates/webclaw-cli/src/main.rs +++ b/crates/webclaw-cli/src/main.rs @@ -410,6 +410,43 @@ enum Commands { #[arg(long)] raw: bool, }, + + /// Web search via Serper.dev using YOUR OWN API key. + /// + /// Returns Google organic results (title, link, snippet). With + /// `--scrape`, each result page is fetched and extracted to markdown. + /// Get a free key at serper.dev, then pass `--serper-key` or set + /// `SERPER_API_KEY`. + /// + /// Example: `webclaw search "rust async runtime" --num 5 --scrape`. + Search { + /// Search query. + query: String, + + /// Serper.dev API key. Falls back to the `SERPER_API_KEY` env var. + #[arg(long, env = "SERPER_API_KEY")] + serper_key: Option, + + /// Number of results to return (1-10). + #[arg(long, default_value = "5")] + num: usize, + + /// Country code for localization (e.g. "us", "gb", "it"). + #[arg(long)] + country: Option, + + /// Language code for localization (e.g. "en", "it"). + #[arg(long)] + lang: Option, + + /// Fetch + extract each result page and include its markdown. + #[arg(long)] + scrape: bool, + + /// Output format: `markdown` (human-readable, default) or `json`. + #[arg(short, long, default_value = "markdown")] + format: OutputFormat, + }, } #[derive(Clone, ValueEnum)] @@ -1573,6 +1610,73 @@ async fn run_crawl(cli: &Cli) -> Result<(), String> { } } +/// Web search via Serper.dev with the caller's own API key. +/// +/// The Serper key is resolved by the caller (flag or `SERPER_API_KEY` +/// env, via clap's `env`) and passed in already-unwrapped. When `scrape` +/// is set, each result page is fetched + extracted through a FetchClient +/// (which carries the browser TLS profile) and its markdown is included. +#[allow(clippy::too_many_arguments)] +async fn run_search( + serper_key: &str, + query: &str, + num: usize, + country: Option<&str>, + lang: Option<&str>, + scrape: bool, + format: &OutputFormat, +) -> Result<(), String> { + // Default fetch config is enough: search localization is handled by + // Serper's gl/hl, and the result-page scrape just needs a standard + // browser profile. Attach cloud fallback when WEBCLAW_API_KEY is set + // so scraped pages behind bot protection can still escalate. + let mut client = webclaw_fetch::FetchClient::new(webclaw_fetch::FetchConfig::default()) + .map_err(|e| format!("client error: {e}"))?; + if let Some(cloud) = webclaw_fetch::cloud::CloudClient::from_env() { + client = client.with_cloud(cloud); + } + + let opts = webclaw_fetch::SearchOptions { + num_results: num, + country: country.map(str::to_string), + lang: lang.map(str::to_string), + scrape, + }; + + let results = webclaw_fetch::search(&client, serper_key, query, &opts) + .await + .map_err(|e| format!("search error: {e}"))?; + + if matches!(format, OutputFormat::Json) { + let json = serde_json::json!({ "query": query, "results": results }); + match serde_json::to_string_pretty(&json) { + Ok(s) => println!("{s}"), + Err(e) => return Err(format!("JSON encode failed: {e}")), + } + return Ok(()); + } + + if results.is_empty() { + eprintln!("no results for \"{query}\""); + return Ok(()); + } + + for r in &results { + println!("{}. {}", r.position, r.title); + println!(" {}", r.link); + if !r.snippet.is_empty() { + println!(" {}", r.snippet); + } + if let Some(ref content) = r.content { + println!(); + println!("{content}"); + } + println!(); + } + + Ok(()) +} + async fn run_map(cli: &Cli) -> Result<(), String> { let url = cli .urls @@ -2589,6 +2693,40 @@ async fn main() { } return; } + Commands::Search { + query, + serper_key, + num, + country, + lang, + scrape, + format, + } => { + let key = match serper_key { + Some(k) if !k.trim().is_empty() => k.clone(), + _ => { + eprintln!( + "error: search requires a Serper.dev API key: pass --serper-key or set SERPER_API_KEY (get one free at serper.dev)" + ); + process::exit(1); + } + }; + if let Err(e) = run_search( + &key, + query, + *num, + country.as_deref(), + lang.as_deref(), + *scrape, + format, + ) + .await + { + eprintln!("error: {e}"); + process::exit(1); + } + return; + } } } diff --git a/crates/webclaw-fetch/src/lib.rs b/crates/webclaw-fetch/src/lib.rs index b859955..a7a95e6 100644 --- a/crates/webclaw-fetch/src/lib.rs +++ b/crates/webclaw-fetch/src/lib.rs @@ -14,6 +14,7 @@ pub mod locale; pub mod progress; pub mod proxy; pub mod reddit; +pub mod search; pub mod sitemap; pub mod tls; pub mod url_security; @@ -27,5 +28,6 @@ pub use http::HeaderMap; pub use locale::{accept_language_for_tld, accept_language_for_url}; pub use progress::{PROGRESS_INTERVAL, with_progress}; pub use proxy::{parse_proxy_file, parse_proxy_line}; +pub use search::{SearchOptions, SearchResult, parse_serper_organic, search}; pub use sitemap::SitemapEntry; pub use webclaw_pdf::PdfMode; diff --git a/crates/webclaw-fetch/src/search.rs b/crates/webclaw-fetch/src/search.rs new file mode 100644 index 0000000..28ebc99 --- /dev/null +++ b/crates/webclaw-fetch/src/search.rs @@ -0,0 +1,322 @@ +//! Web search via Serper.dev (Google results) with optional content scraping. +//! +//! This is the self-hosted search path: the caller supplies their own +//! Serper.dev API key (free tier at serper.dev). The CLI, MCP server, and +//! OSS REST server all route through [`search`] so search works without the +//! hosted webclaw API. +//! +//! Serper returns a plain JSON API, so we hit it with a vanilla wreq client +//! (10s timeout) — no browser TLS fingerprinting needed. When `scrape` is +//! set, the top results are fetched through the caller's [`FetchClient`] +//! (which *does* carry the fingerprinting) and extracted to markdown. +use std::sync::Arc; +use std::time::Duration; + +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use tokio::sync::Semaphore; +use tracing::warn; + +use crate::client::FetchClient; +use crate::error::FetchError; + +/// Serper.dev search endpoint. +const SERPER_URL: &str = "https://google.serper.dev/search"; + +/// Bound on the number of result pages scraped concurrently when +/// `scrape` is enabled. Keeps the fan-out from overwhelming the proxy +/// pool / remote hosts on a large result set. +const SCRAPE_CONCURRENCY: usize = 5; + +/// Options controlling a search request. +#[derive(Debug, Clone)] +pub struct SearchOptions { + /// Number of organic results to request (clamped to `1..=10`). + pub num_results: usize, + /// Country code for localization (Serper `gl`, e.g. `"us"`, `"gb"`). + pub country: Option, + /// Language code for localization (Serper `hl`, e.g. `"en"`, `"it"`). + pub lang: Option, + /// When true, fetch + extract the result pages and fill in `content`. + pub scrape: bool, +} + +impl Default for SearchOptions { + fn default() -> Self { + Self { + num_results: 5, + country: None, + lang: None, + scrape: false, + } + } +} + +/// A single organic search result. When `scrape` was requested and the +/// fetch succeeded, `content` holds the extracted markdown; otherwise it +/// is `None` (a per-result fetch failure never fails the whole search). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SearchResult { + pub title: String, + pub link: String, + pub snippet: String, + pub position: usize, + #[serde(skip_serializing_if = "Option::is_none")] + pub content: Option, +} + +/// Run a web search through Serper.dev. +/// +/// `client` — the caller's [`FetchClient`], used only when `opts.scrape` +/// is set (to fetch + extract the result pages). +/// `serper_key` — the caller's Serper.dev API key. +/// `query` — the search query. +/// `opts` — result count, localization, and whether to scrape. +/// +/// Returns the organic results in Serper's order. With `scrape` enabled, +/// the top results are fetched concurrently (bounded) and their extracted +/// markdown is attached to `content`. +pub async fn search( + client: &FetchClient, + serper_key: &str, + query: &str, + opts: &SearchOptions, +) -> Result, FetchError> { + let num = opts.num_results.clamp(1, 10); + + let response = call_serper( + serper_key, + query, + num, + opts.country.as_deref(), + opts.lang.as_deref(), + ) + .await?; + + let mut results = parse_serper_organic(&response); + + if opts.scrape && !results.is_empty() { + scrape_results(client, &mut results).await; + } + + Ok(results) +} + +/// POST the query to Serper.dev and return the raw JSON response. +/// +/// Builds a plain wreq client (no browser emulation — Serper is a JSON +/// API, not a bot-protected page). Non-2xx responses are surfaced as a +/// [`FetchError::Build`] carrying the status and body so the caller can +/// show Serper's own error (bad key, quota exceeded, etc.). +async fn call_serper( + api_key: &str, + query: &str, + num: usize, + country: Option<&str>, + lang: Option<&str>, +) -> Result { + let http = wreq::Client::builder() + .timeout(Duration::from_secs(10)) + .build() + .map_err(|e| FetchError::Build(format!("failed to build serper client: {e}")))?; + + let mut body = json!({ "q": query, "num": num }); + if let Some(gl) = country { + body["gl"] = json!(gl); + } + if let Some(hl) = lang { + body["hl"] = json!(hl); + } + // Serialize ourselves rather than `.json()` — the wreq `json` feature + // is not enabled in this crate and isn't worth pulling in for one call. + let payload = serde_json::to_vec(&body) + .map_err(|e| FetchError::Build(format!("serper request encode error: {e}")))?; + + let resp = http + .post(SERPER_URL) + .header("X-API-KEY", api_key) + .header("Content-Type", "application/json") + .body(payload) + .send() + .await?; + + let status = resp.status(); + if !status.is_success() { + let code = status.as_u16(); + let text = resp.text().await.unwrap_or_default(); + return Err(FetchError::Build(format!("serper returned {code}: {text}"))); + } + + let text = resp + .text() + .await + .map_err(|e| FetchError::BodyDecode(format!("serper response read error: {e}")))?; + serde_json::from_str::(&text) + .map_err(|e| FetchError::BodyDecode(format!("serper response parse error: {e}"))) +} + +/// Parse the `organic` array of a Serper response into [`SearchResult`]s. +/// +/// Pure (no network), so it is unit-tested against a fixture. Entries +/// missing `title` or `link` are skipped; `snippet` defaults to empty. +/// `position` is 1-based over the kept entries. +pub fn parse_serper_organic(response: &Value) -> Vec { + let Some(organic) = response.get("organic").and_then(|v| v.as_array()) else { + return Vec::new(); + }; + + organic + .iter() + .filter_map(|item| { + let title = item.get("title")?.as_str()?.to_string(); + let link = item.get("link")?.as_str()?.to_string(); + let snippet = item + .get("snippet") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + Some(SearchResult { + title, + link, + snippet, + // Filled in after collection so it tracks kept entries, + // not the raw array index (which may include skips). + position: 0, + content: None, + }) + }) + .enumerate() + .map(|(i, mut r)| { + r.position = i + 1; + r + }) + .collect() +} + +/// Fetch + extract the result pages and attach markdown to `content`. +/// +/// Bounded by [`SCRAPE_CONCURRENCY`]. A per-result fetch or extraction +/// failure leaves that result's `content` as `None` rather than failing +/// the whole search. +async fn scrape_results(client: &FetchClient, results: &mut [SearchResult]) { + let sem = Arc::new(Semaphore::new(SCRAPE_CONCURRENCY)); + + // Collect owned links first so the per-result futures don't borrow + // `results`. That keeps the future captures free of the slice's + // lifetime, which is what lets this compile inside the MCP `#[tool]` + // macro's stricter `Send`/lifetime bounds. + let links: Vec = results.iter().map(|r| r.link.clone()).collect(); + + let scrapes = links.into_iter().map(|link| { + let sem = sem.clone(); + async move { + // If the semaphore is closed (shutdown race), skip rather than panic. + let _permit = match sem.acquire().await { + Ok(p) => p, + Err(_) => return None, + }; + match client.fetch(&link).await { + Ok(fetched) => match webclaw_core::extract(&fetched.html, Some(&fetched.url)) { + Ok(extraction) => Some(extraction.content.markdown), + Err(e) => { + warn!(url = %link, error = %e, "search: extraction failed"); + None + } + }, + Err(e) => { + warn!(url = %link, error = %e, "search: fetch failed"); + None + } + } + } + }); + + // `join_all` drives every scrape future concurrently and returns + // results in input order; the semaphore caps how many fetches run at + // once. Result set is tiny (≤10), so the all-at-once poll is fine. + let contents = futures_util::future::join_all(scrapes).await; + for (r, content) in results.iter_mut().zip(contents) { + r.content = content; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn fixture() -> Value { + json!({ + "searchParameters": { "q": "rust async", "type": "search" }, + "organic": [ + { + "title": "Async Rust", + "link": "https://example.com/async", + "snippet": "Learn async in Rust.", + "position": 1 + }, + { + // snippet missing on purpose -> defaults to "" + "title": "Tokio", + "link": "https://tokio.rs" + }, + { + // no link -> skipped, must not shift positions of the rest + "title": "No Link Here" + } + ] + }) + } + + #[test] + fn parses_organic_results() { + let results = parse_serper_organic(&fixture()); + assert_eq!(results.len(), 2); + + assert_eq!(results[0].title, "Async Rust"); + assert_eq!(results[0].link, "https://example.com/async"); + assert_eq!(results[0].snippet, "Learn async in Rust."); + assert_eq!(results[0].position, 1); + assert!(results[0].content.is_none()); + + // Missing snippet -> empty string, and position is 1-based over + // kept entries (the link-less entry is dropped, not counted). + assert_eq!(results[1].title, "Tokio"); + assert_eq!(results[1].snippet, ""); + assert_eq!(results[1].position, 2); + } + + #[test] + fn missing_organic_key_yields_empty() { + assert!(parse_serper_organic(&json!({})).is_empty()); + assert!(parse_serper_organic(&json!({ "organic": "not-an-array" })).is_empty()); + } + + #[test] + fn search_result_serializes_without_null_content() { + let r = SearchResult { + title: "T".into(), + link: "https://e.com".into(), + snippet: "s".into(), + position: 1, + content: None, + }; + let v = serde_json::to_value(&r).unwrap(); + assert!(v.get("content").is_none(), "None content should be skipped"); + + let r2 = SearchResult { + content: Some("# md".into()), + ..r + }; + let v2 = serde_json::to_value(&r2).unwrap(); + assert_eq!(v2.get("content").and_then(|c| c.as_str()), Some("# md")); + } + + #[test] + fn default_options() { + let o = SearchOptions::default(); + assert_eq!(o.num_results, 5); + assert!(!o.scrape); + assert!(o.country.is_none()); + assert!(o.lang.is_none()); + } +} diff --git a/crates/webclaw-mcp/src/server.rs b/crates/webclaw-mcp/src/server.rs index 67cf06a..2554512 100644 --- a/crates/webclaw-mcp/src/server.rs +++ b/crates/webclaw-mcp/src/server.rs @@ -668,13 +668,55 @@ impl WebclawMcp { )) } - /// Search the web for a query and return structured results. Requires WEBCLAW_API_KEY. + /// Search the web for a query and return structured results. + /// + /// Resolves the backend in priority order: + /// 1. `SERPER_API_KEY` set → local Serper.dev search with the user's + /// own key (no hosted API needed). Supports `country`, `lang`, and + /// `scrape` (fetch + extract each result page). + /// 2. else `WEBCLAW_API_KEY` set → the hosted webclaw search API. + /// 3. else → an error explaining both options. #[tool] async fn search(&self, Parameters(params): Parameters) -> Result { - let cloud = self - .cloud - .as_ref() - .ok_or("Search requires WEBCLAW_API_KEY. Get a key at https://webclaw.io")?; + // Local path: user's own Serper key. Preferred when present so the + // tool works without the hosted API and without spending credits. + if let Ok(serper_key) = std::env::var("SERPER_API_KEY") + && !serper_key.trim().is_empty() + { + let opts = webclaw_fetch::SearchOptions { + num_results: params.num_results.unwrap_or(5) as usize, + country: params.country.clone(), + lang: params.lang.clone(), + scrape: params.scrape.unwrap_or(false), + }; + let results = webclaw_fetch::search( + self.fetch_client.as_ref(), + &serper_key, + ¶ms.query, + &opts, + ) + .await + .map_err(|e| format!("search error: {e}"))?; + + let mut output = format!("Found {} results:\n\n", results.len()); + for r in &results { + output.push_str(&format!("{}. {}\n {}\n", r.position, r.title, r.link)); + if !r.snippet.is_empty() { + output.push_str(&format!(" {}\n", r.snippet)); + } + if let Some(ref content) = r.content { + output.push_str(&format!("\n{content}\n")); + } + output.push('\n'); + } + return Ok(output); + } + + // Hosted path: the webclaw cloud API. + let cloud = self.cloud.as_ref().ok_or( + "Search requires a search backend: set SERPER_API_KEY for local search \ + (get one free at serper.dev), or WEBCLAW_API_KEY for the hosted API.", + )?; let mut body = json!({ "query": params.query }); if let Some(num) = params.num_results { diff --git a/crates/webclaw-mcp/src/tools.rs b/crates/webclaw-mcp/src/tools.rs index e4dc310..f17933f 100644 --- a/crates/webclaw-mcp/src/tools.rs +++ b/crates/webclaw-mcp/src/tools.rs @@ -160,9 +160,18 @@ pub struct ResearchParams { pub struct SearchParams { /// Search query pub query: String, - /// Number of results to return (default: 10) + /// Number of results to return (default: 5, max: 10) #[serde(default, deserialize_with = "deser_opt_u32_or_str")] pub num_results: Option, + /// Country code for localization (e.g. "us", "gb", "it"). + /// Only used by the local Serper path (SERPER_API_KEY). + pub country: Option, + /// Language code for localization (e.g. "en", "it"). + /// Only used by the local Serper path (SERPER_API_KEY). + pub lang: Option, + /// When true, fetch + extract each result page and include its + /// markdown. Only used by the local Serper path (SERPER_API_KEY). + pub scrape: Option, } /// Parameters for `vertical_scrape`: run a site-specific extractor by name. diff --git a/crates/webclaw-server/src/error.rs b/crates/webclaw-server/src/error.rs index a63848f..ffd8f00 100644 --- a/crates/webclaw-server/src/error.rs +++ b/crates/webclaw-server/src/error.rs @@ -38,16 +38,24 @@ pub enum ApiError { #[error("internal: {0}")] Internal(String), + + #[error("{0}")] + NotImplemented(String), } impl ApiError { pub fn bad_request(msg: impl Into) -> Self { Self::BadRequest(msg.into()) } - #[allow(dead_code)] pub fn internal(msg: impl Into) -> Self { Self::Internal(msg.into()) } + /// 501 — a capability the operator hasn't configured (e.g. search + /// without `SERPER_API_KEY`). Distinct from `BadRequest` (client's + /// fault) and `Internal` (our fault): it's a deployment-config gap. + pub fn not_implemented(msg: impl Into) -> Self { + Self::NotImplemented(msg.into()) + } fn status(&self) -> StatusCode { match self { @@ -57,6 +65,7 @@ impl ApiError { Self::Fetch(_) => StatusCode::BAD_GATEWAY, Self::Extract(_) | Self::Llm(_) => StatusCode::UNPROCESSABLE_ENTITY, Self::Internal(_) => StatusCode::INTERNAL_SERVER_ERROR, + Self::NotImplemented(_) => StatusCode::NOT_IMPLEMENTED, } } } diff --git a/crates/webclaw-server/src/main.rs b/crates/webclaw-server/src/main.rs index 06f2451..869cf56 100644 --- a/crates/webclaw-server/src/main.rs +++ b/crates/webclaw-server/src/main.rs @@ -94,6 +94,7 @@ async fn main() -> anyhow::Result<()> { ) .route("/crawl", post(routes::crawl::crawl)) .route("/map", post(routes::map::map)) + .route("/search", post(routes::search::search)) .route("/batch", post(routes::batch::batch)) .route("/extract", post(routes::extract::extract)) .route("/extractors", get(routes::structured::list_extractors)) diff --git a/crates/webclaw-server/src/routes/mod.rs b/crates/webclaw-server/src/routes/mod.rs index 01f1052..3ed2273 100644 --- a/crates/webclaw-server/src/routes/mod.rs +++ b/crates/webclaw-server/src/routes/mod.rs @@ -6,6 +6,11 @@ //! (anti-bot bypass with stealth Chrome, JS rendering at scale, //! per-user auth, billing, async job queues, agent loops) are //! intentionally not implemented here. Use api.webclaw.io for those. +//! +//! `POST /v1/search` is supported when the operator supplies their own +//! Serper.dev API key via the `SERPER_API_KEY` env var (free key at +//! serper.dev). Without it, the route returns 501. This is the +//! bring-your-own-key path — no hosted webclaw account required. pub mod batch; pub mod brand; @@ -15,5 +20,6 @@ pub mod extract; pub mod health; pub mod map; pub mod scrape; +pub mod search; pub mod structured; pub mod summarize; diff --git a/crates/webclaw-server/src/routes/search.rs b/crates/webclaw-server/src/routes/search.rs new file mode 100644 index 0000000..5bc480e --- /dev/null +++ b/crates/webclaw-server/src/routes/search.rs @@ -0,0 +1,68 @@ +//! POST /v1/search — web search via Serper.dev using the operator's own key. +//! +//! Enabled only when the server is started with `SERPER_API_KEY` set +//! (get a free key at serper.dev). Without it, this route returns 501 so +//! self-hosters know the capability exists but isn't configured. +//! +//! With `scrape: true`, each result page is fetched + extracted to +//! markdown via the shared [`webclaw_fetch::FetchClient`]. A per-result +//! fetch failure leaves that result's `content` null; it never fails the +//! whole search. + +use axum::{Json, extract::State}; +use serde::Deserialize; +use serde_json::{Value, json}; + +use crate::{error::ApiError, state::AppState}; + +#[derive(Debug, Deserialize)] +pub struct SearchRequest { + pub query: String, + /// Max results to return (default 5, clamped to 1..=10). + #[serde(default = "default_num_results")] + pub num_results: usize, + /// Country code for localization (e.g. "us", "gb", "it"). + pub country: Option, + /// Language code for localization (e.g. "en", "it"). + pub lang: Option, + /// When true, fetch + extract each result page and include its markdown. + #[serde(default)] + pub scrape: bool, +} + +fn default_num_results() -> usize { + 5 +} + +pub async fn search( + State(state): State, + Json(req): Json, +) -> Result, ApiError> { + if req.query.trim().is_empty() { + return Err(ApiError::bad_request("`query` is required")); + } + + let serper_key = state.serper_api_key().ok_or_else(|| { + ApiError::not_implemented( + "search is not configured: start the server with SERPER_API_KEY set \ + (get a free key at serper.dev)", + ) + })?; + + let opts = webclaw_fetch::SearchOptions { + num_results: req.num_results, + country: req.country.clone(), + lang: req.lang.clone(), + scrape: req.scrape, + }; + + let results = webclaw_fetch::search(state.fetch(), serper_key, &req.query, &opts) + .await + .map_err(|e| ApiError::internal(format!("search failed: {e}")))?; + + Ok(Json(json!({ + "query": req.query, + "count": results.len(), + "results": results, + }))) +} diff --git a/crates/webclaw-server/src/state.rs b/crates/webclaw-server/src/state.rs index 6c2e8f7..fc31da0 100644 --- a/crates/webclaw-server/src/state.rs +++ b/crates/webclaw-server/src/state.rs @@ -36,6 +36,9 @@ struct Inner { pub fetch: Arc, /// Inbound bearer-auth token for this server's own `/v1/*` surface. pub api_key: Option, + /// Operator's own Serper.dev API key, read from `SERPER_API_KEY`. + /// Enables `/v1/search`. Unset = `/v1/search` returns 501. + pub serper_api_key: Option, } impl AppState { @@ -66,10 +69,20 @@ impl AppState { fetch = fetch.with_cloud(cloud); } + // Operator's own Serper.dev key enables /v1/search. Empty/unset + // leaves search returning 501 with a setup hint. + let serper_api_key = std::env::var("SERPER_API_KEY") + .ok() + .filter(|k| !k.trim().is_empty()); + if serper_api_key.is_some() { + info!("search enabled — using SERPER_API_KEY for /v1/search"); + } + Ok(Self { inner: Arc::new(Inner { fetch: Arc::new(fetch), api_key: inbound_api_key, + serper_api_key, }), }) } @@ -81,6 +94,11 @@ impl AppState { pub fn api_key(&self) -> Option<&str> { self.inner.api_key.as_deref() } + + /// Operator's Serper.dev key for `/v1/search`, if configured. + pub fn serper_api_key(&self) -> Option<&str> { + self.inner.serper_api_key.as_deref() + } } /// Resolve the outbound cloud key. Prefers `WEBCLAW_CLOUD_API_KEY`; From 179efbcf877e061d8e222c1ebdf1e53a0a4556a6 Mon Sep 17 00:00:00 2001 From: Valerio Date: Wed, 17 Jun 2026 15:33:49 +0200 Subject: [PATCH 02/46] feat(map): layered URL discovery with bounded crawl fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rescued from the stale perf/audit-fixes branch and ported cleanly onto current main (fetch + CLI only — the original commit never touched the server/MCP map surfaces). `--map` used to return only what a site advertises in sitemap.xml, which is nothing for sites with no sitemap (e.g. Hacker News) or a thin one. Now discovery is layered: - webclaw-fetch::discover_urls() / MapOptions — sitemaps first (authoritative, carries lastmod/priority/changefreq); when the sitemap is thin (< min_sitemap_urls) and the fallback is enabled, run a bounded same-origin crawl and harvest links from every fetched page plus the unfetched frontier, deduped against the sitemap set. - sitemap.rs: gzip (.xml.gz) support via a new decode_sitemap_body() + FetchClient::fetch_raw() (raw bytes, no lossy UTF-8); deeper index recursion (3->5); 4 more fallback paths. - CLI: --map-pages / --no-map-crawl / --map-limit; crawler logs now go to stderr so `--map -f json` stays machine-parseable. One new dependency: flate2 (already resolved in the lockfile transitively). Includes the commit's unit tests (map dedup/origin, gzip decode). Original work by the prior author on perf/audit-fixes; this re-applies only the map slice onto main. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 1 + crates/webclaw-cli/src/main.rs | 38 +++- crates/webclaw-fetch/Cargo.toml | 1 + crates/webclaw-fetch/src/client.rs | 28 +++ crates/webclaw-fetch/src/crawler.rs | 4 +- crates/webclaw-fetch/src/lib.rs | 2 + crates/webclaw-fetch/src/map.rs | 326 ++++++++++++++++++++++++++++ crates/webclaw-fetch/src/sitemap.rs | 97 ++++++++- 8 files changed, 485 insertions(+), 12 deletions(-) create mode 100644 crates/webclaw-fetch/src/map.rs diff --git a/Cargo.lock b/Cargo.lock index a7fd4de..fa9557d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3265,6 +3265,7 @@ dependencies = [ "async-trait", "bytes", "calamine", + "flate2", "futures-util", "http", "quick-xml 0.37.5", diff --git a/crates/webclaw-cli/src/main.rs b/crates/webclaw-cli/src/main.rs index 6f59be6..98e61ef 100644 --- a/crates/webclaw-cli/src/main.rs +++ b/crates/webclaw-cli/src/main.rs @@ -313,6 +313,18 @@ struct Cli { #[arg(long)] map: bool, + /// Max pages for --map's crawl fallback when the sitemap is thin [default: 150] + #[arg(long)] + map_pages: Option, + + /// Disable --map's crawl fallback (sitemap-only discovery) + #[arg(long)] + no_map_crawl: bool, + + /// Cap the number of URLs --map returns (default: uncapped) + #[arg(long)] + map_limit: Option, + // -- LLM options -- /// Extract structured JSON using LLM (pass a JSON schema string or @file) #[arg(long)] @@ -508,7 +520,13 @@ fn init_logging(verbose: bool) { EnvFilter::try_from_env("WEBCLAW_LOG").unwrap_or_else(|_| EnvFilter::new(default)) }; - tracing_subscriber::fmt().with_env_filter(filter).init(); + // Logs go to stderr, never stdout: stdout carries the actual result + // (markdown / JSON / URL list). A stray WARN on stdout corrupts + // machine-readable output — e.g. `--map --format json` piped to a parser. + tracing_subscriber::fmt() + .with_env_filter(filter) + .with_writer(std::io::stderr) + .init(); } /// Build FetchConfig from CLI flags. @@ -1688,12 +1706,22 @@ async fn run_map(cli: &Cli) -> Result<(), String> { let client = FetchClient::new(build_fetch_config(cli)).map_err(|e| format!("client error: {e}"))?; - let entries = webclaw_fetch::sitemap::discover(&client, url) - .await - .map_err(|e| format!("sitemap discovery failed: {e}"))?; + // Layered discovery: sitemaps first, bounded crawl fallback when thin. + let mut opts = webclaw_fetch::MapOptions::default(); + if let Some(pages) = cli.map_pages { + opts.max_crawl_pages = pages; + } + if cli.no_map_crawl { + opts.crawl_fallback = false; + } + if let Some(limit) = cli.map_limit { + opts.max_urls = Some(limit); + } + + let entries = webclaw_fetch::discover_urls(&client, url, &opts).await; if entries.is_empty() { - eprintln!("no sitemap URLs found for {url}"); + eprintln!("no URLs found for {url}"); } else { eprintln!("discovered {} URLs", entries.len()); } diff --git a/crates/webclaw-fetch/Cargo.toml b/crates/webclaw-fetch/Cargo.toml index 4671dc1..3ac6f18 100644 --- a/crates/webclaw-fetch/Cargo.toml +++ b/crates/webclaw-fetch/Cargo.toml @@ -32,6 +32,7 @@ reqwest = { version = "0.12", default-features = false, features = ["json", "rus serde_json.workspace = true calamine = "0.34" zip = "2" +flate2 = "1" [dev-dependencies] tempfile = "3" diff --git a/crates/webclaw-fetch/src/client.rs b/crates/webclaw-fetch/src/client.rs index 7553bb5..0ef270a 100644 --- a/crates/webclaw-fetch/src/client.rs +++ b/crates/webclaw-fetch/src/client.rs @@ -170,6 +170,13 @@ impl Response { fn into_text(self) -> String { String::from_utf8_lossy(&self.body).into_owned() } + + /// Consume the response and return the raw, undecoded body bytes. + /// Used by [`FetchClient::fetch_raw`] for binary payloads (e.g. gzipped + /// sitemaps) that must not be run through lossy UTF-8 decoding. + fn into_body(self) -> bytes::Bytes { + self.body + } } /// Internal representation of the client pool strategy. @@ -458,6 +465,27 @@ impl FetchClient { Err(last_err.unwrap_or_else(|| FetchError::Build("all retries exhausted".into()))) } + /// Fetch a URL and return the raw, undecoded response body as bytes. + /// + /// Unlike [`fetch`](Self::fetch), this does **not** run the body through + /// `String::from_utf8_lossy`, so binary payloads survive intact. This is + /// required for gzipped sitemaps (`.xml.gz`): such files are served with + /// `Content-Type: application/gzip` and *no* `Content-Encoding`, so wreq + /// never auto-inflates them — the bytes arrive as raw gzip and the lossy + /// String path would mangle them. Callers detect the gzip magic + /// (`0x1f 0x8b`) and gunzip before parsing. + /// + /// No retry wrapper: callers (sitemap discovery) already tolerate + /// per-URL failures by skipping. Returns `(status, body)`. + pub async fn fetch_raw(&self, url: &str) -> Result<(u16, bytes::Bytes), FetchError> { + let parsed_url = crate::url_security::validate_public_http_url(url).await?; + let url = parsed_url.as_str(); + let client = self.pick_client(url); + let resp = client.get(url).send().await?; + let response = Response::from_wreq(resp).await?; + Ok((response.status(), response.into_body())) + } + /// Fetch a URL then extract structured content. #[instrument(skip(self), fields(url = %url))] pub async fn fetch_and_extract( diff --git a/crates/webclaw-fetch/src/crawler.rs b/crates/webclaw-fetch/src/crawler.rs index 1403256..581e2f7 100644 --- a/crates/webclaw-fetch/src/crawler.rs +++ b/crates/webclaw-fetch/src/crawler.rs @@ -528,7 +528,7 @@ impl Crawler { } /// Canonical origin string for comparing same-origin: "scheme://host[:port]". -fn origin_key(url: &Url) -> String { +pub(crate) fn origin_key(url: &Url) -> String { let port_suffix = match url.port() { Some(p) => format!(":{p}"), None => String::new(), @@ -563,7 +563,7 @@ fn root_domain(url: &Url) -> String { /// Normalize a URL for dedup: strip fragment, remove trailing slash (except root "/"), /// lowercase scheme + host. Preserves query params and path case. -fn normalize(url: &Url) -> String { +pub(crate) fn normalize(url: &Url) -> String { let scheme = url.scheme(); let host = url.host_str().unwrap_or("").to_ascii_lowercase(); let port_suffix = match url.port() { diff --git a/crates/webclaw-fetch/src/lib.rs b/crates/webclaw-fetch/src/lib.rs index a7a95e6..0fe7dfa 100644 --- a/crates/webclaw-fetch/src/lib.rs +++ b/crates/webclaw-fetch/src/lib.rs @@ -11,6 +11,7 @@ pub mod extractors; pub mod fetcher; pub mod linkedin; pub mod locale; +pub mod map; pub mod progress; pub mod proxy; pub mod reddit; @@ -26,6 +27,7 @@ pub use error::FetchError; pub use fetcher::Fetcher; pub use http::HeaderMap; pub use locale::{accept_language_for_tld, accept_language_for_url}; +pub use map::{MapOptions, discover_urls}; pub use progress::{PROGRESS_INTERVAL, with_progress}; pub use proxy::{parse_proxy_file, parse_proxy_line}; pub use search::{SearchOptions, SearchResult, parse_serper_organic, search}; diff --git a/crates/webclaw-fetch/src/map.rs b/crates/webclaw-fetch/src/map.rs new file mode 100644 index 0000000..97219fc --- /dev/null +++ b/crates/webclaw-fetch/src/map.rs @@ -0,0 +1,326 @@ +//! Layered URL discovery for the `map` command. +//! +//! `sitemap::discover` only finds URLs a site explicitly advertises in its +//! `sitemap.xml`. Plenty of sites have no sitemap (news.ycombinator.com), a +//! stale one, or a thin one that lists a handful of section roots. For those, +//! a sitemap-only map returns almost nothing. +//! +//! This module adds a second layer: when the sitemap yields fewer than a +//! threshold of URLs, run a *bounded* same-origin crawl and harvest every URL +//! it touches — fetched pages, the visited set, **and** the remaining frontier +//! (links queued but never fetched because the page cap was hit). That last +//! bucket is the gold: a 150-page crawl of a link-dense site surfaces several +//! thousand frontier URLs, turning a useless map into a real one. +//! +//! Strategy (layered, sitemap-first): +//! 1. Sitemaps via [`sitemap::discover`] — authoritative, carries metadata +//! (lastmod / priority / changefreq). +//! 2. If sitemaps are thin (`< min_sitemap_urls`) and the fallback is enabled, +//! a bounded crawl fills in the rest. Crawl-discovered URLs carry no +//! metadata (`None` everywhere) since they come from link harvesting, not a +//! sitemap. +//! +//! Sitemap entries always come first in the returned vec; crawl-discovered +//! URLs are appended, deduplicated against the sitemap set using the *same* +//! normalization the crawler uses ([`crawler::normalize`]) so map output stays +//! internally consistent. + +use std::collections::HashSet; +use std::time::Duration; + +use url::Url; + +use crate::client::{FetchClient, FetchConfig}; +use crate::crawler::{self, CrawlConfig, Crawler}; +use crate::sitemap::{self, SitemapEntry}; + +/// Tuning knobs for [`discover_urls`]. +#[derive(Debug, Clone)] +pub struct MapOptions { + /// Hard cap on pages the fallback crawl will fetch. The crawl surfaces far + /// more URLs than this via the unfetched frontier, so a small number still + /// yields a large map while keeping the crawl fast and polite. + pub max_crawl_pages: usize, + /// How deep the fallback crawl follows links (1 = links off the seed only). + pub crawl_depth: usize, + /// Sitemap-URL count below which the crawl fallback kicks in. A site with a + /// rich sitemap (≥ this many URLs) skips the crawl entirely. + pub min_sitemap_urls: usize, + /// Master switch for the crawl fallback. When `false`, behaves exactly like + /// the old sitemap-only `discover`. + pub crawl_fallback: bool, + /// Optional cap on URLs returned. `None` (default) = uncapped: return every + /// URL discovered (the crawl is already bounded by `max_crawl_pages`, so the + /// uncapped set is the links harvested from the fetched pages). Set `Some(n)` + /// to truncate. + pub max_urls: Option, +} + +impl Default for MapOptions { + fn default() -> Self { + Self { + max_crawl_pages: 150, + crawl_depth: 2, + min_sitemap_urls: 200, + crawl_fallback: true, + max_urls: None, + } + } +} + +/// Discover URLs for a site using the layered strategy described in the module +/// docs: sitemaps first, then a bounded crawl fallback when the sitemap is +/// thin. +/// +/// Never errors — sitemap and crawl failures are swallowed and simply yield +/// fewer URLs (an empty vec in the worst case), matching `sitemap::discover`'s +/// "absence is not an error" contract. +pub async fn discover_urls( + client: &FetchClient, + base_url: &str, + opts: &MapOptions, +) -> Vec { + // Layer 1: sitemaps. + let mut entries = sitemap::discover(client, base_url) + .await + .unwrap_or_default(); + + // Track normalized URLs we've already emitted, for cross-layer dedup. + let mut seen: HashSet = entries.iter().filter_map(normalize_str).collect(); + + // Layer 2: bounded crawl fallback, only when the sitemap is thin. + if !opts.crawl_fallback || entries.len() >= opts.min_sitemap_urls { + return entries; + } + + let Some(base_origin) = Url::parse(base_url).ok().map(|u| crawler::origin_key(&u)) else { + // Unparseable base URL — nothing sensible to crawl against. + return entries; + }; + + let config = CrawlConfig { + fetch: FetchConfig::default(), + max_depth: opts.crawl_depth, + max_pages: opts.max_crawl_pages, + // Politeness + scope: same-origin only (crawler default), modest delay. + delay: Duration::from_millis(50), + ..CrawlConfig::default() + }; + + let crawler = match Crawler::new(base_url, config) { + Ok(c) => c, + Err(_) => return entries, + }; + + let result = crawler.crawl(base_url, None).await; + + // Richest source first: every link harvested from each fetched page. A + // directory/index page holds hundreds of same-origin links, and this set is + // NOT bound by the crawler's internal frontier cap. Then the URLs the crawl + // itself touched (fetched, visited, queued-but-unfetched frontier). + let mut discovered: Vec = Vec::new(); + for p in &result.pages { + discovered.push(p.url.clone()); + if let Some(ex) = p.extraction.as_ref() { + let page_base = Url::parse(&p.url).ok(); + for link in &ex.content.links { + // Resolve relative/protocol-relative hrefs against the page URL + // so the same-origin filter and dedup see absolute URLs. + let abs = match &page_base { + Some(b) => b.join(&link.href).ok(), + None => Url::parse(&link.href).ok(), + }; + if let Some(u) = abs { + discovered.push(u.to_string()); + } + } + } + } + discovered.extend(result.visited); + discovered.extend(result.remaining_frontier.into_iter().map(|(url, _)| url)); + + append_crawled(&mut entries, &mut seen, discovered, &base_origin); + + // Uncapped by default; only truncate if the caller set an explicit limit + // (sitemap entries added first keep priority). + if let Some(cap) = opts.max_urls { + entries.truncate(cap); + } + entries +} + +/// Normalize a raw URL string to the crawler's canonical form, returning `None` +/// if it doesn't parse. +fn normalize_url(raw: &str) -> Option { + Url::parse(raw).ok().map(|u| crawler::normalize(&u)) +} + +/// Normalize a [`SitemapEntry`]'s URL for the dedup set. +fn normalize_str(entry: &SitemapEntry) -> Option { + normalize_url(&entry.url) +} + +/// Append crawl-discovered URLs to `entries`, skipping any that are off-origin, +/// unparseable, or already present (by normalized form). +/// +/// Split out from [`discover_urls`] so the union/dedup/same-origin logic is +/// unit-testable without touching the network. Mutates `entries` and `seen` in +/// place; crawl URLs get empty metadata. +fn append_crawled( + entries: &mut Vec, + seen: &mut HashSet, + discovered: impl IntoIterator, + base_origin: &str, +) { + for raw in discovered { + let Ok(parsed) = Url::parse(&raw) else { + continue; + }; + // Same-origin filter: drop anything whose origin differs from the seed. + if crawler::origin_key(&parsed) != base_origin { + continue; + } + let norm = crawler::normalize(&parsed); + if seen.insert(norm.clone()) { + entries.push(SitemapEntry { + url: norm, + last_modified: None, + priority: None, + change_freq: None, + }); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn entry(url: &str) -> SitemapEntry { + SitemapEntry { + url: url.to_string(), + last_modified: None, + priority: None, + change_freq: None, + } + } + + fn origin_of(url: &str) -> String { + crawler::origin_key(&Url::parse(url).unwrap()) + } + + #[test] + fn append_adds_new_same_origin_urls() { + let mut entries = vec![entry("https://example.com/")]; + let mut seen: HashSet = entries.iter().filter_map(normalize_str).collect(); + + append_crawled( + &mut entries, + &mut seen, + vec![ + "https://example.com/about".to_string(), + "https://example.com/contact".to_string(), + ], + &origin_of("https://example.com"), + ); + + let urls: Vec<&str> = entries.iter().map(|e| e.url.as_str()).collect(); + assert_eq!( + urls, + vec![ + "https://example.com/", + "https://example.com/about", + "https://example.com/contact", + ] + ); + } + + #[test] + fn append_dedups_against_sitemap_and_self() { + let mut entries = vec![entry("https://example.com/about")]; + let mut seen: HashSet = entries.iter().filter_map(normalize_str).collect(); + + append_crawled( + &mut entries, + &mut seen, + vec![ + // Same as sitemap entry (trailing slash normalizes away). + "https://example.com/about/".to_string(), + // Fragment + duplicate -> only one new entry survives. + "https://example.com/new#frag".to_string(), + "https://example.com/new".to_string(), + ], + &origin_of("https://example.com"), + ); + + let urls: Vec<&str> = entries.iter().map(|e| e.url.as_str()).collect(); + assert_eq!( + urls, + vec!["https://example.com/about", "https://example.com/new"] + ); + } + + #[test] + fn append_filters_off_origin() { + let mut entries = Vec::new(); + let mut seen = HashSet::new(); + + append_crawled( + &mut entries, + &mut seen, + vec![ + "https://example.com/keep".to_string(), + "https://evil.com/drop".to_string(), + "https://sub.example.com/drop".to_string(), // different origin + "ftp://example.com/drop".to_string(), // unparseable as http origin match + ], + &origin_of("https://example.com"), + ); + + let urls: Vec<&str> = entries.iter().map(|e| e.url.as_str()).collect(); + assert_eq!(urls, vec!["https://example.com/keep"]); + } + + #[test] + fn append_treats_www_as_same_origin() { + // origin_key strips a leading `www.`, so www and apex collapse. + let mut entries = Vec::new(); + let mut seen = HashSet::new(); + + append_crawled( + &mut entries, + &mut seen, + vec!["https://www.example.com/page".to_string()], + &origin_of("https://example.com"), + ); + + assert_eq!(entries.len(), 1); + } + + #[test] + fn crawl_urls_carry_no_metadata() { + let mut entries = Vec::new(); + let mut seen = HashSet::new(); + + append_crawled( + &mut entries, + &mut seen, + vec!["https://example.com/x".to_string()], + &origin_of("https://example.com"), + ); + + assert_eq!(entries.len(), 1); + assert!(entries[0].last_modified.is_none()); + assert!(entries[0].priority.is_none()); + assert!(entries[0].change_freq.is_none()); + } + + #[test] + fn map_options_defaults() { + let o = MapOptions::default(); + assert_eq!(o.max_crawl_pages, 150); + assert_eq!(o.crawl_depth, 2); + assert_eq!(o.min_sitemap_urls, 200); + assert!(o.crawl_fallback); + } +} diff --git a/crates/webclaw-fetch/src/sitemap.rs b/crates/webclaw-fetch/src/sitemap.rs index 931db32..81140dc 100644 --- a/crates/webclaw-fetch/src/sitemap.rs +++ b/crates/webclaw-fetch/src/sitemap.rs @@ -18,12 +18,20 @@ use crate::error::FetchError; /// Maximum depth when recursively fetching sitemap index files. /// Prevents infinite loops from circular sitemap references. -const MAX_RECURSION_DEPTH: usize = 3; +/// +/// Raised 3→5: large sites (gov.uk, news publishers) nest sitemap indexes +/// more than three levels deep — a top index → per-section index → +/// per-month index → urlset is already four hops. Three cut those off. +const MAX_RECURSION_DEPTH: usize = 5; /// Common sitemap paths to try when robots.txt doesn't list any. const FALLBACK_SITEMAP_PATHS: &[&str] = &[ "/sitemap.xml", "/sitemap_index.xml", + "/sitemap-index.xml", + "/sitemap1.xml", + "/sitemaps.xml", + "/sitemap/index.xml", "/wp-sitemap.xml", "/sitemap/sitemap-index.xml", ]; @@ -105,10 +113,12 @@ async fn fetch_sitemaps( for sitemap_url in urls { debug!(url = %sitemap_url, depth, "fetching sitemap"); - let xml = match client.fetch(sitemap_url).await { - Ok(result) if result.status == 200 => result.html, - Ok(result) => { - debug!(url = %sitemap_url, status = result.status, "sitemap not found"); + // Fetch raw bytes so gzipped sitemaps survive intact. `fetch` runs + // the body through `from_utf8_lossy`, which corrupts binary gzip. + let body = match client.fetch_raw(sitemap_url).await { + Ok((200, body)) => body, + Ok((status, _)) => { + debug!(url = %sitemap_url, status, "sitemap not found"); continue; } Err(e) => { @@ -117,6 +127,14 @@ async fn fetch_sitemaps( } }; + let xml = match decode_sitemap_body(&body) { + Some(xml) => xml, + None => { + debug!(url = %sitemap_url, "failed to decode sitemap body, skipping"); + continue; + } + }; + match detect_sitemap_type(&xml) { SitemapType::UrlSet => { let parsed = parse_urlset(&xml); @@ -147,6 +165,33 @@ async fn fetch_sitemaps( } } +/// Decode a raw sitemap body into a UTF-8 XML string. +/// +/// Sitemaps are commonly served gzipped (`.xml.gz`) with +/// `Content-Type: application/gzip` and *no* `Content-Encoding`, so the HTTP +/// layer never inflates them. We detect the gzip magic bytes (`0x1f 0x8b`) +/// and gunzip in-process; otherwise the body is treated as plain XML. +/// +/// Returns `None` if a gzip stream fails to inflate. Plain (non-gzip) bodies +/// always succeed via lossy UTF-8 decode, mirroring the previous behaviour. +pub(crate) fn decode_sitemap_body(body: &[u8]) -> Option { + if body.starts_with(&[0x1f, 0x8b]) { + use std::io::Read; + + let mut decoder = flate2::read::GzDecoder::new(body); + let mut out = String::new(); + match decoder.read_to_string(&mut out) { + Ok(_) => Some(out), + Err(e) => { + warn!(error = %e, "failed to gunzip sitemap body"); + None + } + } + } else { + Some(String::from_utf8_lossy(body).into_owned()) + } +} + // --------------------------------------------------------------------------- // Pure parsing functions (no I/O, fully testable) // --------------------------------------------------------------------------- @@ -669,5 +714,47 @@ mod tests { assert!(FALLBACK_SITEMAP_PATHS.contains(&"/sitemap_index.xml")); assert!(FALLBACK_SITEMAP_PATHS.contains(&"/wp-sitemap.xml")); assert!(FALLBACK_SITEMAP_PATHS.contains(&"/sitemap/sitemap-index.xml")); + // Paths added for robustness (item 3). + assert!(FALLBACK_SITEMAP_PATHS.contains(&"/sitemap-index.xml")); + assert!(FALLBACK_SITEMAP_PATHS.contains(&"/sitemap1.xml")); + assert!(FALLBACK_SITEMAP_PATHS.contains(&"/sitemaps.xml")); + assert!(FALLBACK_SITEMAP_PATHS.contains(&"/sitemap/index.xml")); + } + + #[test] + fn decode_plain_xml_body() { + let xml = r#""#; + let got = decode_sitemap_body(xml.as_bytes()).expect("plain body decodes"); + assert_eq!(got, xml); + } + + #[test] + fn decode_gzipped_body() { + use std::io::Write; + + let xml = r#" + + https://example.com/gz-page +"#; + + // Gzip-compress the XML, then confirm decode_sitemap_body inflates it + // and the parser finds the URL. + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(xml.as_bytes()).unwrap(); + let gz = encoder.finish().unwrap(); + + assert_eq!(&gz[..2], &[0x1f, 0x8b], "gzip magic present"); + + let decoded = decode_sitemap_body(&gz).expect("gzip body inflates"); + let entries = parse_urlset(&decoded); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].url, "https://example.com/gz-page"); + } + + #[test] + fn decode_corrupt_gzip_returns_none() { + // Starts with gzip magic but the rest is garbage -> inflate fails. + let bad = [0x1f, 0x8b, 0x08, 0x00, 0xde, 0xad, 0xbe, 0xef]; + assert!(decode_sitemap_body(&bad).is_none()); } } From 884f06a5d3c75e6081544fa63eebaad57c0b44a0 Mon Sep 17 00:00:00 2001 From: Valerio Date: Wed, 17 Jun 2026 15:37:36 +0200 Subject: [PATCH 03/46] fix(mcp): accept boolean params sent as JSON strings (#62) Follow-up to #58/#59, which fixed numeric params but left the booleans. MCP clients (e.g. Claude Desktop) send `true` as the JSON string `"true"`, which serde's default bool deserializer rejects with `invalid type: string "true", expected a boolean`, failing the call. Adds a `deser_opt_bool_or_str` helper (same untagged pattern as the #59 numeric helpers) that accepts a JSON boolean OR "true"/"false" (case-insensitive, trimmed) and rejects anything else with a clear error. Numeric-looking strings like "1" are intentionally NOT coerced to bool. Applied to every Option tool param: - scrape -> only_main_content - crawl -> use_sitemap - research -> deep - search -> scrape (added by the standalone-search slice, #63) 16 unit tests (bool / "true"-string / absent->None / garbage->error per field). No new dependencies. Fixes #62. Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/webclaw-mcp/src/tools.rs | 149 +++++++++++++++++++++++++++++++- 1 file changed, 147 insertions(+), 2 deletions(-) diff --git a/crates/webclaw-mcp/src/tools.rs b/crates/webclaw-mcp/src/tools.rs index f17933f..c20a9e8 100644 --- a/crates/webclaw-mcp/src/tools.rs +++ b/crates/webclaw-mcp/src/tools.rs @@ -12,8 +12,11 @@ use serde::Deserialize; // // "invalid type: string \"3\", expected u32" // -// These two helpers accept both forms transparently so callers never see that -// error regardless of which representation their client sends. +// These helpers accept both forms transparently so callers never see that +// error regardless of which representation their client sends. The same +// problem hits booleans: clients send `"true"`/`"false"` as JSON strings, +// which serde's default bool deserialiser rejects — `deser_opt_bool_or_str` +// covers that case. fn deser_opt_u32_or_str<'de, D>(d: D) -> Result, D::Error> where @@ -57,6 +60,31 @@ where } } +fn deser_opt_bool_or_str<'de, D>(d: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + #[derive(serde::Deserialize)] + #[serde(untagged)] + enum BoolOrStr { + Bool(bool), + Str(String), + } + match Option::::deserialize(d)? { + None => Ok(None), + Some(BoolOrStr::Bool(b)) => Ok(Some(b)), + // Accept "true"/"false" case-insensitively (trimmed). Reject anything + // else with a clear message rather than silently coercing it. + Some(BoolOrStr::Str(s)) => match s.trim().to_ascii_lowercase().as_str() { + "true" => Ok(Some(true)), + "false" => Ok(Some(false)), + _ => Err(serde::de::Error::custom(format!( + "expected a bool, got string \"{s}\"" + ))), + }, + } +} + // ── Parameter structs ─────────────────────────────────────────────────────── #[derive(Debug, Deserialize, JsonSchema)] @@ -70,6 +98,7 @@ pub struct ScrapeParams { /// CSS selectors to exclude from output pub exclude_selectors: Option>, /// If true, extract only the main content (article/main element) + #[serde(default, deserialize_with = "deser_opt_bool_or_str")] pub only_main_content: Option, /// Browser profile: "chrome" (default), "firefox", or "random" pub browser: Option, @@ -91,6 +120,7 @@ pub struct CrawlParams { #[serde(default, deserialize_with = "deser_opt_usize_or_str")] pub concurrency: Option, /// Seed the frontier from sitemap discovery before crawling + #[serde(default, deserialize_with = "deser_opt_bool_or_str")] pub use_sitemap: Option, /// Output format for each page: "markdown" (default), "llm", "text" pub format: Option, @@ -151,6 +181,7 @@ pub struct ResearchParams { /// Research query or question to investigate pub query: String, /// Enable deep research mode for more thorough investigation (default: false) + #[serde(default, deserialize_with = "deser_opt_bool_or_str")] pub deep: Option, /// Topic hint to guide research focus (e.g. "technology", "finance", "science") pub topic: Option, @@ -171,6 +202,7 @@ pub struct SearchParams { pub lang: Option, /// When true, fetch + extract each result page and include its /// markdown. Only used by the local Serper path (SERPER_API_KEY). + #[serde(default, deserialize_with = "deser_opt_bool_or_str")] pub scrape: Option, } @@ -365,4 +397,117 @@ mod tests { ); assert!(e.is_err(), "expected Err, got {e:?}"); } + + // ── Boolean param string-coercion (issue #62) ─────────────────────────── + + // ScrapeParams.only_main_content + #[test] + fn scrape_only_main_content_from_bool() { + let v: ScrapeParams = + serde_json::from_str(r#"{"url":"https://x.com","only_main_content":true}"#).unwrap(); + assert_eq!(v.only_main_content, Some(true)); + } + + #[test] + fn scrape_only_main_content_from_string() { + let t: ScrapeParams = + serde_json::from_str(r#"{"url":"https://x.com","only_main_content":"true"}"#).unwrap(); + assert_eq!(t.only_main_content, Some(true)); + let f: ScrapeParams = + serde_json::from_str(r#"{"url":"https://x.com","only_main_content":"false"}"#).unwrap(); + assert_eq!(f.only_main_content, Some(false)); + } + + #[test] + fn scrape_only_main_content_absent_is_none() { + let v: ScrapeParams = serde_json::from_str(r#"{"url":"https://x.com"}"#).unwrap(); + assert_eq!(v.only_main_content, None); + } + + #[test] + fn scrape_only_main_content_non_bool_string_errors() { + let e = serde_json::from_str::( + r#"{"url":"https://x.com","only_main_content":"yes"}"#, + ); + assert!(e.is_err(), "expected Err, got {e:?}"); + } + + // CrawlParams.use_sitemap + #[test] + fn crawl_use_sitemap_from_bool() { + let v: CrawlParams = + serde_json::from_str(r#"{"url":"https://x.com","use_sitemap":false}"#).unwrap(); + assert_eq!(v.use_sitemap, Some(false)); + } + + #[test] + fn crawl_use_sitemap_from_string() { + let v: CrawlParams = + serde_json::from_str(r#"{"url":"https://x.com","use_sitemap":"true"}"#).unwrap(); + assert_eq!(v.use_sitemap, Some(true)); + } + + #[test] + fn crawl_use_sitemap_absent_is_none() { + let v: CrawlParams = serde_json::from_str(r#"{"url":"https://x.com"}"#).unwrap(); + assert_eq!(v.use_sitemap, None); + } + + #[test] + fn crawl_use_sitemap_non_bool_string_errors() { + let e = + serde_json::from_str::(r#"{"url":"https://x.com","use_sitemap":"nope"}"#); + assert!(e.is_err(), "expected Err, got {e:?}"); + } + + // ResearchParams.deep + #[test] + fn research_deep_from_bool() { + let v: ResearchParams = serde_json::from_str(r#"{"query":"rust","deep":true}"#).unwrap(); + assert_eq!(v.deep, Some(true)); + } + + #[test] + fn research_deep_from_string() { + let v: ResearchParams = serde_json::from_str(r#"{"query":"rust","deep":"true"}"#).unwrap(); + assert_eq!(v.deep, Some(true)); + } + + #[test] + fn research_deep_absent_is_none() { + let v: ResearchParams = serde_json::from_str(r#"{"query":"rust"}"#).unwrap(); + assert_eq!(v.deep, None); + } + + #[test] + fn research_deep_non_bool_string_errors() { + // Numeric-looking strings are NOT accepted for bools (avoids ambiguity). + let e = serde_json::from_str::(r#"{"query":"rust","deep":"1"}"#); + assert!(e.is_err(), "expected Err, got {e:?}"); + } + + // SearchParams.scrape + #[test] + fn search_scrape_from_bool() { + let v: SearchParams = serde_json::from_str(r#"{"query":"rust","scrape":true}"#).unwrap(); + assert_eq!(v.scrape, Some(true)); + } + + #[test] + fn search_scrape_from_string_case_insensitive() { + let v: SearchParams = serde_json::from_str(r#"{"query":"rust","scrape":"True"}"#).unwrap(); + assert_eq!(v.scrape, Some(true)); + } + + #[test] + fn search_scrape_absent_is_none() { + let v: SearchParams = serde_json::from_str(r#"{"query":"rust"}"#).unwrap(); + assert_eq!(v.scrape, None); + } + + #[test] + fn search_scrape_non_bool_string_errors() { + let e = serde_json::from_str::(r#"{"query":"rust","scrape":"maybe"}"#); + assert!(e.is_err(), "expected Err, got {e:?}"); + } } From 51d0c538f1c6866fbdb6d591f5ff4999c8bd1448 Mon Sep 17 00:00:00 2001 From: Valerio Date: Wed, 17 Jun 2026 16:10:45 +0200 Subject: [PATCH 04/46] chore(release): bump version to 0.6.12 Bundle three changes landed since 0.6.11: - feat(search): standalone web search via Serper.dev (#63) - feat(map): layered URL discovery with bounded crawl fallback (#64) - fix(mcp): accept boolean params sent as JSON strings (#62 / #65) Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 9 +++++++++ Cargo.lock | 14 +++++++------- Cargo.toml | 2 +- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ddcf40e..6bafbce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,15 @@ Format follows [Keep a Changelog](https://keepachangelog.com/). ## [Unreleased] +## [0.6.12] - 2026-06-17 + +### Added +- **Standalone web search** using your own [Serper.dev](https://serper.dev) key — no hosted webclaw account needed. Available across the CLI (`webclaw search "query" --num 5 --scrape`, key via `--serper-key` or `SERPER_API_KEY`), the MCP `search` tool (local-first when `SERPER_API_KEY` is set, hosted API otherwise), and the self-hosted REST server (`POST /v1/search`, enabled when started with `SERPER_API_KEY`). With `--scrape`, the top result pages are fetched and extracted to markdown. +- **Layered URL discovery for `--map`**: when a site has no sitemap or only a thin one, map now falls back to a bounded same-origin crawl and harvests links from every fetched page plus the unfetched frontier, returning far more URLs. Adds gzipped-sitemap (`.xml.gz`) support, deeper sitemap-index recursion, more fallback paths, and `--map-pages` / `--no-map-crawl` / `--map-limit` controls. Crawler logs now go to stderr so `--map --format json` stays machine-parseable. + +### Fixed +- MCP tools now accept boolean arguments whether the client sends them as JSON booleans or as the strings `"true"`/`"false"` (case-insensitive). Some MCP clients (e.g. Claude Desktop) send booleans as strings, which previously failed the call with a deserialization error. Affects `scrape` (only_main_content), `crawl` (use_sitemap), `research` (deep), and `search` (scrape). This completes the earlier numeric-parameter fix. + ## [0.6.11] - 2026-06-16 ### Added diff --git a/Cargo.lock b/Cargo.lock index fa9557d..6d06447 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3221,7 +3221,7 @@ dependencies = [ [[package]] name = "webclaw-cli" -version = "0.6.11" +version = "0.6.12" dependencies = [ "clap", "dotenvy", @@ -3242,7 +3242,7 @@ dependencies = [ [[package]] name = "webclaw-core" -version = "0.6.11" +version = "0.6.12" dependencies = [ "ego-tree", "once_cell", @@ -3260,7 +3260,7 @@ dependencies = [ [[package]] name = "webclaw-fetch" -version = "0.6.11" +version = "0.6.12" dependencies = [ "async-trait", "bytes", @@ -3288,7 +3288,7 @@ dependencies = [ [[package]] name = "webclaw-llm" -version = "0.6.11" +version = "0.6.12" dependencies = [ "async-trait", "reqwest", @@ -3301,7 +3301,7 @@ dependencies = [ [[package]] name = "webclaw-mcp" -version = "0.6.11" +version = "0.6.12" dependencies = [ "dirs", "dotenvy", @@ -3321,7 +3321,7 @@ dependencies = [ [[package]] name = "webclaw-pdf" -version = "0.6.11" +version = "0.6.12" dependencies = [ "pdf-extract", "thiserror", @@ -3330,7 +3330,7 @@ dependencies = [ [[package]] name = "webclaw-server" -version = "0.6.11" +version = "0.6.12" dependencies = [ "anyhow", "axum", diff --git a/Cargo.toml b/Cargo.toml index bf7a8bc..a748849 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ resolver = "2" members = ["crates/*"] [workspace.package] -version = "0.6.11" +version = "0.6.12" edition = "2024" license = "AGPL-3.0" repository = "https://github.com/0xMassi/webclaw" From 3c54bea300b10e3aad8e97b8607a79201e9755fa Mon Sep 17 00:00:00 2001 From: Valerio Date: Wed, 17 Jun 2026 16:41:45 +0200 Subject: [PATCH 05/46] perf: hot-path extraction speedups (selector hoist, shared og, QuickJS gating) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rescued from the stale perf/audit-fixes branch — the *perf-only* subset of that branch's big mixed commit, ported cleanly onto current main with byte-identical extraction output. - markdown: hoist the `img[alt]` / `a[href]` selectors out of the per-node noise path into `Lazy` statics (stop recompiling them per element). - extractors: single shared `og()` / `parse_og()` module replaces the per-field Open Graph re-scan duplicated across 7 vertical extractors (amazon, ebay, ecommerce, etsy, substack, trustpilot, youtube). Each vertical now does one pass. Raw-vs-unescaped behaviour preserved exactly. - core: gate the QuickJS VM on a cheap marker check (skip it entirely when the page has no JS-assigned data) and reuse the already-parsed document instead of re-parsing the HTML. - fetch: connection-pool tuning on the wreq client (connect_timeout, idle pool, max-idle-per-host, tcp keepalive) for connection reuse. Output-equivalence is covered by existing tests (amazon quot-entity, trustpilot title parse, ecommerce/youtube/etsy/substack og fallbacks) — all green. No new dependencies; no public API change. Deliberately EXCLUDED from this slice (separate concerns bundled in the original commit): the `#[non_exhaustive]` API-breaking changes, the LLM/PDF/ server reliability hardening (much already shipped in 0.6.8), the tooling (cargo-deny, release profile, MSRV), and the retry-loop dedup refactor (a code-cleanup with no runtime benefit — not worth churning client.rs for). Original work by the prior author on perf/audit-fixes; this re-applies only the performance subset onto main. Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/webclaw-core/src/js_eval.rs | 31 ++++++++ crates/webclaw-core/src/lib.rs | 4 +- crates/webclaw-core/src/markdown.rs | 6 +- .../src/extractors/amazon_product.rs | 36 ++------- .../src/extractors/ebay_listing.rs | 22 ++---- .../src/extractors/ecommerce_product.rs | 23 ++---- .../src/extractors/etsy_listing.rs | 27 +++---- crates/webclaw-fetch/src/extractors/mod.rs | 1 + crates/webclaw-fetch/src/extractors/og.rs | 79 +++++++++++++++++++ .../src/extractors/substack_post.rs | 25 ++---- .../src/extractors/trustpilot_reviews.rs | 75 ++++++++---------- .../src/extractors/youtube_video.rs | 22 ++---- crates/webclaw-fetch/src/tls.rs | 6 +- 13 files changed, 200 insertions(+), 157 deletions(-) create mode 100644 crates/webclaw-fetch/src/extractors/og.rs diff --git a/crates/webclaw-core/src/js_eval.rs b/crates/webclaw-core/src/js_eval.rs index e1fb2de..2f78246 100644 --- a/crates/webclaw-core/src/js_eval.rs +++ b/crates/webclaw-core/src/js_eval.rs @@ -16,6 +16,29 @@ static SCRIPT_SELECTOR: Lazy = Lazy::new(|| Selector::parse("script"). static HTML_TAG_RE: Lazy = Lazy::new(|| Regex::new(r"<[^>]+>").unwrap()); const JS_EVAL_TIMEOUT: Duration = Duration::from_millis(250); +/// Markers that, if absent from the HTML, prove the QuickJS scan cannot find +/// any data blob. The scan only ever surfaces `globalThis.__*` object/array +/// properties, and the seeded `__next_f` only emits when non-empty. Every +/// realistic way an inline script populates such a global goes through one of +/// these substrings (`window.`/`self.__next` assignments, or the +/// `__NEXT_DATA__`/`__NUXT__`/`application/json` payload conventions). If none +/// are present, running the VM is guaranteed to return zero blobs, so skipping +/// it is output-neutral. Conservative by design: any of these may appear in +/// non-script HTML too, which only makes us skip *less* often, never more. +const JS_CANDIDATE_MARKERS: [&str; 5] = [ + "window.", + "__NEXT_DATA__", + "__NUXT__", + "application/json", + "self.__next", +]; + +/// Returns true if the HTML plausibly contains JS-assigned data the QuickJS +/// scan could surface. When false, the VM is provably a no-op and is skipped. +pub fn has_js_candidate_data(html: &str) -> bool { + JS_CANDIDATE_MARKERS.iter().any(|m| html.contains(m)) +} + /// A blob of data extracted from JS execution. pub struct JsDataBlob { pub name: String, @@ -24,9 +47,17 @@ pub struct JsDataBlob { } /// Execute inline `