perf(core): hot-path extraction speedups + senior-grade hardening

Extraction ~22% faster on the corpus benchmark with byte-identical output:
- hoist recompiled CSS selectors in the markdown noise path
- single-pass shared og() meta parsing across vertical extractors
- output-safe QuickJS gating (skip the JS VM when no candidate data) +
  reuse the already-parsed document instead of re-parsing
- wreq connect_timeout + connection-pool tuning; dedup the retry loop

Reliability + correctness:
- char-boundary-safe truncation of LLM error bodies (shared helper)
- HTTP connect/read timeouts on all LLM provider clients
- isolate pdf-extract behind catch_unwind + spawn_blocking
- OSS server: crawl inherits the shared fetch profile; ProviderChain built
  once in AppState; request TimeoutLayer

API / safety / docs:
- #[non_exhaustive] on public enums + result structs (+ builders)
- #![forbid(unsafe_code)] on pure crates, deny on llm
- //! crate docs + doctests; scrub bypass/vendor/target specifics from
  public crate docs and comments

Tooling: [profile.release] lto/codegen-units/strip, MSRV pin, deny.toml +
cargo-deny CI, macOS test matrix. CLI main.rs split into focused modules.

crates/webclaw-server/Cargo.toml

@@ -2,10 +2,14 @@
 name = "webclaw-server"
 version.workspace = true
 edition.workspace = true
+rust-version.workspace = true
 license.workspace = true
 repository.workspace = true
 description = "Minimal REST API server for self-hosting webclaw extraction. Wraps the OSS extraction crates with HTTP endpoints. NOT the production hosted API at api.webclaw.io — this is a stateless, single-binary reference server for local + self-hosted deployments."
 
+[lints]
+workspace = true
+
 [[bin]]
 name = "webclaw-server"
 path = "src/main.rs"
@@ -18,7 +22,7 @@ webclaw-pdf    = { workspace = true }
 
 axum           = { version = "0.8", features = ["macros"] }
 tokio          = { workspace = true }
-tower-http     = { version = "0.6", features = ["trace", "cors"] }
+tower-http     = { version = "0.6", features = ["trace", "cors", "timeout"] }
 clap           = { workspace = true, features = ["derive", "env"] }
 serde          = { workspace = true }
 serde_json     = { workspace = true }
@@ -27,3 +31,9 @@ tracing-subscriber = { workspace = true, features = ["env-filter"] }
 anyhow         = "1"
 thiserror      = { workspace = true }
 subtle         = "2.6"
+
+[dev-dependencies]
+# `ServiceExt::oneshot` drives the router in-process for hermetic handler
+# tests (no TCP listener, no network).
+tower          = { version = "0.5", features = ["util"] }
+http-body-util = "0.1"

crates/webclaw-server/src/main.rs

@@ -26,12 +26,20 @@ use axum::{
 };
 use clap::Parser;
 use tower_http::cors::{Any, CorsLayer};
+use tower_http::timeout::TimeoutLayer;
 use tower_http::trace::TraceLayer;
 use tracing::info;
 use tracing_subscriber::{EnvFilter, fmt};
 
 use crate::state::AppState;
 
+/// Hard ceiling on how long any single request may run before the server
+/// returns `408 Request Timeout` and drops the work. Generous enough for a
+/// cold scrape + LLM round-trip, but bounds the inline `/v1/crawl` handler
+/// (up to 500 pages, no job queue) so a slow crawl can't pin a connection
+/// and a worker indefinitely.
+const REQUEST_TIMEOUT: Duration = Duration::from_secs(120);
+
 #[derive(Parser, Debug)]
 #[command(
     name = "webclaw-server",
@@ -84,8 +92,29 @@ async fn main() -> anyhow::Result<()> {
         );
     }
 
-    let state = AppState::new(args.api_key.clone())?;
+    let state = AppState::new(args.api_key.clone()).await?;
 
+    let app = build_app(state);
+
+    let addr = SocketAddr::from((args.host, args.port));
+    let listener = tokio::net::TcpListener::bind(addr).await?;
+    let auth_status = if args.api_key.is_some() {
+        "bearer auth required"
+    } else {
+        "open mode (no auth)"
+    };
+    info!(%addr, mode = auth_status, "webclaw-server listening");
+
+    axum::serve(listener, app).await?;
+    Ok(())
+}
+
+/// Build the fully-layered axum router for a given [`AppState`].
+///
+/// Split out from `main` so the handler tests can exercise the exact same
+/// routing + middleware stack (auth, timeout) in-process via
+/// `tower::ServiceExt::oneshot`, with no TCP listener.
+fn build_app(state: AppState) -> Router {
     let v1 = Router::new()
         .route("/scrape", post(routes::scrape::scrape))
         .route(
@@ -102,7 +131,7 @@ async fn main() -> anyhow::Result<()> {
         .route("/brand", post(routes::brand::brand))
         .layer(from_fn_with_state(state.clone(), auth::require_bearer));
 
-    let app = Router::new()
+    Router::new()
         .route("/health", get(routes::health::health))
         .nest("/v1", v1)
         .layer(
@@ -115,20 +144,14 @@ async fn main() -> anyhow::Result<()> {
                 .allow_headers(Any)
                 .max_age(Duration::from_secs(3600)),
         )
+        // Caps total request time; returns 408 if exceeded. Applied
+        // outermost so it covers every route, including the inline crawl.
+        .layer(TimeoutLayer::with_status_code(
+            axum::http::StatusCode::REQUEST_TIMEOUT,
+            REQUEST_TIMEOUT,
+        ))
         .layer(TraceLayer::new_for_http())
-        .with_state(state);
-
-    let addr = SocketAddr::from((args.host, args.port));
-    let listener = tokio::net::TcpListener::bind(addr).await?;
-    let auth_status = if args.api_key.is_some() {
-        "bearer auth required"
-    } else {
-        "open mode (no auth)"
-    };
-    info!(%addr, mode = auth_status, "webclaw-server listening");
-
-    axum::serve(listener, app).await?;
-    Ok(())
+        .with_state(state)
 }
 
 fn is_unspecified_addr(addr: IpAddr) -> bool {
@@ -137,3 +160,133 @@ fn is_unspecified_addr(addr: IpAddr) -> bool {
         IpAddr::V6(ip) => ip.is_unspecified(),
     }
 }
+
+#[cfg(test)]
+mod tests {
+    //! Hermetic handler tests. Each builds the real router via
+    //! [`build_app`] and drives it in-process with
+    //! [`tower::ServiceExt::oneshot`] — no TCP listener, no outbound
+    //! network. Endpoints that would fetch a URL are reached only on paths
+    //! that short-circuit before any network call (auth rejection, format
+    //! validation, the static `/v1/extractors` catalog, `/health`).
+
+    use super::*;
+    use axum::body::Body;
+    use axum::http::{Request, StatusCode};
+    use http_body_util::BodyExt;
+    use tower::ServiceExt;
+
+    const TEST_KEY: &str = "test-secret-key";
+
+    async fn app_with_key(key: Option<&str>) -> Router {
+        // `AppState::new` probes Ollama once at startup. With no Ollama
+        // running the probe returns fast (connection refused) and the
+        // tests below never touch the chain, so they stay hermetic either
+        // way — no env juggling required.
+        let state = AppState::new(key.map(str::to_owned))
+            .await
+            .expect("build state");
+        build_app(state)
+    }
+
+    fn get(uri: &str) -> Request<Body> {
+        Request::builder()
+            .uri(uri)
+            .body(Body::empty())
+            .expect("request")
+    }
+
+    fn get_auth(uri: &str, header: &str) -> Request<Body> {
+        Request::builder()
+            .uri(uri)
+            .header("authorization", header)
+            .body(Body::empty())
+            .expect("request")
+    }
+
+    async fn json_body(resp: axum::response::Response) -> serde_json::Value {
+        let bytes = resp.into_body().collect().await.expect("body").to_bytes();
+        serde_json::from_slice(&bytes).expect("json")
+    }
+
+    #[tokio::test]
+    async fn health_returns_version() {
+        let app = app_with_key(None).await;
+        let resp = app.oneshot(get("/health")).await.expect("response");
+        assert_eq!(resp.status(), StatusCode::OK);
+        let body = json_body(resp).await;
+        assert_eq!(body["status"], "ok");
+        assert_eq!(body["service"], "webclaw-server");
+        assert_eq!(body["version"], env!("CARGO_PKG_VERSION"));
+    }
+
+    #[tokio::test]
+    async fn missing_key_is_unauthorized() {
+        let app = app_with_key(Some(TEST_KEY)).await;
+        let resp = app.oneshot(get("/v1/extractors")).await.expect("response");
+        assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
+    }
+
+    #[tokio::test]
+    async fn wrong_key_is_unauthorized() {
+        let app = app_with_key(Some(TEST_KEY)).await;
+        let resp = app
+            .oneshot(get_auth("/v1/extractors", "Bearer wrong-key"))
+            .await
+            .expect("response");
+        assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
+    }
+
+    #[tokio::test]
+    async fn correct_key_authorized() {
+        let app = app_with_key(Some(TEST_KEY)).await;
+        // `/v1/extractors` is a static catalog — passes auth, no network.
+        let resp = app
+            .oneshot(get_auth("/v1/extractors", &format!("Bearer {TEST_KEY}")))
+            .await
+            .expect("response");
+        assert_eq!(resp.status(), StatusCode::OK);
+    }
+
+    #[tokio::test]
+    async fn lowercase_bearer_accepted() {
+        let app = app_with_key(Some(TEST_KEY)).await;
+        let resp = app
+            .oneshot(get_auth("/v1/extractors", &format!("bearer {TEST_KEY}")))
+            .await
+            .expect("response");
+        assert_eq!(resp.status(), StatusCode::OK);
+    }
+
+    #[tokio::test]
+    async fn open_mode_allows_unauthenticated() {
+        // No api key configured => auth middleware passes everything.
+        let app = app_with_key(None).await;
+        let resp = app.oneshot(get("/v1/extractors")).await.expect("response");
+        assert_eq!(resp.status(), StatusCode::OK);
+    }
+
+    #[tokio::test]
+    async fn unknown_format_is_bad_request() {
+        // Format validation now runs before the fetch, so a bogus format
+        // returns 400 without any network call.
+        let app = app_with_key(None).await;
+        let req = Request::builder()
+            .method("POST")
+            .uri("/v1/scrape")
+            .header("content-type", "application/json")
+            .body(Body::from(
+                r#"{"url":"https://example.com","formats":["bogus"]}"#,
+            ))
+            .expect("request");
+        let resp = app.oneshot(req).await.expect("response");
+        assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
+        let body = json_body(resp).await;
+        assert!(
+            body["error"]
+                .as_str()
+                .is_some_and(|e| e.contains("unknown format")),
+            "expected unknown-format error, got {body:?}"
+        );
+    }
+}

crates/webclaw-server/src/routes/crawl.rs

@@ -9,7 +9,7 @@ use axum::{Json, extract::State};
 use serde::Deserialize;
 use serde_json::{Value, json};
 use std::time::Duration;
-use webclaw_fetch::{CrawlConfig, Crawler, FetchConfig};
+use webclaw_fetch::{CrawlConfig, Crawler};
 
 use crate::{error::ApiError, state::AppState};
 
@@ -30,7 +30,7 @@ pub struct CrawlRequest {
 }
 
 pub async fn crawl(
-    State(_state): State<AppState>,
+    State(state): State<AppState>,
     Json(req): Json<CrawlRequest>,
 ) -> Result<Json<Value>, ApiError> {
     if req.url.trim().is_empty() {
@@ -42,7 +42,10 @@ pub async fn crawl(
     let concurrency = req.concurrency.unwrap_or(5).min(20);
 
     let config = CrawlConfig {
-        fetch: FetchConfig::default(),
+        // Inherit the shared client's profile/proxy/timeout instead of
+        // `FetchConfig::default()` (which is Chrome). The rest of the
+        // server fetches as Firefox; crawl now matches.
+        fetch: state.fetch_config().clone(),
         max_depth,
         max_pages,
         concurrency,

crates/webclaw-server/src/routes/diff.rs

@@ -36,36 +36,16 @@ impl PreviousSnapshot {
     fn into_extraction(self) -> ExtractionResult {
         match self {
             Self::Full(r) => r,
-            Self::Minimal { markdown, metadata } => ExtractionResult {
-                metadata: metadata.unwrap_or_else(empty_metadata),
-                content: Content {
-                    markdown,
-                    plain_text: String::new(),
-                    links: Vec::new(),
-                    images: Vec::new(),
-                    code_blocks: Vec::new(),
-                    raw_html: None,
-                },
-                domain_data: None,
-                structured_data: Vec::new(),
-            },
+            Self::Minimal { markdown, metadata } => ExtractionResult::new(
+                metadata.unwrap_or_else(empty_metadata),
+                Content::default().with_markdown(markdown),
+            ),
         }
     }
 }
 
 fn empty_metadata() -> Metadata {
-    Metadata {
-        title: None,
-        description: None,
-        author: None,
-        published_date: None,
-        language: None,
-        url: None,
-        site_name: None,
-        image: None,
-        favicon: None,
-        word_count: 0,
-    }
+    Metadata::default()
 }
 
 pub async fn diff_route(

crates/webclaw-server/src/routes/extract.rs

@@ -4,14 +4,14 @@
 //! * `schema` — JSON Schema describing what to extract.
 //! * `prompt` — natural-language instructions.
 //!
-//! At least one must be provided. The provider chain is built per
-//! request from env (Ollama -> OpenAI -> Anthropic). Self-hosters
-//! get the same fallback behaviour as the CLI.
+//! At least one must be provided. The provider chain (Ollama -> OpenAI
+//! -> Anthropic) is built once at startup and shared via `AppState`.
+//! Self-hosters get the same fallback behaviour as the CLI.
 
 use axum::{Json, extract::State};
 use serde::Deserialize;
 use serde_json::{Value, json};
-use webclaw_llm::{ProviderChain, extract::extract_json, extract::extract_with_prompt};
+use webclaw_llm::{extract::extract_json, extract::extract_with_prompt};
 
 use crate::{error::ApiError, state::AppState};
 
@@ -59,7 +59,7 @@ pub async fn extract(
         ));
     }
 
-    let chain = ProviderChain::default().await;
+    let chain = state.llm_chain();
     if chain.is_empty() {
         return Err(ApiError::Llm(
             "no LLM providers configured (set OLLAMA_HOST, OPENAI_API_KEY, or ANTHROPIC_API_KEY)"
@@ -69,10 +69,10 @@ pub async fn extract(
 
     let model = req.model.as_deref();
     let data = if let Some(schema) = req.schema.as_ref() {
-        extract_json(&content, schema, &chain, model).await?
+        extract_json(&content, schema, chain, model).await?
     } else {
         let prompt = req.prompt.as_deref().unwrap_or_default();
-        extract_with_prompt(&content, prompt, &chain, model).await?
+        extract_with_prompt(&content, prompt, chain, model).await?
     };
 
     Ok(Json(json!({

crates/webclaw-server/src/routes/scrape.rs

@@ -52,8 +52,18 @@ pub async fn scrape(
     if req.url.trim().is_empty() {
         return Err(ApiError::bad_request("`url` is required"));
     }
-    let url = webclaw_fetch::url_security::validate_public_http_url(&req.url).await?;
     let formats = req.formats.as_vec();
+    // Validate requested formats up front so a typo fails fast with a 400
+    // instead of after a full (wasted) fetch + extract.
+    if let Some(bad) = formats
+        .iter()
+        .find(|f| !matches!(f.as_str(), "markdown" | "text" | "llm" | "html" | "json"))
+    {
+        return Err(ApiError::bad_request(format!(
+            "unknown format: '{bad}' (allowed: markdown, text, llm, html, json)"
+        )));
+    }
+    let url = webclaw_fetch::url_security::validate_public_http_url(&req.url).await?;
 
     let options = ExtractionOptions {
         include_selectors: req.include_selectors,

crates/webclaw-server/src/routes/summarize.rs

@@ -3,7 +3,7 @@
 use axum::{Json, extract::State};
 use serde::Deserialize;
 use serde_json::{Value, json};
-use webclaw_llm::{ProviderChain, summarize::summarize};
+use webclaw_llm::summarize::summarize;
 
 use crate::{error::ApiError, state::AppState};
 
@@ -36,7 +36,7 @@ pub async fn summarize_route(
         ));
     }
 
-    let chain = ProviderChain::default().await;
+    let chain = state.llm_chain();
     if chain.is_empty() {
         return Err(ApiError::Llm(
             "no LLM providers configured (set OLLAMA_HOST, OPENAI_API_KEY, or ANTHROPIC_API_KEY)"
@@ -44,7 +44,7 @@ pub async fn summarize_route(
         ));
     }
 
-    let summary = summarize(&content, req.max_sentences, &chain, req.model.as_deref()).await?;
+    let summary = summarize(&content, req.max_sentences, chain, req.model.as_deref()).await?;
 
     Ok(Json(json!({
         "url": req.url,

crates/webclaw-server/src/state.rs

@@ -20,6 +20,7 @@ use std::sync::Arc;
 use tracing::info;
 use webclaw_fetch::cloud::CloudClient;
 use webclaw_fetch::{BrowserProfile, FetchClient, FetchConfig};
+use webclaw_llm::ProviderChain;
 
 /// Single-process state shared across all request handlers.
 #[derive(Clone)]
@@ -34,6 +35,16 @@ struct Inner {
     /// auto-deref `&Arc<FetchClient>` -> `&FetchClient`, so this costs
     /// them nothing.
     pub fetch: Arc<FetchClient>,
+    /// The exact [`FetchConfig`] the shared `fetch` client was built from.
+    /// Endpoints that spin up their own client (e.g. `/v1/crawl`, which
+    /// builds a `Crawler` with its own internal `FetchClient`) clone this
+    /// so they inherit the same browser profile / proxy / timeout instead
+    /// of silently falling back to `FetchConfig::default()` (Chrome).
+    pub fetch_config: FetchConfig,
+    /// LLM provider chain (Ollama -> OpenAI -> Anthropic), built once at
+    /// startup. `/v1/extract` and `/v1/summarize` borrow this instead of
+    /// rebuilding the chain (and re-probing Ollama) on every request.
+    pub llm_chain: Arc<ProviderChain>,
     /// Inbound bearer-auth token for this server's own `/v1/*` surface.
     pub api_key: Option<String>,
 }
@@ -45,12 +56,15 @@ impl AppState {
     ///
     /// `inbound_api_key` is the bearer token clients must present;
     /// cloud-fallback credentials come from the env (checked here).
-    pub fn new(inbound_api_key: Option<String>) -> anyhow::Result<Self> {
+    ///
+    /// Async because the LLM provider chain probes Ollama for availability
+    /// once at startup; doing it here keeps it off the per-request hot path.
+    pub async fn new(inbound_api_key: Option<String>) -> anyhow::Result<Self> {
         let config = FetchConfig {
             browser: BrowserProfile::Firefox,
             ..FetchConfig::default()
         };
-        let mut fetch = FetchClient::new(config)
+        let mut fetch = FetchClient::new(config.clone())
             .map_err(|e| anyhow::anyhow!("failed to build fetch client: {e}"))?;
 
         // Cloud fallback: only activates when the operator has provided
@@ -66,9 +80,13 @@ impl AppState {
             fetch = fetch.with_cloud(cloud);
         }
 
+        let llm_chain = Arc::new(ProviderChain::default().await);
+
         Ok(Self {
             inner: Arc::new(Inner {
                 fetch: Arc::new(fetch),
+                fetch_config: config,
+                llm_chain,
                 api_key: inbound_api_key,
             }),
         })
@@ -78,6 +96,19 @@ impl AppState {
         &self.inner.fetch
     }
 
+    /// The [`FetchConfig`] the shared client was built from. Cloned by
+    /// endpoints that need to construct their own client with identical
+    /// settings (currently `/v1/crawl`).
+    pub fn fetch_config(&self) -> &FetchConfig {
+        &self.inner.fetch_config
+    }
+
+    /// The shared LLM provider chain. Borrowed by `/v1/extract` and
+    /// `/v1/summarize`; `&ProviderChain` coerces to `&dyn LlmProvider`.
+    pub fn llm_chain(&self) -> &ProviderChain {
+        &self.inner.llm_chain
+    }
+
     pub fn api_key(&self) -> Option<&str> {
         self.inner.api_key.as_deref()
     }