Merge pull request #95 from 0xMassi/docs/trendshift-badge

docs(readme): add Trendshift trending badge

.github/FUNDING.yml

@@ -10,4 +10,4 @@ lfx_crowdfunding:
 polar:
 buy_me_a_coffee:
 thanks_dev:
-custom:
+custom: ["https://webclaw.io/sponsor"]

.github/workflows/release.yml

@@ -3,6 +3,15 @@ name: Release
 on:
   push:
     tags: ["v*"]
+  # Manual re-publish of the Docker image for an existing release, without
+  # rebuilding binaries or cutting a new version. Runs only the docker (+
+  # homebrew) jobs against the given tag's already-published release assets.
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Existing release tag to (re)build + push the Docker image for, e.g. v0.6.9"
+        required: true
+        type: string
 
 permissions:
   contents: read
@@ -12,6 +21,9 @@ env:
 
 jobs:
   build:
+    # Binaries are only built when a tag is pushed. A manual dispatch reuses
+    # the existing release's binaries, so it skips this job entirely.
+    if: github.event_name == 'push'
     permissions:
       contents: read
     name: Build ${{ matrix.target }}
@@ -24,9 +36,23 @@ jobs:
             os: macos-latest
           - target: aarch64-apple-darwin
             os: macos-latest
+          # Linux targets pin to ubuntu-22.04 (glibc 2.35), NOT ubuntu-latest
+          # (24.04 = glibc 2.39): glibc is forward- but not backward-compatible,
+          # so binaries built on 24.04 require GLIBC_2.38 and won't start on
+          # older LTS distros (Debian 12, Ubuntu 22.04). The aarch64 cross
+          # toolchain tracks the runner's distro too, so this lowers both
+          # targets. See #73. Don't bump to ubuntu-latest.
           - target: x86_64-unknown-linux-gnu
-            os: ubuntu-latest
+            os: ubuntu-22.04
           - target: aarch64-unknown-linux-gnu
+            os: ubuntu-22.04
+          # musl static builds: glibc-independent, run on ANY Linux (Alpine,
+          # Amazon Linux 2023, RHEL 9, old distros) where the gnu builds can't.
+          # cargo-zigbuild links statically, so the host glibc is irrelevant —
+          # ubuntu-latest is fine. See #73.
+          - target: x86_64-unknown-linux-musl
+            os: ubuntu-latest
+          - target: aarch64-unknown-linux-musl
             os: ubuntu-latest
           - target: x86_64-pc-windows-msvc
             os: windows-latest
@@ -66,8 +92,30 @@ jobs:
           choco install nasm -y
           echo "C:\Program Files\NASM" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
 
+      # musl static builds use zig as the C/C++ cross-compiler for BoringSSL,
+      # driven by cargo-zigbuild. Build scripts (bindgen) still run as the glibc
+      # host so libclang loads fine; the linked output is fully static and runs
+      # on any Linux regardless of glibc. (A native Alpine build can't do this —
+      # its static build scripts can't dlopen libclang.)
+      - name: Setup zig + cargo-zigbuild (musl)
+        if: contains(matrix.target, 'musl')
+        run: |
+          python3 -m pip install --user --break-system-packages ziglang
+          mkdir -p "$HOME/.local/bin"
+          printf '#!/bin/sh\nexec python3 -m ziglang "$@"\n' > "$HOME/.local/bin/zig"
+          chmod +x "$HOME/.local/bin/zig"
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+          "$HOME/.local/bin/zig" version
+          cargo install cargo-zigbuild --locked
+
       - name: Build
-        run: cargo build --release --target ${{ matrix.target }}
+        shell: bash
+        run: |
+          if [[ "${{ matrix.target }}" == *-musl ]]; then
+            cargo zigbuild --release --target ${{ matrix.target }}
+          else
+            cargo build --release --target ${{ matrix.target }}
+          fi
 
       - name: Package
         shell: bash
@@ -105,6 +153,7 @@ jobs:
 
   release:
     name: Release
+    if: github.event_name == 'push'
     needs: build
     runs-on: ubuntu-latest
     permissions:
@@ -127,16 +176,59 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           tag="${GITHUB_REF#refs/tags/}"
+          # SemVer prerelease tags (e.g. v1.2.3-rc1) contain a hyphen — mark
+          # them as prereleases so they never surface as the stable release.
+          prerelease=""
+          case "$tag" in *-*) prerelease="--prerelease" ;; esac
           gh release create "$tag" \
             artifacts/*.tar.gz \
             artifacts/*.zip \
             artifacts/SHA256SUMS \
             --repo "$GITHUB_REPOSITORY" \
+            $prerelease \
             --generate-notes
 
+  npm:
+    name: npm (@webclaw/mcp)
+    needs: release
+    # Publish the zero-install MCP launcher on STABLE tags only (prerelease
+    # tags contain a hyphen and are skipped). Requires the NPM_TOKEN repo
+    # secret: an npm automation token with publish rights on the @webclaw scope.
+    if: ${{ github.event_name == 'push' && !contains(github.ref_name, '-') }}
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v5
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          registry-url: https://registry.npmjs.org
+      - name: Pin launcher to this release and publish
+        working-directory: packages/webclaw-mcp
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+        run: |
+          tag="${GITHUB_REF#refs/tags/}"   # e.g. v0.6.17 (core release)
+          # Pin which prebuilt binary this launcher downloads to the just-released tag.
+          sed -i -E "/const RELEASE_TAG =/ s|\"v[0-9.]+\"|\"${tag}\"|" webclaw-mcp.mjs
+          # The launcher is versioned INDEPENDENTLY of the core release (a
+          # launcher-only fix must be shippable without a core bump). Publish the
+          # next patch above whatever is currently on npm — no commit-back needed.
+          cur="$(npm view @webclaw/mcp version 2>/dev/null || echo 0.0.0)"
+          npm version "$cur" --no-git-tag-version --allow-same-version >/dev/null
+          npm version patch --no-git-tag-version >/dev/null
+          echo "publishing @webclaw/mcp@$(node -p "require('./package.json').version") pinned to ${tag}:"
+          grep -n "RELEASE_TAG =" webclaw-mcp.mjs
+          npm publish --access public
+
   docker:
     name: Docker
     needs: release
+    # Runs after a successful release on tag push, or standalone via
+    # workflow_dispatch to (re)publish an existing tag's image. `always()` lets
+    # it run even though `release` is skipped on a manual dispatch.
+    if: ${{ always() && (github.event_name == 'workflow_dispatch' || needs.release.result == 'success') }}
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -156,49 +248,55 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      # Download pre-built binaries for both architectures
+      # The pushed tag, or the workflow_dispatch input for a manual re-publish.
+      - name: Resolve tag
+        id: tag
+        run: echo "tag=${{ github.event.inputs.tag || github.ref_name }}" >> "$GITHUB_OUTPUT"
+
+      # Download pre-built binaries into TARGETARCH-named dirs (amd64/arm64) so
+      # a single multi-platform build picks the matching binary per platform.
       - name: Download release binaries
         run: |
-          tag="${GITHUB_REF#refs/tags/}"
+          tag="${{ steps.tag.outputs.tag }}"
+          declare -A arch=( [x86_64-unknown-linux-gnu]=amd64 [aarch64-unknown-linux-gnu]=arm64 )
           for target in x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu; do
             dir="webclaw-${tag}-${target}"
             curl -sSL "https://github.com/0xMassi/webclaw/releases/download/${tag}/${dir}.tar.gz" -o "${target}.tar.gz"
             tar xzf "${target}.tar.gz"
-            mkdir -p "binaries-${target}"
-            cp "${dir}/webclaw" "binaries-${target}/webclaw"
-            cp "${dir}/webclaw-mcp" "binaries-${target}/webclaw-mcp"
-            cp "${dir}/webclaw-server" "binaries-${target}/webclaw-server"
-            chmod +x "binaries-${target}"/*
+            a="${arch[$target]}"
+            mkdir -p "binaries-${a}"
+            cp "${dir}/webclaw" "${dir}/webclaw-mcp" "${dir}/webclaw-server" "binaries-${a}/"
+            chmod +x "binaries-${a}"/*
           done
           ls -laR binaries-*/
 
-      # Build per-arch images with plain docker build (no buildx manifest nesting)
+      # One atomic multi-platform build + push. buildx assembles a single
+      # manifest list and pushes it in one shot, so there is no separate
+      # `imagetools create` step to race GHCR's read-after-write (that is what
+      # failed before: "v0.6.9-arm64: not found"). Provenance/SBOM attestations
+      # are disabled so each platform entry stays a plain image manifest.
       - name: Build and push
         run: |
-          tag="${GITHUB_REF#refs/tags/}"
-
-          # amd64
-          docker build -f Dockerfile.ci --build-arg BINARY_DIR=binaries-x86_64-unknown-linux-gnu \
-            --platform linux/amd64 -t ghcr.io/0xmassi/webclaw:${tag}-amd64 --push .
-
-          # arm64
-          docker build -f Dockerfile.ci --build-arg BINARY_DIR=binaries-aarch64-unknown-linux-gnu \
-            --platform linux/arm64 -t ghcr.io/0xmassi/webclaw:${tag}-arm64 --push .
-
-          # Multi-arch manifest
-          docker manifest create ghcr.io/0xmassi/webclaw:${tag} \
-            ghcr.io/0xmassi/webclaw:${tag}-amd64 \
-            ghcr.io/0xmassi/webclaw:${tag}-arm64
-          docker manifest push ghcr.io/0xmassi/webclaw:${tag}
-
-          docker manifest create ghcr.io/0xmassi/webclaw:latest \
-            ghcr.io/0xmassi/webclaw:${tag}-amd64 \
-            ghcr.io/0xmassi/webclaw:${tag}-arm64
-          docker manifest push ghcr.io/0xmassi/webclaw:latest
+          tag="${{ steps.tag.outputs.tag }}"
+          # Only move :latest for stable tags. Prereleases (hyphenated, e.g.
+          # v1.2.3-rc1) still publish :${tag} for testing but must not clobber
+          # :latest.
+          latest=""
+          case "$tag" in *-*) ;; *) latest="-t ghcr.io/0xmassi/webclaw:latest" ;; esac
+          docker buildx build -f Dockerfile.ci \
+            --platform linux/amd64,linux/arm64 \
+            --provenance=false --sbom=false \
+            -t "ghcr.io/0xmassi/webclaw:${tag}" \
+            $latest \
+            --push .
 
   homebrew:
     name: Update Homebrew
     needs: [release, docker]
+    # Runs once Docker succeeds, on both tag push and manual re-publish.
+    # Skipped for prereleases (hyphenated tags like v1.2.3-rc1) so the formula
+    # keeps pointing at the latest stable, never an rc.
+    if: ${{ always() && needs.docker.result == 'success' && !contains(github.event.inputs.tag || github.ref_name, '-') }}
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -207,7 +305,7 @@ jobs:
         env:
           COMMITTER_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }}
         run: |
-          tag="${GITHUB_REF#refs/tags/}"
+          tag="${{ github.event.inputs.tag || github.ref_name }}"
           base="https://github.com/0xMassi/webclaw/releases/download/${tag}"
 
           # Download all tarballs (Linux + macOS) and compute SHAs

.gitignore

@@ -13,3 +13,6 @@ local-test-results.json
 # not code and shouldn't live in git. Track deliberately-saved research
 # output under a different name.
 research-*.json
+# MCP registry publisher credentials (mcp-publisher writes these into
+# packages/create-webclaw/). They are OAuth/API tokens — never commit them.
+.mcpregistry_*_token

.mcp.json

@@ -0,0 +1,8 @@
+{
+  "mcpServers": {
+    "webclaw": {
+      "command": "npx",
+      "args": ["-y", "@webclaw/mcp"]
+    }
+  }
+}

CHANGELOG.md

@@ -3,6 +3,89 @@
 All notable changes to webclaw are documented here.
 Format follows [Keep a Changelog](https://keepachangelog.com/).
 
+## [0.6.16] - 2026-07-22
+
+### Added
+- **`@webclaw/mcp` — zero-install MCP launcher.** `npx -y @webclaw/mcp` downloads the prebuilt `webclaw-mcp` binary once, verifies it against the release `SHA256SUMS`, caches it, and runs it as an MCP stdio server. This is now the canonical way to add webclaw to an MCP client (`{"command": "npx", "args": ["-y", "@webclaw/mcp"]}`) — no Rust build — and it makes the server introspectable in MCP registries (Glama, Smithery, the MCP registry). `create-webclaw` stays as the auto-detect installer.
+- **Atlas Cloud provider in the LLM chain (opt-in).** Set `ATLASCLOUD_API_KEY` to add [Atlas Cloud](https://atlascloud.ai) as a provider for the LLM features (extraction, summarization). It's added at the end of the chain — Ollama → OpenAI → Gemini → Anthropic → Atlas Cloud — so it only runs when you configure it and never preempts an already-configured provider. Override the model with `ATLASCLOUD_MODEL` (default `qwen/qwen3.5-flash`) and the endpoint with `ATLASCLOUD_BASE_URL`.
+
+### Fixed
+- **`--urls-file` with a single URL now works.** A file containing exactly one URL (with no positional URL on the command line) was rejected with *"no input provided"*: it fell between the batch path, which skips single-entry files, and the single-URL path, which only reads positional arguments. A lone file URL is now treated exactly like `webclaw <url>`, so it works with stdout, `--output-dir`, `--brand`, `--diff-with`, `--crawl`, `--watch`, and the LLM paths.
+
+## [0.6.15] - 2026-07-18
+
+### Added
+- **`lead` and `lead_batch` MCP tools.** Turn a company URL into an outreach-ready lead — founders and leadership with their LinkedIn and X, plus company summary, socials, tech stack, pricing, and public emails — straight from any MCP client (Claude Desktop, Cursor, Claude Code, …). `lead` enriches one company; `lead_batch` enriches up to 25 at once and blocks until the batch finishes. Both require `WEBCLAW_API_KEY`.
+
+### Fixed
+- **`webclaw --help` no longer prints secret environment values.** Secret env vars are masked in help output instead of being shown in the clear.
+
+## [0.6.14] - 2026-06-27
+
+A distribution release: the extraction engine is unchanged from 0.6.13, but the
+Linux binaries now run on far more systems.
+
+### Added
+- **Static musl Linux binaries** (`x86_64-unknown-linux-musl`, `aarch64-unknown-linux-musl`) that run on any Linux regardless of glibc — Alpine, Amazon Linux 2023, RHEL 9, and older distros. Built with cargo-zigbuild and shipped alongside the existing gnu binaries.
+
+### Changed
+- **gnu Linux binaries now run on older glibc.** They are built on Ubuntu 22.04 (glibc 2.35) instead of the latest runner (glibc 2.39), so they start on Debian 12, Ubuntu 22.04, and other LTS distros that previously failed with `version 'GLIBC_2.38' not found`.
+
+### Fixed
+- **`create-webclaw` install (`create-webclaw@0.1.5`)** failed on every platform: it looked up the wrong release-asset name and used `unzip` (absent on Windows), silently falling back to `cargo install`. It now downloads the correct versioned asset, extracts via PowerShell on Windows, lifts the binary out of the archive's versioned subdirectory, and surfaces GitHub API rate-limit errors instead of hiding them.
+- **Self-hosting setup scripts** wrote `WEBCLAW_AUTH_KEY` into the generated `.env`, but the server reads `WEBCLAW_API_KEY` — on a public bind this made the server refuse to start. `setup.sh` and `deploy/hetzner.sh` now write the correct variable.
+
+## [0.6.13] - 2026-06-17
+
+### Performance
+- Faster content extraction with byte-identical output. The markdown noise filter no longer recompiles its CSS selectors on every element; the vertical extractors share a single Open Graph meta parse instead of re-scanning the page per field; the JavaScript sandbox is skipped entirely when a page has no JS-assigned data (and reuses the already-parsed document instead of re-parsing); and the HTTP client now tunes its connection pool (connect timeout, idle-pool reuse, keep-alive) for better connection reuse across requests.
+
+## [0.6.12] - 2026-06-17
+
+### Added
+- **Standalone web search** using your own [Serper.dev](https://serper.dev) key — no hosted webclaw account needed. Available across the CLI (`webclaw search "query" --num 5 --scrape`, key via `--serper-key` or `SERPER_API_KEY`), the MCP `search` tool (local-first when `SERPER_API_KEY` is set, hosted API otherwise), and the self-hosted REST server (`POST /v1/search`, enabled when started with `SERPER_API_KEY`). With `--scrape`, the top result pages are fetched and extracted to markdown.
+- **Layered URL discovery for `--map`**: when a site has no sitemap or only a thin one, map now falls back to a bounded same-origin crawl and harvests links from every fetched page plus the unfetched frontier, returning far more URLs. Adds gzipped-sitemap (`.xml.gz`) support, deeper sitemap-index recursion, more fallback paths, and `--map-pages` / `--no-map-crawl` / `--map-limit` controls. Crawler logs now go to stderr so `--map --format json` stays machine-parseable.
+
+### Fixed
+- MCP tools now accept boolean arguments whether the client sends them as JSON booleans or as the strings `"true"`/`"false"` (case-insensitive). Some MCP clients (e.g. Claude Desktop) send booleans as strings, which previously failed the call with a deserialization error. Affects `scrape` (only_main_content), `crawl` (use_sitemap), `research` (deep), and `search` (scrape). This completes the earlier numeric-parameter fix.
+
+## [0.6.11] - 2026-06-16
+
+### Added
+- New **Google Gemini** provider in the LLM provider chain. Set `GEMINI_API_KEY` (and optionally `GEMINI_MODEL`, default `gemini-2.5-flash`) to enable it; the chain tries Ollama → OpenAI → Gemini → Anthropic and uses the first available provider.
+
+### Fixed
+- The Anthropic provider's default model pointed at a retired model id that now returns `404`, which could fail extraction/summarization when falling back to Anthropic. It now defaults to a current model and is overridable via `ANTHROPIC_MODEL`.
+
+## [0.6.10] - 2026-06-15
+
+### Fixed
+- MCP tools that take numeric arguments now accept those values whether the client sends them as numbers or as numeric strings. Some MCP clients (e.g. Claude Desktop) send `"5"` instead of `5`, which previously failed the call with a deserialization error. Affects `crawl` (depth, max_pages, concurrency), `batch` (concurrency), `search` (num_results), and `summarize` (max_sentences).
+
+## [0.6.9] - 2026-06-10
+
+### Fixed
+- The multi-arch Docker image (linux/amd64 + linux/arm64) now publishes reliably on each release. The build moved to Buildx so registry pushes no longer fail intermittently, and the Homebrew formula update that depends on it is no longer skipped.
+
+## [0.6.8] - 2026-06-10
+
+### Fixed
+- Pages with multibyte text (accented or CJK characters) no longer panic or get mangled during extraction. API-endpoint discovery now cuts oversized scripts on a character boundary instead of crashing mid-character, and structured-data parsing preserves non-ASCII string values instead of turning them into mojibake.
+- LLM error messages from a provider no longer panic when the error body contains multibyte characters near the truncation point.
+- LLM provider requests now have explicit connect and overall timeouts, so a stalled or unreachable provider fails fast instead of hanging.
+- Batch extraction in the MCP server no longer aborts the whole batch when a single URL fails to resolve; bad URLs are reported as individual per-URL errors and the rest still run.
+- CLI crawl and batch runs now wait for the completion webhook to actually send before exiting, replacing a fixed delay that could cut the request off or waste time.
+- Homepage warm-up requests now include the port for hosts on a non-default port, so those sites are warmed correctly.
+
+---
+
+## [0.6.7] — 2026-06-09
+
+### Changed
+- Updated the HTTP/TLS engine (wreq 6.0.0-rc.29, wreq-util 3.0.0-rc.12). This pulls in upstream robustness fixes: no more panic on responses with non-UTF8 header values, a fix for short reads when decoding large compressed bodies, and the TCP nodelay setting is restored. Browser TLS fingerprints are unchanged.
+
+---
+
 ## [0.6.6] — 2026-06-09
 
 ### Added

CLAUDE.md

@@ -15,7 +15,8 @@ webclaw/
                       # + proxy pool rotation (per-request)
                       # + PDF content-type detection
                       # + document parsing (DOCX, XLSX, CSV)
-    webclaw-llm/      # LLM provider chain (Ollama -> OpenAI -> Anthropic)
+                      # + layered URL discovery (map) + Serper web search (BYO key)
+    webclaw-llm/      # LLM provider chain (Ollama -> OpenAI -> Gemini -> Anthropic)
                       # + JSON schema extraction, prompt extraction, summarization
     webclaw-pdf/      # PDF text extraction via pdf-extract
     webclaw-mcp/      # MCP server (Model Context Protocol) for AI agents
@@ -30,28 +31,39 @@ Three binaries: `webclaw` (CLI), `webclaw-mcp` (MCP server), `webclaw-server` (R
 - `extractor.rs` — Readability-style scoring: text density, semantic tags, link density penalty
 - `noise.rs` — Shared noise filter: tags, ARIA roles, class/ID patterns. Tailwind-safe.
 - `data_island.rs` — JSON data island extraction for React SPAs, Next.js, Contentful CMS
+- `structured_data.rs` — JSON-LD, Next.js `__NEXT_DATA__`, and SvelteKit data-island extraction
+- `js_eval.rs` — QuickJS sandbox (rquickjs) that runs inline `<script>` tags to recover JS-assigned blobs (`window.__PRELOADED_STATE__`, Next.js `self.__next_f`) the static path can't see. Behind the default `quickjs` feature, gated `cfg(not(target_arch = "wasm32"))` — rquickjs links a C lib and won't build for wasm. Never ungate it (see Hard Rules). Runtime-gated for speed: the VM is skipped entirely when the page has no JS-candidate markers (`has_js_candidate_data`), and it reuses the already-parsed document instead of re-parsing.
+- `endpoints.rs` — API surface discovery: REST paths, GraphQL, and WebSocket endpoints mined from inline scripts + JS bundle text (regex over string literals, DoS-bounded). Pure: caller passes raw text.
 - `markdown.rs` — HTML to markdown with URL resolution, asset collection
-- `llm.rs` — 9-step LLM optimization pipeline (image strip, emphasis strip, link dedup, stat merge, whitespace collapse)
+- `llm/` — directory module (`mod` + `body`/`cleanup`/`images`/`links`/`metadata`): 9-step LLM optimization pipeline (image strip, emphasis strip, link dedup, stat merge, whitespace collapse)
 - `domain.rs` — Domain detection from URL patterns + DOM heuristics
 - `metadata.rs` — OG, Twitter Card, standard meta tag extraction
-- `types.rs` — Core data structures (ExtractionResult, Metadata, Content)
-- `filter.rs` — CSS selector include/exclude filtering (ExtractionOptions)
+- `types.rs` — Core data structures (ExtractionResult, Metadata, Content, plus ExtractionOptions for include/exclude CSS selectors — applied in `extractor.rs`; there is no `filter.rs`)
 - `diff.rs` — Content change tracking engine (snapshot diffing)
 - `brand.rs` — Brand identity extraction from DOM structure and CSS
+- `reddit.rs` — old.reddit.com thread vertical extractor (parses server-rendered HTML directly; no JS/API key). Test fixtures under `testdata/reddit/*.html` are `exclude`d from the published crate (Cargo.toml).
 - `youtube.rs` — `ytInitialPlayerResponse` parser, structured markdown for `youtube.com/watch` URLs (title, channel, views, published, duration, description). Produces the legacy markdown shape — for transcripts and a structured `YoutubeData` block see the production server's `youtube_transcript.rs` short-circuit (yt-dlp via proxy pool).
 
 ### Fetch Modules (`webclaw-fetch`)
-- `client.rs` — FetchClient with wreq BoringSSL TLS impersonation; implements the public `Fetcher` trait so callers (including server adapters) can swap in alternative implementations
-- `browser.rs` — Browser profiles: Chrome (142/136/133/131), Firefox (144/135/133/128)
+- `client.rs` — `FetchClient` with wreq BoringSSL TLS impersonation; also implements batch (`BatchResult`/`BatchExtractResult` — there is no `batch.rs`). Implements the public `Fetcher` trait so callers (incl. server adapters) can swap implementations.
+- `fetcher.rs` — the public `Fetcher` trait (`Send + Sync`). Vertical extractors take `&dyn Fetcher`, not `&FetchClient`.
+- `browser.rs` — `BrowserProfile`/`BrowserVariant` enums only (Chrome, ChromeMacos, Firefox, Safari, SafariIos26, Edge). No version numbers live here.
+- `tls.rs` — the real fingerprint builder: per-variant wreq `Emulation` (cipher/sigalg/curve lists, TLS extension order, HTTP/2 SETTINGS, header wire-order). Browser versions are set HERE: Chrome 145, Firefox 135, Edge 145, Safari 18.3.1, Safari iOS 26. SafariIos26 composes on top of `wreq_util::Profile::SafariIos26`. SSRF-safe redirect policy lives here too.
+- `extractors/` — ~30 vertical site extractors (Amazon, eBay, GitHub, Instagram, LinkedIn, Reddit, YouTube, npm, PyPI, HuggingFace, Etsy, Shopify, WooCommerce, Trustpilot, arXiv, Hacker News, StackOverflow, ...); `extractors/mod.rs` is the dispatch table. All reach the network through `&dyn Fetcher`. Shared helpers (not verticals themselves): `extractors/og.rs` (single-pass Open Graph `og:*` parser, `raw()` vs `unescaped()`), `extractors/github_common.rs` (shared GitHub API fetch + status handling), `extractors/jsonld_product.rs` / `ecommerce_product.rs` (shared JSON-LD product walker reused by the e-commerce verticals).
+- `reddit.rs` / `linkedin.rs` — top-level fetch-side verticals (distinct from `extractors/` and from `webclaw-core`'s parsers): `reddit.rs` rewrites Reddit hosts to `old.reddit.com` (the `*.json` API is blocked) so `webclaw-core::reddit` can parse server-rendered HTML; `linkedin.rs` reconstructs post + comments from the SPA's HTML-escaped JSON in `<code>` tags (the `included` typed-entity array).
+- `progress.rs` — wraps a slow fetch future in `tokio::select!` against an interval, emitting a periodic `# webclaw: still fetching <URL> (Ns)` line to STDERR.
 - `crawler.rs` — BFS same-origin crawler with configurable depth/concurrency/delay
-- `sitemap.rs` — Sitemap discovery and parsing (sitemap.xml, robots.txt)
-- `batch.rs` — Multi-URL concurrent extraction
+- `sitemap.rs` — Sitemap discovery and parsing (sitemap.xml, robots.txt; gzip `.xml.gz` supported via `decode_sitemap_body`, sitemap-index recursion)
+- `map.rs` — layered URL discovery (`discover_urls` / `MapOptions`): sitemaps first, then a bounded same-origin crawl fallback when the sitemap is thin, harvesting links from fetched pages + the unfetched frontier (deduped against the sitemap set)
+- `search.rs` — web search via Serper.dev with the caller's own key (`search` / `SearchOptions` / `SearchResult`; pure `parse_serper_organic`). Plain wreq client (JSON API, no fingerprinting); optional bounded concurrent fetch+extract of result pages. Powers the CLI `search` subcommand, the MCP `search` tool, and the OSS server `POST /v1/search`.
 - `proxy.rs` — Proxy pool with per-request rotation
 - `document.rs` — Document parsing: DOCX, XLSX, CSV auto-detection and extraction
-- `search.rs` — Web search via Serper.dev with parallel result scraping
+- `cloud.rs` — `CloudClient` for hosted antibot escalation, exposed via `Fetcher::cloud()`
+- `locale.rs` — Accept-Language by TLD (`accept_language_for_tld` / `_for_url`)
+- `url_security.rs` — SSRF guards + SSRF-safe redirect policy
 
 ### LLM Modules (`webclaw-llm`)
-- Provider chain: Ollama (local-first) -> OpenAI -> Anthropic
+- Provider chain (`chain.rs`): Ollama (local-first, always added; availability checked at call time) -> OpenAI -> Gemini -> Anthropic -> Atlas Cloud. Gemini sits ahead of Anthropic so Google Cloud credits are preferred; Anthropic is the last-resort fallback. Atlas Cloud is opt-in and added last — only when `ATLASCLOUD_API_KEY` is set — so it never preempts an already-configured provider (`ATLASCLOUD_MODEL` / `ATLASCLOUD_BASE_URL` override its model/endpoint). Each provider lives in `providers/` (`ollama.rs`, `openai.rs`, `gemini.rs`, `anthropic.rs`, `atlascloud.rs`).
 - JSON schema extraction, prompt-based extraction, summarization
 
 ### PDF Modules (`webclaw-pdf`)
@@ -59,26 +71,31 @@ Three binaries: `webclaw` (CLI), `webclaw-mcp` (MCP server), `webclaw-server` (R
 
 ### MCP Server (`webclaw-mcp`)
 - Model Context Protocol server over stdio transport
-- 8 tools: scrape, crawl, map, batch, extract, summarize, diff, brand
+- 14 tools: scrape, crawl, map, batch, extract, summarize, diff, brand, research, search, list_extractors, vertical_scrape, lead, lead_batch. `search` is local-first via the caller's `SERPER_API_KEY` (falls back to the hosted API when unset); `research`, `lead`, and `lead_batch` are hosted-only cloud proxies (require `WEBCLAW_API_KEY`) — `lead`/`lead_batch` call `/v1/lead` and the async `/v1/lead/batch`. The rest run locally.
 - Works with Claude Desktop, Claude Code, and any MCP client
 - Uses `rmcp` crate (official Rust MCP SDK)
 
 ### REST API Server (`webclaw-server`)
 - Axum 0.8, stateless, no database, no job queue
-- 8 POST routes + /health, JSON shapes mirror api.webclaw.io where the
-  capability exists in OSS
+- 10 POST routes (incl. `POST /v1/scrape/{vertical}` and `POST /v1/search`) +
+  `GET /v1/extractors` + `GET /health`. JSON shapes mirror api.webclaw.io
+  where the capability exists in OSS. The vertical surface
+  (`routes/structured.rs`) mirrors the MCP `list_extractors` /
+  `vertical_scrape` tools. `POST /v1/search` is gated on `SERPER_API_KEY`
+  (returns 501 when unset).
 - Constant-time bearer-token auth via `subtle::ConstantTimeEq` when
   `--api-key` / `WEBCLAW_API_KEY` is set; otherwise open mode
 - Hard caps: crawl ≤ 500 pages, batch ≤ 100 URLs, 20 concurrent
 - Does NOT include: anti-bot bypass, JS rendering, async jobs,
-  multi-tenant auth, billing, proxy rotation, search/research/watch/
+  multi-tenant auth, billing, proxy rotation, research/watch/
   agent-scrape. Those live behind api.webclaw.io and are closed-source.
+  (Web search IS available here as a bring-your-own-Serper-key path.)
 
 ## Hard Rules
 
-- **Core has ZERO network dependencies** — takes `&str` HTML, returns structured output. Keep it WASM-compatible.
-- **webclaw-fetch uses wreq 6.x** (BoringSSL). No `[patch.crates-io]` forks needed; wreq handles TLS internally.
-- **No special RUSTFLAGS** — `.cargo/config.toml` is currently empty of build flags. Don't add any.
+- **Core has ZERO network dependencies** — takes `&str` HTML, returns structured output. Keep it WASM-compatible. The `quickjs` feature (default ON) pulls in rquickjs, which links a C lib and can't target wasm32; it's gated `cfg(not(target_arch = "wasm32"))` in `lib.rs`. CI compiles webclaw-core for wasm32 both with AND without default features — never ungate that.
+- **webclaw-fetch pins wreq exactly**: `wreq = "=6.0.0-rc.29"` + `wreq-util = "=3.0.0-rc.12"` (BoringSSL). The `=` pin is deliberate — these are release candidates with no semver stability between rc.N builds. No `[patch.crates-io]` forks needed; wreq handles TLS internally.
+- **No build flags in `.cargo/config.toml`** (it is comments-only) — don't add any locally. BUT CI (`.github/workflows/ci.yml`, `deps.yml`) DOES export `RUSTFLAGS: "--cfg reqwest_unstable"` for the wreq path; don't remove it from CI.
 - **webclaw-llm uses plain reqwest**. LLM APIs don't need TLS fingerprinting, so no wreq dep.
 - **Vertical extractors take `&dyn Fetcher`**, not `&FetchClient`. This lets the production server plug in a `ProductionFetcher` that adds domain_hints routing and antibot escalation on top of the same wreq client.
 - **qwen3 thinking tags** (`<think>`) are stripped at both provider and consumer levels.
@@ -86,12 +103,29 @@ Three binaries: `webclaw` (CLI), `webclaw-mcp` (MCP server), `webclaw-server` (R
 ## Build & Test
 
 ```bash
-cargo build --release           # Both binaries
+cargo build --release           # All three binaries (webclaw, webclaw-mcp, webclaw-server)
 cargo test --workspace          # All tests
 cargo test -p webclaw-core      # Core only
 cargo test -p webclaw-llm       # LLM only
 ```
 
+CI (`.github/workflows/ci.yml`, with `RUSTFLAGS=--cfg reqwest_unstable`) runs four jobs — match them locally before pushing:
+- `cargo test --workspace`
+- `cargo fmt --check --all` + `cargo clippy --all -- -D warnings` (warnings fail CI)
+- `cargo check --target wasm32-unknown-unknown -p webclaw-core` **with and without** `--no-default-features` (guards the WASM-safe rule)
+- `cargo doc --no-deps --workspace`
+
+## Repo Layout & Packaging
+
+Workspace is version **0.6.15**, edition **2024**, license **AGPL-3.0** (matters for the public-OSS scrubbing rules). No crate declares `rust-version`, so MSRV is implicit — edition 2024 floors it at Rust 1.85+; CI pins `dtolnay/rust-toolchain@stable`.
+
+Artifacts outside `crates/` that need separate attention:
+- `packages/create-webclaw/` — `npx create-webclaw` Node scaffolder that installs/configures the MCP server for AI agents (Claude, Cursor, Windsurf, ...). Versioned independently (own `package.json`) — bump it separately when MCP setup changes.
+- `smithery.yaml` + `glama.json` — MCP-registry manifests (Smithery stdio config spawning `webclaw-mcp` with optional `WEBCLAW_API_KEY`; Glama). Update when the MCP launch command or env changes.
+- `examples/` — runnable demos (cloudflare-diagnostics, firecrawl-compatible-api, html-to-markdown-rag, mcp-web-scraping, proxy-backed-crawling).
+- `Dockerfile` / `Dockerfile.ci` / `docker-compose.yml`, `benchmarks/` (`/benchmark` skill).
+- The Claude Code / agent skill is NOT in this repo — it lives in its own skills.sh repo `github.com/0xMassi/webclaw-skill` (`skills/webclaw` + `skills/lead-enrichment`), installed via `npx skills add 0xMassi/webclaw-skill`.
+
 ## CLI
 
 ```bash
@@ -107,12 +141,18 @@ webclaw https://example.com --only-main-content
 webclaw url1 url2 url3 --proxy-file proxies.txt
 webclaw --urls-file urls.txt --concurrency 10
 
-# Sitemap discovery
+# URL discovery (--map): sitemaps first, bounded crawl fallback when the sitemap is thin
 webclaw https://docs.example.com --map
+webclaw https://news.ycombinator.com --map --map-pages 150 --map-limit 500
+webclaw https://docs.example.com --map --no-map-crawl   # sitemap-only (no crawl fallback)
 
 # Crawling (with sitemap seeding)
 webclaw https://docs.example.com --crawl --depth 2 --max-pages 50 --sitemap
 
+# Web search via Serper.dev (bring your own key: --serper-key or SERPER_API_KEY)
+webclaw search "rust async runtime" --num 5
+webclaw search "best web scraper" --scrape -f json   # also fetch + extract result pages
+
 # Change tracking
 webclaw https://example.com -f json > snap.json
 webclaw https://example.com --diff-with snap.json
@@ -140,8 +180,8 @@ cat page.html | webclaw --stdin
 
 - Scoring minimum: 50 chars text length
 - Semantic bonus: +50 for `<article>`/`<main>`, +25 for content class/ID
-- Link density: >50% = 0.1x score, >30% = 0.5x
-- Data island fallback triggers when DOM word count < 30
+- Link density (generic divs): >50% = 0.1x score, >30% = 0.5x. Semantic nodes (article/main/role=main) get a milder curve: >70% = 0.3x, >50% = 0.5x (`extractor.rs`)
+- Data island fallback triggers when DOM word count < 500 (`SPARSE_THRESHOLD` in `data_island.rs`)
 - Eyebrow text max: 80 chars
 
 ## MCP Setup
@@ -151,7 +191,8 @@ Add to Claude Desktop config (`~/Library/Application Support/Claude/claude_deskt
 {
   "mcpServers": {
     "webclaw": {
-      "command": "/path/to/webclaw-mcp"
+      "command": "npx",
+      "args": ["-y", "@webclaw/mcp"]
     }
   }
 }

Cargo.lock

@@ -28,18 +28,6 @@ dependencies = [
  "cpufeatures",
 ]
 
-[[package]]
-name = "ahash"
-version = "0.8.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
-dependencies = [
- "cfg-if",
- "once_cell",
- "version_check",
- "zerocopy",
-]
-
 [[package]]
 name = "aho-corasick"
 version = "1.1.4"
@@ -64,6 +52,12 @@ dependencies = [
  "alloc-no-stdlib",
 ]
 
+[[package]]
+name = "allocator-api2"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
+
 [[package]]
 name = "android_system_properties"
 version = "0.1.5"
@@ -272,9 +266,9 @@ dependencies = [
 
 [[package]]
 name = "bitflags"
-version = "2.11.0"
+version = "2.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af"
+checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8"
 
 [[package]]
 name = "block-buffer"
@@ -285,31 +279,6 @@ dependencies = [
  "generic-array",
 ]
 
-[[package]]
-name = "boring-sys2"
-version = "5.0.0-alpha.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "455d79965f5155dcc88a7abce112c3590883889131b799beda10bf9a813ed669"
-dependencies = [
- "bindgen",
- "cmake",
- "fs_extra",
- "fslock",
-]
-
-[[package]]
-name = "boring2"
-version = "5.0.0-alpha.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "183ccc3854411c035410dcdbffafca62084f3a6c33f013c77e83c025d2a08a28"
-dependencies = [
- "bitflags",
- "boring-sys2",
- "foreign-types",
- "libc",
- "openssl-macros",
-]
-
 [[package]]
 name = "brotli"
 version = "8.0.2"
@@ -331,6 +300,31 @@ dependencies = [
  "alloc-stdlib",
 ]
 
+[[package]]
+name = "btls"
+version = "0.5.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2c5e60b8c8d282c86360cab651ded04ab0335a7b5390c8d34145cbeab8cacf5f"
+dependencies = [
+ "bitflags",
+ "btls-sys",
+ "foreign-types",
+ "libc",
+ "openssl-macros",
+]
+
+[[package]]
+name = "btls-sys"
+version = "0.5.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b1b8638a2e1c38a5ae4efa90ae57e643baec35a30d03fc5b399b893adc4954b"
+dependencies = [
+ "bindgen",
+ "cmake",
+ "fs_extra",
+ "fslock",
+]
+
 [[package]]
 name = "bumpalo"
 version = "3.20.2"
@@ -865,6 +859,12 @@ version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
 
+[[package]]
+name = "foldhash"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
+
 [[package]]
 name = "foreign-types"
 version = "0.5.0"
@@ -1089,19 +1089,13 @@ version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
 
-[[package]]
-name = "hashbrown"
-version = "0.13.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e"
-
 [[package]]
 name = "hashbrown"
 version = "0.15.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
 dependencies = [
- "foldhash",
+ "foldhash 0.1.5",
 ]
 
 [[package]]
@@ -1110,6 +1104,17 @@ version = "0.16.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
 
+[[package]]
+name = "hashbrown"
+version = "0.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
+dependencies = [
+ "allocator-api2",
+ "equivalent",
+ "foldhash 0.2.0",
+]
+
 [[package]]
 name = "heck"
 version = "0.5.0"
@@ -1172,9 +1177,9 @@ dependencies = [
 
 [[package]]
 name = "http2"
-version = "0.5.15"
+version = "0.5.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c45c6490693ee8a8d0d95fdbdf76fead9fb87548f7894137259a7c6d22821948"
+checksum = "569ef7a780e853c4e1768f58a3c8168193b82cdcbab66638a0b1c6583ec5995e"
 dependencies = [
  "atomic-waker",
  "bytes",
@@ -1183,7 +1188,6 @@ dependencies = [
  "futures-sink",
  "http",
  "indexmap",
- "parking_lot",
  "slab",
  "smallvec",
  "tokio",
@@ -1495,9 +1499,9 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
 
 [[package]]
 name = "libc"
-version = "0.2.183"
+version = "0.2.186"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d"
+checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
 
 [[package]]
 name = "libloading"
@@ -1563,6 +1567,15 @@ dependencies = [
  "weezl",
 ]
 
+[[package]]
+name = "lru"
+version = "0.18.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a860605968fce16869fd239cf4237a82f3ac470723415db603b0e8b6c8d4fb9"
+dependencies = [
+ "hashbrown 0.17.1",
+]
+
 [[package]]
 name = "lru-slab"
 version = "0.1.2"
@@ -2375,17 +2388,6 @@ dependencies = [
  "syn",
 ]
 
-[[package]]
-name = "schnellru"
-version = "0.2.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "356285bbf17bea63d9e52e96bd18f039672ac92b55b8cb997d6162a2a37d1649"
-dependencies = [
- "ahash",
- "cfg-if",
- "hashbrown 0.13.2",
-]
-
 [[package]]
 name = "scopeguard"
 version = "1.2.0"
@@ -2779,9 +2781,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokio"
-version = "1.50.0"
+version = "1.52.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d"
+checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe"
 dependencies = [
  "bytes",
  "libc",
@@ -2795,20 +2797,20 @@ dependencies = [
 ]
 
 [[package]]
-name = "tokio-boring2"
-version = "5.0.0-alpha.13"
+name = "tokio-btls"
+version = "0.5.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f81df1210d791f31d72d840de8fbd80b9c3cb324956523048b1413e2bd55756"
+checksum = "2e1fd638ec35427faf3b8f412e0fdd6fae76591d79dba40f38fa667d22bc44dd"
 dependencies = [
- "boring2",
+ "btls",
  "tokio",
 ]
 
 [[package]]
 name = "tokio-macros"
-version = "2.6.1"
+version = "2.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c"
+checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -3219,7 +3221,7 @@ dependencies = [
 
 [[package]]
 name = "webclaw-cli"
-version = "0.6.6"
+version = "0.6.16"
 dependencies = [
  "clap",
  "dotenvy",
@@ -3240,7 +3242,7 @@ dependencies = [
 
 [[package]]
 name = "webclaw-core"
-version = "0.6.6"
+version = "0.6.16"
 dependencies = [
  "ego-tree",
  "once_cell",
@@ -3258,11 +3260,13 @@ dependencies = [
 
 [[package]]
 name = "webclaw-fetch"
-version = "0.6.6"
+version = "0.6.16"
 dependencies = [
  "async-trait",
  "bytes",
  "calamine",
+ "flate2",
+ "futures-util",
  "http",
  "quick-xml 0.37.5",
  "rand 0.8.5",
@@ -3284,7 +3288,7 @@ dependencies = [
 
 [[package]]
 name = "webclaw-llm"
-version = "0.6.6"
+version = "0.6.16"
 dependencies = [
  "async-trait",
  "reqwest",
@@ -3297,7 +3301,7 @@ dependencies = [
 
 [[package]]
 name = "webclaw-mcp"
-version = "0.6.6"
+version = "0.6.16"
 dependencies = [
  "dirs",
  "dotenvy",
@@ -3317,7 +3321,7 @@ dependencies = [
 
 [[package]]
 name = "webclaw-pdf"
-version = "0.6.6"
+version = "0.6.16"
 dependencies = [
  "pdf-extract",
  "thiserror",
@@ -3326,7 +3330,7 @@ dependencies = [
 
 [[package]]
 name = "webclaw-server"
-version = "0.6.6"
+version = "0.6.16"
 dependencies = [
  "anyhow",
  "axum",
@@ -3347,9 +3351,9 @@ dependencies = [
 
 [[package]]
 name = "webpki-root-certs"
-version = "1.0.6"
+version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "804f18a4ac2676ffb4e8b5b5fa9ae38af06df08162314f96a68d2a363e21a8ca"
+checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c"
 dependencies = [
  "rustls-pki-types",
 ]
@@ -3696,17 +3700,14 @@ dependencies = [
 
 [[package]]
 name = "wreq"
-version = "6.0.0-rc.28"
+version = "6.0.0-rc.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f79937f6c4df65b3f6f78715b9de2977afe9ee3b3436483c7949a24511e25935"
+checksum = "3f0eba5f5814a94e5f1a99156f187133464e525b66bdbc69a9627d46530af2e1"
 dependencies = [
- "ahash",
- "boring2",
- "brotli",
+ "btls",
+ "btls-sys",
  "bytes",
  "cookie",
- "flate2",
- "futures-channel",
  "futures-util",
  "http",
  "http-body",
@@ -3715,29 +3716,64 @@ dependencies = [
  "httparse",
  "ipnet",
  "libc",
+ "lru",
  "percent-encoding",
  "pin-project-lite",
- "schnellru",
- "smallvec",
  "socket2",
+ "sync_wrapper",
  "tokio",
- "tokio-boring2",
+ "tokio-btls",
+ "tokio-util",
  "tower",
  "tower-http",
  "url",
- "want",
  "webpki-root-certs",
- "zstd",
+ "wreq-proto",
+ "wreq-rt",
+]
+
+[[package]]
+name = "wreq-proto"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a43942f024bb303f1042c9aa3c87fa1d9149f507c65db6e5220a11ccdb207387"
+dependencies = [
+ "bytes",
+ "futures-channel",
+ "futures-util",
+ "http",
+ "http-body",
+ "http2",
+ "httparse",
+ "pin-project-lite",
+ "smallvec",
+ "tokio",
+ "tokio-util",
+ "want",
+]
+
+[[package]]
+name = "wreq-rt"
+version = "0.2.2-rc.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "99e9bce67a3fa3dd3f1503f066d86661c9caf399a763d3bd184da7afaf886c8b"
+dependencies = [
+ "pin-project-lite",
+ "tokio",
+ "wreq-proto",
 ]
 
 [[package]]
 name = "wreq-util"
-version = "3.0.0-rc.10"
+version = "3.0.0-rc.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c6bbe24d28beb9ceb58b514bd6a613c759d3b706f768b9d2950d5d35b543c04"
+checksum = "baa5d2ab72139256916ca352a3d05c53d74e1dd360052eb5ba7691033c417c65"
 dependencies = [
+ "brotli",
+ "flate2",
  "typed-builder",
  "wreq",
+ "zstd",
 ]
 
 [[package]]

Cargo.toml

@@ -3,7 +3,7 @@ resolver = "2"
 members = ["crates/*"]
 
 [workspace.package]
-version = "0.6.6"
+version = "0.6.16"
 edition = "2024"
 license = "AGPL-3.0"
 repository = "https://github.com/0xMassi/webclaw"

Dockerfile.ci

@@ -1,7 +1,6 @@
 # Slim runtime image — uses pre-built binaries from the release.
 # The full Dockerfile (multi-stage Rust build) is for local development.
 # CI uses this to avoid 60+ min QEMU cross-compilation.
-ARG BINARY_DIR=binaries
 
 FROM ubuntu:24.04
 
@@ -10,10 +9,13 @@ FROM ubuntu:24.04
 # CI runners and breaks the multi-arch release build. No build-time network.
 COPY --from=gcr.io/distroless/static-debian12 /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 
-ARG BINARY_DIR
-COPY ${BINARY_DIR}/webclaw /usr/local/bin/webclaw
-COPY ${BINARY_DIR}/webclaw-mcp /usr/local/bin/webclaw-mcp
-COPY ${BINARY_DIR}/webclaw-server /usr/local/bin/webclaw-server
+# TARGETARCH (amd64 / arm64) is provided automatically by buildx for each
+# target platform, so one multi-platform build copies the matching binaries.
+# The release workflow stages them in binaries-amd64 / binaries-arm64.
+ARG TARGETARCH
+COPY binaries-${TARGETARCH}/webclaw /usr/local/bin/webclaw
+COPY binaries-${TARGETARCH}/webclaw-mcp /usr/local/bin/webclaw-mcp
+COPY binaries-${TARGETARCH}/webclaw-server /usr/local/bin/webclaw-server
 
 # Default REST API port when running `webclaw-server` inside the container.
 EXPOSE 3000
@@ -25,8 +27,9 @@ ENV WEBCLAW_HOST=0.0.0.0
 
 # Entrypoint shim: forwards webclaw args/URL to the binary, but exec's other
 # commands directly so this image can be used as a FROM base with custom CMD.
-COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
-RUN chmod +x /usr/local/bin/docker-entrypoint.sh
+# `--chmod` sets the bit at copy time so the build needs no in-container `RUN`
+# (and thus no QEMU emulation for the arm64 platform).
+COPY --chmod=755 docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
 
 ENTRYPOINT ["docker-entrypoint.sh"]
 CMD ["webclaw", "--help"]

README.md

@@ -1,3 +1,5 @@
+<p align="center"><strong>English</strong> | <a href="./README_zh-CN.md">简体中文</a></p>
+
 <p align="center">
   <a href="https://webclaw.io">
     <img src=".github/banner.png" alt="webclaw" width="760" />
@@ -25,6 +27,10 @@
   <a href="https://webclaw.io/docs"><img src="https://shieldcn.dev/badge/Docs-Read.svg?variant=branded&logo=readthedocs" alt="Docs" /></a>
 </p>
 
+<p align="center">
+  <a href="https://trendshift.io/repositories/24218?utm_source=trendshift-badge&amp;utm_medium=badge&amp;utm_campaign=badge-trendshift-24218" target="_blank" rel="noopener noreferrer"><img src="https://trendshift.io/api/badge/trendshift/repositories/24218/daily?language=Rust" alt="0xMassi/webclaw | Trendshift" width="250" height="55"/></a>
+</p>
+
 <p align="center">
   <img src="assets/demo.gif" alt="webclaw extracting clean markdown from a page" width="760" />
 </p>
@@ -77,7 +83,7 @@ brew install webclaw
 
 ### Prebuilt binaries
 
-Download macOS and Linux binaries from [GitHub Releases](https://github.com/0xMassi/webclaw/releases).
+Download macOS, Linux, and Windows binaries from [GitHub Releases](https://github.com/0xMassi/webclaw/releases).
 
 ### Docker
 
@@ -142,7 +148,7 @@ webclaw https://docs.rust-lang.org --crawl --depth 2 --max-pages 50
 - [HTML to Markdown for RAG](examples/html-to-markdown-rag/)
 - [Firecrawl-compatible API](examples/firecrawl-compatible-api/)
 - [MCP web scraping](examples/mcp-web-scraping/)
-- [Proxy-backed crawling](examples/proxy-backed-crawling/)
+- [Proxy-backed crawling with ColdProxy](examples/proxy-backed-crawling/)
 - [Cloudflare diagnostics](examples/cloudflare-diagnostics/)
 
 ### Extract brand assets
@@ -164,22 +170,21 @@ webclaw https://example.com/pricing --diff-with pricing-old.json
 
 webclaw ships with an MCP server for AI agents.
 
-```bash
-npx create-webclaw
-```
-
-Manual config:
+Zero-install — point any MCP client at the npx launcher:
 
 ```json
 {
   "mcpServers": {
     "webclaw": {
-      "command": "~/.webclaw/webclaw-mcp"
+      "command": "npx",
+      "args": ["-y", "@webclaw/mcp"]
     }
   }
 }
 ```
 
+Or run `npx create-webclaw` to auto-detect your AI tools and write their configs for you.
+
 Then ask your agent things like:
 
 ```text
@@ -196,6 +201,22 @@ Extract the brand colors, fonts, and logos from this company website.
 
 ---
 
+## Use as an agent skill
+
+Add webclaw to Claude Code, Cursor, Windsurf, and other MCP agents in one command:
+
+```bash
+npx skills add 0xMassi/webclaw-skill
+```
+
+Your agent gets scrape, crawl, map, extract, summarize, diff, brand, and search
+as native tools. Most sites extract locally with no API key. Set `WEBCLAW_API_KEY`
+to handle bot-protected and JavaScript-rendered pages.
+
+Find it on [skills.sh](https://www.skills.sh/0xMassi/webclaw-skill/webclaw).
+
+---
+
 ## Tools
 
 | Tool | What it does | Local |
@@ -390,7 +411,7 @@ Please remove secrets, cookies, private tokens, and customer data from logs befo
 <table>
   <tr>
     <td align="center">
-      <a href="https://coldproxy.com/">
+      <a href="https://coldproxy.com/?utm_source=github&utm_medium=sponsorship&utm_campaign=webclaw-sponsor">
         <img src="./assets/sponsors/coldproxy-banner.png" alt="ColdProxy" width="720" />
       </a>
     </td>
@@ -400,7 +421,10 @@ Please remove secrets, cookies, private tokens, and customer data from logs befo
       <strong>ColdProxy</strong> supports webclaw as an Infrastructure Partner, providing residential IPv4,
       residential IPv6, and datacenter IPv6 proxy infrastructure across 195+ countries for public data
       collection, regional testing, monitoring, and web scraping workflows. Explore
-      <a href="https://coldproxy.com/">ColdProxy</a>'s latest plans and available offers directly on the website.
+      <a href="https://coldproxy.com/?utm_source=github&utm_medium=sponsorship&utm_campaign=webclaw-sponsor">ColdProxy</a>'s latest plans and available offers directly on the website.
+      Use code <code>webclaw8Off</code> for 8% off your first payment.
+      See the <a href="examples/proxy-backed-crawling/#using-coldproxy">proxy-backed crawling guide</a>
+      for a hands-on walkthrough of wiring ColdProxy into webclaw.
     </td>
   </tr>
 </table>
@@ -412,40 +436,29 @@ Please remove secrets, cookies, private tokens, and customer data from logs befo
 <table>
   <tr>
     <td width="340" align="center">
-      <a href="https://quantumproxies.net/?utm_source=webclaw&utm_medium=github&utm_campaign=sponsor">
-        <img src="./assets/sponsors/quantum-proxies-banner.png" alt="Quantum Proxies" width="300" />
+      <a href="https://go.nodemaven.com/webclawgh">
+        <img src="./assets/sponsors/nodemaven-banner.png" alt="NodeMaven" width="300" />
       </a>
     </td>
     <td>
-      <strong>Quantum Proxies</strong> provides fast, reliable residential and ISP proxy infrastructure for developers running large-scale extraction workloads.
-      Get 20% off any plan with code <code>WEBCLAW20</code> at
-      <a href="https://quantumproxies.net/?utm_source=webclaw&utm_medium=github&utm_campaign=sponsor">quantumproxies.net</a>.
+      <strong>NodeMaven</strong> is the most reliable proxy provider with the highest-quality IPs on the market.
+      Best solution for automation, web scraping, SEO research, and social media management: 99.9% uptime,
+      sticky sessions up to 7 days, IP filtering (all proxies under a 97% fraud score), no KYC, and cashback up
+      to 10% on traffic. Use <code>WEBCLAW35</code> for 35% off Mobile and Residential proxies, or
+      <code>WEBCLAW40</code> for 40% off ISP (Static) proxies at
+      <a href="https://go.nodemaven.com/webclawgh">NodeMaven</a>.
     </td>
   </tr>
   <tr>
     <td width="340" align="center">
-      <a href="https://proxy-seller.com/?partner=KXMQNNLIGHXR4B">
-        <img src="./assets/sponsors/proxy-seller-banner.png" alt="Proxy-Seller" width="300" />
+      <a href="https://mangoproxy.com/?utm_source=github&utm_medium=partner&utm_campaign=0xmassi">
+        <img src="./assets/sponsors/mangoproxy-banner.png" alt="MangoProxy" width="300" />
       </a>
     </td>
     <td>
-      <strong>Proxy-Seller</strong> maintains a global network of residential and datacenter proxies optimized for web extraction at scale.
-      The service supports high-volume concurrent scraping, geographic rotation, and integration with web extraction tools.
-      Use code <code>WBC15</code> for 15% off IPv4, IPv6, ISP, and Residential proxies, and 10% off Mobile at
-      <a href="https://proxy-seller.com/?partner=KXMQNNLIGHXR4B">proxy-seller.com</a>.
-    </td>
-  </tr>
-  <tr>
-    <td width="340" align="center">
-      <a href="https://www.rapidproxy.io/?ref=webclaw">
-        <img src="./assets/sponsors/rapidproxy-banner.png" alt="RapidProxy" width="300" />
-      </a>
-    </td>
-    <td>
-      <strong>RapidProxy</strong> delivers fast, reliable proxy infrastructure for large-scale data collection.
-      With 90M+ residential IPs, smart rotation, high concurrency, AI-powered CAPTCHA bypass, and non-expiring traffic, it helps keep scraping workflows stable at scale.
-      Use code <code>webclaw</code> for 10% off, or
-      <a href="https://www.rapidproxy.io/?ref=webclaw">Try it free</a>.
+      <strong>MangoProxy</strong> provides residential, ISP, datacenter, and mobile proxies across 200+ locations, backed by a 90M+ IP pool with HTTP and SOCKS5 support and high stability for web scraping and data collection at scale.
+      Use code <code>0XMASSI</code> for 8% off ISP (Static) proxies at
+      <a href="https://mangoproxy.com/?utm_source=github&utm_medium=partner&utm_campaign=0xmassi">mangoproxy.com</a>.
     </td>
   </tr>
 </table>
@@ -477,11 +490,11 @@ Thanks to everyone improving webclaw through issues, examples, docs, bug reports
 
 ## Star History
 
-<a href="https://www.star-history.com/?repos=0xMassi%2Fwebclaw&type=date&legend=top-left">
+<a href="https://github.com/0xMassi/webclaw/stargazers">
  <picture>
-   <source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/chart?repos=0xMassi/webclaw&type=date&theme=dark&legend=top-left" />
-   <source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/chart?repos=0xMassi/webclaw&type=date&legend=top-left" />
-   <img alt="Star History Chart" src="https://api.star-history.com/chart?repos=0xMassi/webclaw&type=date&legend=top-left" />
+   <source media="(prefers-color-scheme: dark)" srcset="https://shieldcn.dev/chart/github/stars/0xMassi/webclaw.svg?mode=dark&theme=zinc&bg=transparent&border=false&logo=false&icon=Star" />
+   <source media="(prefers-color-scheme: light)" srcset="https://shieldcn.dev/chart/github/stars/0xMassi/webclaw.svg?mode=light&theme=zinc&bg=transparent&border=false&logo=false&icon=Star" />
+   <img alt="Star History Chart" src="https://shieldcn.dev/chart/github/stars/0xMassi/webclaw.svg?mode=light&theme=zinc&bg=transparent&border=false&logo=false&icon=Star" />
  </picture>
 </a>
 

README_zh-CN.md

@@ -0,0 +1,495 @@
+<p align="center"><a href="./README.md">English</a> | <strong>简体中文</strong></p>
+
+<p align="center">
+  <a href="https://webclaw.io">
+    <img src=".github/banner.png" alt="webclaw" width="760" />
+  </a>
+</p>
+
+<h1 align="center">webclaw</h1>
+
+<p align="center">
+  <strong>将任意网页转化为干净的 Markdown、JSON 与 LLM 就绪的上下文。</strong><br/>
+  <sub>面向 AI 智能体与 RAG 流水线的 CLI、MCP 服务器、REST API 与多语言 SDK。</sub>
+</p>
+
+<p align="center">
+  <a href="https://github.com/0xMassi/webclaw/stargazers"><img src="https://shieldcn.dev/github/stars/0xMassi/webclaw.svg?variant=branded&logo=github" alt="Stars" /></a>
+  <a href="https://github.com/0xMassi/webclaw/releases"><img src="https://shieldcn.dev/github/tag/0xMassi/webclaw.svg?variant=branded&logo=rust" alt="Version" /></a>
+  <a href="https://github.com/0xMassi/webclaw/blob/main/LICENSE"><img src="https://shieldcn.dev/github/license/0xMassi/webclaw.svg?variant=branded" alt="License" /></a>
+  <a href="https://www.npmjs.com/package/create-webclaw"><img src="https://shieldcn.dev/npm/dt/create-webclaw.svg?variant=branded" alt="npm installs" /></a>
+</p>
+
+<p align="center">
+  <a href="https://discord.gg/KDfd48EpnW"><img src="https://shieldcn.dev/badge/Discord-Join.svg?variant=branded&logo=discord" alt="Discord" /></a>
+  <a href="https://x.com/webclaw_io"><img src="https://shieldcn.dev/badge/Follow-@webclaw__io.svg?variant=branded&logo=x" alt="X / Twitter" /></a>
+  <a href="https://webclaw.io"><img src="https://shieldcn.dev/badge/Hosted-webclaw.io.svg?variant=branded&logo=safari" alt="Hosted webclaw" /></a>
+  <a href="https://webclaw.io/docs"><img src="https://shieldcn.dev/badge/Docs-Read.svg?variant=branded&logo=readthedocs" alt="Docs" /></a>
+</p>
+
+<p align="center">
+  <img src="assets/demo.gif" alt="webclaw 从网页中提取干净的 Markdown" width="760" />
+</p>
+
+---
+
+大多数网页抓取工具，只会给你的智能体两种糟糕的结果之一：
+
+- 被拦截的页面、登录墙，或空壳应用（app shell）
+- 塞满导航栏、脚本、样式、广告和重复模板的原始 HTML
+
+[webclaw.io](https://webclaw.io) 是 webclaw 的托管式网页提取 API。本仓库包含开源的 CLI、MCP 服务器、提取引擎，以及可自托管的服务端。
+
+webclaw 用 Rust 编写，把一个 URL 变成你的工具真正用得上的干净内容——它是一个**开源、可自托管的 Firecrawl 替代方案**。
+
+```bash
+webclaw https://example.com --format markdown
+```
+
+```md
+# Example Domain
+
+This domain is for use in illustrative examples in documents.
+
+You may use this domain in literature without prior coordination or asking for permission.
+```
+
+在终端里直接用，通过 MCP 接入 Claude / Cursor，从你的应用调用托管 API，或者自托管这套开源服务端——任你选择。
+
+---
+
+## 安装
+
+### 智能体一键配置
+
+把 webclaw 接入 Claude Code、Claude Desktop、Cursor、Windsurf、OpenCode、Codex CLI 以及其他兼容 MCP 的工具，最快的方式是：
+
+```bash
+npx create-webclaw
+```
+
+安装器会自动检测已支持的客户端，并为你写好 MCP 服务器的配置。
+
+### Homebrew
+
+```bash
+brew tap 0xMassi/webclaw
+brew install webclaw
+```
+
+### 预编译二进制
+
+从 [GitHub Releases](https://github.com/0xMassi/webclaw/releases) 下载 macOS、Linux 和 Windows 二进制文件。
+
+### Docker
+
+```bash
+docker run --rm ghcr.io/0xmassi/webclaw https://example.com
+```
+
+### Cargo
+
+```bash
+cargo install --git https://github.com/0xMassi/webclaw.git webclaw-cli
+cargo install --git https://github.com/0xMassi/webclaw.git webclaw-mcp
+```
+
+如果从源码构建时因缺少本机构建工具而失败，请先安装对应平台的依赖：
+
+| 操作系统 | 命令 |
+| --- | --- |
+| Debian / Ubuntu | `sudo apt install -y pkg-config libssl-dev cmake clang git build-essential` |
+| Fedora / RHEL | `sudo dnf install -y pkg-config openssl-devel cmake clang git make gcc` |
+| Arch | `sudo pacman -S pkg-config openssl cmake clang git base-devel` |
+| macOS | `xcode-select --install` |
+
+---
+
+## 快速开始
+
+### 抓取单个页面
+
+```bash
+webclaw https://stripe.com --format markdown
+```
+
+### 返回 LLM 优化文本
+
+```bash
+webclaw https://docs.anthropic.com --format llm
+```
+
+### 只保留正文内容
+
+```bash
+webclaw https://example.com/blog/post --only-main-content
+```
+
+### 包含或排除选择器
+
+```bash
+webclaw https://example.com \
+  --include "article, main, .content" \
+  --exclude "nav, footer, .sidebar, .ad"
+```
+
+### 爬取一个文档站点
+
+```bash
+webclaw https://docs.rust-lang.org --crawl --depth 2 --max-pages 50
+```
+
+### 工作流示例
+
+- [HTML 转 Markdown 用于 RAG](examples/html-to-markdown-rag/)
+- [兼容 Firecrawl 的 API](examples/firecrawl-compatible-api/)
+- [基于 MCP 的网页抓取](examples/mcp-web-scraping/)
+- [使用 ColdProxy 的代理支持爬取](examples/proxy-backed-crawling/)
+
+### 提取品牌资产
+
+```bash
+webclaw https://github.com --brand
+```
+
+### 对比页面随时间的变化
+
+```bash
+webclaw https://example.com/pricing --format json > pricing-old.json
+webclaw https://example.com/pricing --diff-with pricing-old.json
+```
+
+---
+
+## MCP 服务器
+
+webclaw 内置了面向 AI 智能体的 MCP 服务器。
+
+无需安装——把任意 MCP 客户端指向 npx 启动器即可：
+
+```json
+{
+  "mcpServers": {
+    "webclaw": {
+      "command": "npx",
+      "args": ["-y", "@webclaw/mcp"]
+    }
+  }
+}
+```
+
+或者运行 `npx create-webclaw`，让它自动检测你的 AI 工具并替你写好配置。
+
+配好之后，就可以让你的智能体做这样的事：
+
+```text
+抓取这些竞品的定价页面，并总结它们之间的差异。
+```
+
+```text
+爬取这个文档站点，为 RAG 索引准备干净的上下文。
+```
+
+```text
+从这家公司的官网提取品牌色、字体和 logo。
+```
+
+---
+
+## 作为智能体 Skill 使用
+
+一条命令，就能把 webclaw 添加到 Claude Code、Cursor、Windsurf 及其他 MCP 智能体：
+
+```bash
+npx skills add 0xMassi/webclaw-skill
+```
+
+你的智能体会把 scrape、crawl、map、extract、summarize、diff、brand、search 作为原生工具使用。多数站点无需 API Key 即可在本地提取；设置 `WEBCLAW_API_KEY` 后即可处理受保护以及需要 JavaScript 渲染的页面。
+
+可在 [skills.sh](https://www.skills.sh/0xMassi/webclaw-skill/webclaw) 上找到它。
+
+---
+
+## 工具
+
+| 工具 | 作用 | 本地 |
+| --- | --- | :-: |
+| `scrape` | 将单个 URL 提取为 markdown、text、JSON、LLM 格式或 HTML | 是 |
+| `crawl` | 跟随同源链接并提取发现的页面 | 是 |
+| `map` | 发现 URL，但不逐页提取 | 是 |
+| `batch` | 并行抓取多个 URL | 是 |
+| `extract` | 将页面内容转换为结构化数据 | 是（本地或已配置的 LLM） |
+| `summarize` | 总结一个页面 | 是（本地或已配置的 LLM） |
+| `diff` | 对比页面内容快照 | 是 |
+| `brand` | 提取颜色、字体、logo 与元数据 | 是 |
+| `search` | 搜索网络并抓取结果 | 托管 API |
+| `research` | 多来源深度研究工作流 | 托管 API |
+
+---
+
+## SDK
+
+```bash
+npm install @webclaw/sdk
+pip install webclaw
+go get github.com/0xMassi/webclaw-go
+```
+
+<details>
+<summary>TypeScript</summary>
+
+```ts
+import { Webclaw } from "@webclaw/sdk";
+
+const client = new Webclaw({ apiKey: process.env.WEBCLAW_API_KEY! });
+
+const page = await client.scrape({
+  url: "https://example.com",
+  formats: ["markdown"],
+  only_main_content: true,
+});
+
+console.log(page.markdown);
+```
+
+</details>
+
+<details>
+<summary>Python</summary>
+
+```python
+from webclaw import Webclaw
+
+client = Webclaw(api_key="wc_your_key")
+
+page = client.scrape(
+    "https://example.com",
+    formats=["markdown"],
+    only_main_content=True,
+)
+
+print(page.markdown)
+```
+
+</details>
+
+<details>
+<summary>cURL</summary>
+
+```bash
+curl -X POST https://api.webclaw.io/v1/scrape \
+  -H "Authorization: Bearer $WEBCLAW_API_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "url": "https://example.com",
+    "formats": ["markdown"],
+    "only_main_content": true
+  }'
+```
+
+</details>
+
+---
+
+## 输出格式
+
+| 格式 | 适用场景 |
+| --- | --- |
+| `markdown` | 保留结构的干净页面内容 |
+| `llm` | 面向智能体与 RAG 流水线的紧凑 LLM 上下文 |
+| `text` | 最少格式的纯文本 |
+| `json` | 结构化元数据、链接、图片与已提取字段 |
+| `html` | 清洗后的 HTML，便于自定义处理 |
+
+---
+
+## 本地优先，按需上云
+
+核心提取路径下，CLI 与 MCP 服务器无需账号即可在本地运行。
+
+当你需要以下能力时，使用托管 API [webclaw.io](https://webclaw.io)：
+
+- 无需自行运维基础设施即可访问受保护的站点
+- JavaScript 渲染
+- 异步的爬取与研究任务
+- 网络搜索
+- 页面监控（watches）与生产用量统计
+- 供应用代码调用的 SDK
+
+```bash
+export WEBCLAW_API_KEY=wc_your_key
+
+webclaw https://example.com --cloud
+```
+
+---
+
+## 你可以用它构建什么
+
+| 场景 | 示例 |
+| --- | --- |
+| AI 智能体的网页访问 | 为 Claude、Cursor 或其他 MCP 客户端提供干净的页面上下文 |
+| RAG 数据入库 | 爬取文档、帮助中心、博客与知识库 |
+| 竞品监控 | 追踪定价页、更新日志、文档与产品页 |
+| 结构化提取 | 把杂乱的页面变成带类型的 JSON，用于自动化 |
+| 研究工作流 | 搜索、抓取、总结并引用多个来源 |
+| 品牌情报 | 提取 logo、颜色、字体与社交元数据 |
+
+## 架构
+
+```text
+webclaw/
+  crates/
+    webclaw-core     HTML 转 markdown、text、JSON 与 LLM 就绪输出
+    webclaw-fetch    抓取、爬取、批处理与站点映射
+    webclaw-llm      本地与托管 LLM 提供方支持
+    webclaw-pdf      PDF 文本提取
+    webclaw-mcp      面向 AI 智能体的 MCP 服务器
+    webclaw-cli      命令行界面
+```
+
+`webclaw-core` 是纯粹的提取逻辑：没有网络 I/O，接口面小，可脱离抓取层独立使用。
+
+---
+
+## 配置
+
+| 变量 | 说明 |
+| --- | --- |
+| `WEBCLAW_API_KEY` | 托管 API 密钥 |
+| `OLLAMA_HOST` | 本地 LLM 功能所用的 Ollama 地址 |
+| `OPENAI_API_KEY` | 兼容 OpenAI 的 LLM 提供方密钥 |
+| `OPENAI_BASE_URL` | 兼容 OpenAI 的接口地址 |
+| `ANTHROPIC_API_KEY` | 兼容 Anthropic 的 LLM 提供方密钥 |
+| `ANTHROPIC_BASE_URL` | 兼容 Anthropic 的接口地址 |
+| `WEBCLAW_PROXY` | 单个代理 URL |
+| `WEBCLAW_PROXY_FILE` | 代理池文件 |
+
+---
+
+## 参与贡献
+
+眼下最有价值的贡献都很务实、也很小：
+
+- 为真实的智能体与 RAG 工作流补充示例
+- 改进 SDK 代码片段
+- 上报提取效果不佳的页面
+- 为杂乱的 HTML 添加会失败的测试样例（fixtures）
+- 完善面向 MCP 客户端与本地配置的文档
+- 在更多 Linux / macOS 环境上测试 CLI
+
+不错的入手点：
+
+- [Good first issues](https://github.com/0xMassi/webclaw/issues?q=label%3A%22good+first+issue%22)
+- [提交一个 bug 报告](https://github.com/0xMassi/webclaw/issues/new)
+- [发起一场讨论](https://github.com/0xMassi/webclaw/discussions)
+
+如果某个页面提取效果不好，请附上：
+
+```text
+URL:
+命令或 API 请求:
+期望输出:
+实际输出:
+使用的格式: markdown / llm / text / json / html
+使用方式: CLI / MCP / SDK / API:
+```
+
+发帖前，请从日志中移除密钥、Cookie、私有令牌和客户数据。
+
+---
+
+## 基础设施合作伙伴
+
+<table>
+  <tr>
+    <td align="center">
+      <a href="https://coldproxy.com/?utm_source=github&utm_medium=sponsorship&utm_campaign=webclaw-sponsor">
+        <img src="./assets/sponsors/coldproxy-banner.png" alt="ColdProxy" width="720" />
+      </a>
+    </td>
+  </tr>
+  <tr>
+    <td>
+      <strong>ColdProxy</strong> 作为基础设施合作伙伴支持 webclaw，提供覆盖 195+ 个国家/地区的住宅 IPv4、
+      住宅 IPv6 与数据中心 IPv6 代理基础设施，适用于公开数据采集、区域测试、监控与网页抓取工作流。
+      在官网了解 <a href="https://coldproxy.com/?utm_source=github&utm_medium=sponsorship&utm_campaign=webclaw-sponsor">ColdProxy</a> 的最新套餐与优惠。
+      使用优惠码 <code>webclaw8Off</code>，首单可享 8% 折扣。
+      详见<a href="examples/proxy-backed-crawling/#using-coldproxy">代理支持爬取指南</a>，了解如何把 ColdProxy 接入 webclaw。
+    </td>
+  </tr>
+</table>
+
+---
+
+## 工作室合作伙伴
+
+<table>
+  <tr>
+    <td width="340" align="center">
+      <a href="https://go.nodemaven.com/webclawgh">
+        <img src="./assets/sponsors/nodemaven-banner.png" alt="NodeMaven" width="300" />
+      </a>
+    </td>
+    <td>
+      <strong>NodeMaven</strong> 提供市场上最可靠、IP 质量最高的代理服务。适用于自动化、网页抓取、SEO 研究
+      与社媒管理：99.9% 在线率、最长 7 天的粘性会话、IP 过滤（所有代理欺诈评分低于 97%）、无需 KYC，
+      并提供最高 10% 的流量返现。在 <a href="https://go.nodemaven.com/webclawgh">NodeMaven</a> 使用
+      <code>WEBCLAW35</code> 享移动与住宅代理 35% 折扣，或用 <code>WEBCLAW40</code> 享 ISP（静态）代理 40% 折扣。
+    </td>
+  </tr>
+  <tr>
+    <td width="340" align="center">
+      <a href="https://mangoproxy.com/?utm_source=github&utm_medium=partner&utm_campaign=0xmassi">
+        <img src="./assets/sponsors/mangoproxy-banner.png" alt="MangoProxy" width="300" />
+      </a>
+    </td>
+    <td>
+      <strong>MangoProxy</strong> 提供覆盖 200+ 地区的住宅、ISP、数据中心与移动代理，背靠 9000 万+ IP 池，
+      支持 HTTP 与 SOCKS5，为大规模网页抓取与数据采集提供高稳定性。在
+      <a href="https://mangoproxy.com/?utm_source=github&utm_medium=partner&utm_campaign=0xmassi">mangoproxy.com</a>
+      使用优惠码 <code>0XMASSI</code> 享 ISP（静态）代理 8% 折扣。
+    </td>
+  </tr>
+</table>
+
+---
+
+## 社区插件
+
+将 webclaw 集成到 AI 智能体平台的第三方插件：
+
+| 插件 | 平台 | 作用 |
+|---|---|---|
+| [openclaw-webclaw](https://github.com/jal-co/openclaw-webclaw) | [OpenClaw](https://openclaw.ai) | 原生 webclaw v1 API 插件，含 9 个工具：scrape、search、crawl、extract、summarize、diff、map、batch、brand |
+| [hermes-webclaw](https://github.com/jal-co/hermes-webclaw) | [Hermes Agent](https://github.com/NousResearch/hermes-agent) | 面向完整 v1 API 的网络搜索提供方与 9 个专用工具。通过 `hermes plugins install jal-co/hermes-webclaw` 安装 |
+
+做了 webclaw 集成？欢迎 [提交 PR](https://github.com/0xMassi/webclaw/pulls) 把它加到这里。
+
+---
+
+## 贡献者
+
+感谢每一位通过 issue、示例、文档、bug 报告与 PR 改进 webclaw 的人。
+
+<a href="https://github.com/0xMassi/webclaw/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=0xMassi/webclaw" alt="webclaw contributors" />
+</a>
+
+---
+
+## Star 历史
+
+<a href="https://github.com/0xMassi/webclaw/stargazers">
+ <picture>
+   <source media="(prefers-color-scheme: dark)" srcset="https://shieldcn.dev/chart/github/stars/0xMassi/webclaw.svg?mode=dark&theme=zinc&bg=transparent&border=false&logo=false&icon=Star" />
+   <source media="(prefers-color-scheme: light)" srcset="https://shieldcn.dev/chart/github/stars/0xMassi/webclaw.svg?mode=light&theme=zinc&bg=transparent&border=false&logo=false&icon=Star" />
+   <img alt="Star History Chart" src="https://shieldcn.dev/chart/github/stars/0xMassi/webclaw.svg?mode=light&theme=zinc&bg=transparent&border=false&logo=false&icon=Star" />
+ </picture>
+</a>
+
+---
+
+## 许可证
+
+[AGPL-3.0](LICENSE)

SKILL.md

@@ -1,634 +0,0 @@
----
-name: webclaw
-description: Web extraction engine with antibot bypass. Scrape, crawl, extract, summarize, search, map, diff, monitor, research, and analyze any URL — including Cloudflare-protected sites. Use when you need reliable web content, the built-in web_fetch fails, or you need structured data extraction from web pages.
-homepage: https://webclaw.io
-user-invocable: true
-metadata: {"openclaw":{"emoji":"🦀","requires":{"env":["WEBCLAW_API_KEY"]},"primaryEnv":"WEBCLAW_API_KEY","homepage":"https://webclaw.io","install":[{"id":"npx","kind":"node","bins":["webclaw-mcp"],"label":"npx create-webclaw"}]}}
----
-
-# webclaw
-
-High-quality web extraction with automatic antibot bypass. Beats Firecrawl on extraction quality and handles Cloudflare, DataDome, and JS-rendered pages automatically.
-
-## When to use this skill
-
-- **Always** when you need to fetch web content and want reliable results
-- When `web_fetch` returns empty/blocked content (403, Cloudflare challenges)
-- When you need structured data extraction (pricing tables, product info)
-- When you need to crawl an entire site or discover all URLs
-- When you need LLM-optimized content (cleaner than raw markdown)
-- When you need to summarize a page without reading the full content
-- When you need to detect content changes between visits
-- When you need brand identity analysis (colors, fonts, logos)
-- When you need web search results with optional page scraping
-- When you need deep multi-source research on a topic
-- When you need AI-guided scraping to accomplish a goal on a page
-- When you need to monitor a URL for changes over time
-
-## API base
-
-All requests go to `https://api.webclaw.io/v1/`.
-
-Authentication: `Authorization: Bearer $WEBCLAW_API_KEY`
-
-## Endpoints
-
-### 1. Scrape — extract content from a single URL
-
-```bash
-curl -X POST https://api.webclaw.io/v1/scrape \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "url": "https://example.com",
-    "formats": ["markdown"],
-    "only_main_content": true
-  }'
-```
-
-**Request fields:**
-
-| Field | Type | Default | Description |
-|-------|------|---------|-------------|
-| `url` | string | required | URL to scrape |
-| `formats` | string[] | `["markdown"]` | Output formats: `markdown`, `text`, `llm`, `json` |
-| `include_selectors` | string[] | `[]` | CSS selectors to keep (e.g. `["article", ".content"]`) |
-| `exclude_selectors` | string[] | `[]` | CSS selectors to remove (e.g. `["nav", "footer", ".ads"]`) |
-| `only_main_content` | bool | `false` | Extract only the main article/content area |
-| `no_cache` | bool | `false` | Skip cache, fetch fresh |
-| `max_cache_age` | int | server default | Max acceptable cache age in seconds |
-
-**Response:**
-
-```json
-{
-  "url": "https://example.com",
-  "metadata": {
-    "title": "Example",
-    "description": "...",
-    "language": "en",
-    "word_count": 1234
-  },
-  "markdown": "# Page Title\n\nContent here...",
-  "cache": { "status": "miss" }
-}
-```
-
-**Format options:**
-- `markdown` — clean markdown, best for general use
-- `text` — plain text without formatting
-- `llm` — optimized for LLM consumption: includes page title, URL, and cleaned content with link references. Best for feeding to AI models.
-- `json` — full extraction result with all metadata
-
-**When antibot bypass activates** (automatic, no extra config):
-```json
-{
-  "antibot": {
-    "bypass": true,
-    "elapsed_ms": 3200
-  }
-}
-```
-
-### 2. Crawl — scrape an entire website
-
-Starts an async job. Poll for results.
-
-**Start crawl:**
-```bash
-curl -X POST https://api.webclaw.io/v1/crawl \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "url": "https://docs.example.com",
-    "max_depth": 3,
-    "max_pages": 50,
-    "use_sitemap": true
-  }'
-```
-
-Response: `{ "job_id": "abc-123", "status": "running" }`
-
-**Poll status:**
-```bash
-curl https://api.webclaw.io/v1/crawl/abc-123 \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY"
-```
-
-Response when complete:
-```json
-{
-  "job_id": "abc-123",
-  "status": "completed",
-  "total": 47,
-  "completed": 45,
-  "errors": 2,
-  "pages": [
-    {
-      "url": "https://docs.example.com/intro",
-      "markdown": "# Introduction\n...",
-      "metadata": { "title": "Intro", "word_count": 500 }
-    }
-  ]
-}
-```
-
-**Request fields:**
-
-| Field | Type | Default | Description |
-|-------|------|---------|-------------|
-| `url` | string | required | Starting URL |
-| `max_depth` | int | `3` | How many links deep to follow |
-| `max_pages` | int | `100` | Maximum pages to crawl |
-| `use_sitemap` | bool | `false` | Seed URLs from sitemap.xml |
-| `formats` | string[] | `["markdown"]` | Output formats per page |
-| `include_selectors` | string[] | `[]` | CSS selectors to keep |
-| `exclude_selectors` | string[] | `[]` | CSS selectors to remove |
-| `only_main_content` | bool | `false` | Main content only |
-
-### 3. Map — discover all URLs on a site
-
-Fast URL discovery without full content extraction.
-
-```bash
-curl -X POST https://api.webclaw.io/v1/map \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY" \
-  -H "Content-Type: application/json" \
-  -d '{"url": "https://example.com"}'
-```
-
-Response:
-```json
-{
-  "url": "https://example.com",
-  "count": 142,
-  "urls": [
-    "https://example.com/about",
-    "https://example.com/pricing",
-    "https://example.com/docs/intro"
-  ]
-}
-```
-
-### 4. Batch — scrape multiple URLs in parallel
-
-```bash
-curl -X POST https://api.webclaw.io/v1/batch \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "urls": [
-      "https://a.com",
-      "https://b.com",
-      "https://c.com"
-    ],
-    "formats": ["markdown"],
-    "concurrency": 5
-  }'
-```
-
-Response:
-```json
-{
-  "total": 3,
-  "completed": 3,
-  "errors": 0,
-  "results": [
-    { "url": "https://a.com", "markdown": "...", "metadata": {} },
-    { "url": "https://b.com", "markdown": "...", "metadata": {} },
-    { "url": "https://c.com", "error": "timeout" }
-  ]
-}
-```
-
-### 5. Extract — LLM-powered structured extraction
-
-Pull structured data from any page using a JSON schema or plain-text prompt.
-
-**With JSON schema:**
-```bash
-curl -X POST https://api.webclaw.io/v1/extract \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "url": "https://example.com/pricing",
-    "schema": {
-      "type": "object",
-      "properties": {
-        "plans": {
-          "type": "array",
-          "items": {
-            "type": "object",
-            "properties": {
-              "name": { "type": "string" },
-              "price": { "type": "string" },
-              "features": { "type": "array", "items": { "type": "string" } }
-            }
-          }
-        }
-      }
-    }
-  }'
-```
-
-**With prompt:**
-```bash
-curl -X POST https://api.webclaw.io/v1/extract \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "url": "https://example.com/pricing",
-    "prompt": "Extract all pricing tiers with names, monthly prices, and key features"
-  }'
-```
-
-Response:
-```json
-{
-  "url": "https://example.com/pricing",
-  "data": {
-    "plans": [
-      { "name": "Starter", "price": "$49/mo", "features": ["10k pages", "Email support"] },
-      { "name": "Pro", "price": "$99/mo", "features": ["100k pages", "Priority support", "API access"] }
-    ]
-  }
-}
-```
-
-### 6. Summarize — get a quick summary of any page
-
-```bash
-curl -X POST https://api.webclaw.io/v1/summarize \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "url": "https://example.com/long-article",
-    "max_sentences": 3
-  }'
-```
-
-Response:
-```json
-{
-  "url": "https://example.com/long-article",
-  "summary": "The article discusses... Key findings include... The author concludes that..."
-}
-```
-
-### 7. Diff — detect content changes
-
-Compare current page content against a previous snapshot.
-
-```bash
-curl -X POST https://api.webclaw.io/v1/diff \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "url": "https://example.com",
-    "previous": {
-      "markdown": "# Old content...",
-      "metadata": { "title": "Old Title" }
-    }
-  }'
-```
-
-Response:
-```json
-{
-  "url": "https://example.com",
-  "status": "changed",
-  "diff": "--- previous\n+++ current\n@@ -1 +1 @@\n-# Old content\n+# New content",
-  "metadata_changes": [
-    { "field": "title", "old": "Old Title", "new": "New Title" }
-  ]
-}
-```
-
-### 8. Brand — extract brand identity
-
-Analyze a website's visual identity: colors, fonts, logo.
-
-```bash
-curl -X POST https://api.webclaw.io/v1/brand \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY" \
-  -H "Content-Type: application/json" \
-  -d '{"url": "https://example.com"}'
-```
-
-Response:
-```json
-{
-  "url": "https://example.com",
-  "brand": {
-    "colors": [
-      { "hex": "#FF6B35", "usage": "primary" },
-      { "hex": "#1A1A2E", "usage": "background" }
-    ],
-    "fonts": ["Inter", "JetBrains Mono"],
-    "logo_url": "https://example.com/logo.svg",
-    "favicon_url": "https://example.com/favicon.ico"
-  }
-}
-```
-
-### 9. Search — web search with optional scraping
-
-Search the web and optionally scrape each result page.
-
-```bash
-curl -X POST https://api.webclaw.io/v1/search \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "query": "best rust web frameworks 2026",
-    "num_results": 5,
-    "scrape": true,
-    "formats": ["markdown"]
-  }'
-```
-
-**Request fields:**
-
-| Field | Type | Default | Description |
-|-------|------|---------|-------------|
-| `query` | string | required | Search query |
-| `num_results` | int | `10` | Number of search results to return |
-| `scrape` | bool | `false` | Also scrape each result page for full content |
-| `formats` | string[] | `["markdown"]` | Output formats when `scrape` is true |
-| `country` | string | none | Country code for localized results (e.g. `"us"`, `"de"`) |
-| `lang` | string | none | Language code for results (e.g. `"en"`, `"fr"`) |
-
-**Response:**
-
-```json
-{
-  "query": "best rust web frameworks 2026",
-  "results": [
-    {
-      "title": "Top Rust Web Frameworks in 2026",
-      "url": "https://blog.example.com/rust-frameworks",
-      "snippet": "A comprehensive comparison of Axum, Actix, and Rocket...",
-      "position": 1,
-      "markdown": "# Top Rust Web Frameworks\n\n..."
-    },
-    {
-      "title": "Choosing a Rust Backend Framework",
-      "url": "https://dev.to/rust-backends",
-      "snippet": "When starting a new Rust web project...",
-      "position": 2,
-      "markdown": "# Choosing a Rust Backend\n\n..."
-    }
-  ]
-}
-```
-
-The `markdown` field on each result is only present when `scrape: true`. Without it, you get titles, URLs, snippets, and positions only.
-
-### 10. Research — deep multi-source research
-
-Starts an async research job that searches, scrapes, and synthesizes information across multiple sources. Poll for results.
-
-**Start research:**
-```bash
-curl -X POST https://api.webclaw.io/v1/research \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "query": "How does Cloudflare Turnstile work and what are its known bypass methods?",
-    "max_iterations": 5,
-    "max_sources": 10,
-    "topic": "security",
-    "deep": true
-  }'
-```
-
-**Request fields:**
-
-| Field | Type | Default | Description |
-|-------|------|---------|-------------|
-| `query` | string | required | Research question or topic |
-| `max_iterations` | int | server default | Maximum research iterations (search-read-analyze cycles) |
-| `max_sources` | int | server default | Maximum number of sources to consult |
-| `topic` | string | none | Topic hint to guide search strategy (e.g. `"security"`, `"finance"`, `"engineering"`) |
-| `deep` | bool | `false` | Enable deep research mode for more thorough analysis (costs 10 credits instead of 1) |
-
-Response: `{ "id": "res-abc-123", "status": "running" }`
-
-**Poll results:**
-```bash
-curl https://api.webclaw.io/v1/research/res-abc-123 \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY"
-```
-
-Response when complete:
-```json
-{
-  "id": "res-abc-123",
-  "status": "completed",
-  "query": "How does Cloudflare Turnstile work and what are its known bypass methods?",
-  "report": "# Cloudflare Turnstile Analysis\n\n## Overview\nCloudflare Turnstile is a CAPTCHA replacement that...\n\n## How It Works\n...\n\n## Known Bypass Methods\n...",
-  "sources": [
-    { "url": "https://developers.cloudflare.com/turnstile/", "title": "Turnstile Documentation" },
-    { "url": "https://blog.cloudflare.com/turnstile-ga/", "title": "Turnstile GA Announcement" }
-  ],
-  "findings": [
-    "Turnstile uses browser environment signals and proof-of-work challenges",
-    "Managed mode auto-selects challenge difficulty based on visitor risk score",
-    "Known bypass approaches include instrumented browser automation"
-  ],
-  "iterations": 5,
-  "elapsed_ms": 34200
-}
-```
-
-**Status values:** `running`, `completed`, `failed`
-
-### 11. Agent Scrape — AI-guided scraping
-
-Use an AI agent to navigate and interact with a page to accomplish a specific goal. The agent can click, scroll, fill forms, and extract data across multiple steps.
-
-```bash
-curl -X POST https://api.webclaw.io/v1/agent-scrape \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "url": "https://example.com/products",
-    "goal": "Find the cheapest laptop with at least 16GB RAM and extract its full specs",
-    "max_steps": 10
-  }'
-```
-
-**Request fields:**
-
-| Field | Type | Default | Description |
-|-------|------|---------|-------------|
-| `url` | string | required | Starting URL |
-| `goal` | string | required | What the agent should accomplish |
-| `max_steps` | int | server default | Maximum number of actions the agent can take |
-
-**Response:**
-
-```json
-{
-  "url": "https://example.com/products",
-  "result": "The cheapest laptop with 16GB+ RAM is the ThinkPad E14 Gen 6 at $649. Specs: AMD Ryzen 5 7535U, 16GB DDR4, 512GB SSD, 14\" FHD IPS display, 57Wh battery.",
-  "steps": [
-    { "action": "navigate", "detail": "Loaded products page" },
-    { "action": "click", "detail": "Clicked 'Laptops' category filter" },
-    { "action": "click", "detail": "Applied '16GB+' RAM filter" },
-    { "action": "click", "detail": "Sorted by price: low to high" },
-    { "action": "extract", "detail": "Extracted specs from first matching product" }
-  ]
-}
-```
-
-### 12. Watch — monitor a URL for changes
-
-Create persistent monitors that check a URL on a schedule and notify via webhook when content changes.
-
-**Create a monitor:**
-```bash
-curl -X POST https://api.webclaw.io/v1/watch \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "url": "https://example.com/pricing",
-    "interval": "0 */6 * * *",
-    "webhook_url": "https://hooks.example.com/pricing-changed",
-    "formats": ["markdown"]
-  }'
-```
-
-**Request fields:**
-
-| Field | Type | Default | Description |
-|-------|------|---------|-------------|
-| `url` | string | required | URL to monitor |
-| `interval` | string | required | Check frequency as cron expression or seconds (e.g. `"0 */6 * * *"` or `"3600"`) |
-| `webhook_url` | string | none | URL to POST when changes are detected |
-| `formats` | string[] | `["markdown"]` | Output formats for snapshots |
-
-Response:
-```json
-{
-  "id": "watch-abc-123",
-  "url": "https://example.com/pricing",
-  "interval": "0 */6 * * *",
-  "webhook_url": "https://hooks.example.com/pricing-changed",
-  "formats": ["markdown"],
-  "created_at": "2026-03-20T10:00:00Z",
-  "last_check": null,
-  "status": "active"
-}
-```
-
-**List all monitors:**
-```bash
-curl https://api.webclaw.io/v1/watch \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY"
-```
-
-Response:
-```json
-{
-  "monitors": [
-    {
-      "id": "watch-abc-123",
-      "url": "https://example.com/pricing",
-      "interval": "0 */6 * * *",
-      "status": "active",
-      "last_check": "2026-03-20T16:00:00Z",
-      "checks": 4
-    }
-  ]
-}
-```
-
-**Get a monitor with snapshots:**
-```bash
-curl https://api.webclaw.io/v1/watch/watch-abc-123 \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY"
-```
-
-Response:
-```json
-{
-  "id": "watch-abc-123",
-  "url": "https://example.com/pricing",
-  "interval": "0 */6 * * *",
-  "status": "active",
-  "snapshots": [
-    {
-      "checked_at": "2026-03-20T16:00:00Z",
-      "status": "changed",
-      "diff": "--- previous\n+++ current\n@@ -5 +5 @@\n-Pro: $99/mo\n+Pro: $119/mo"
-    },
-    {
-      "checked_at": "2026-03-20T10:00:00Z",
-      "status": "baseline"
-    }
-  ]
-}
-```
-
-**Trigger an immediate check:**
-```bash
-curl -X POST https://api.webclaw.io/v1/watch/watch-abc-123/check \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY"
-```
-
-**Delete a monitor:**
-```bash
-curl -X DELETE https://api.webclaw.io/v1/watch/watch-abc-123 \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY"
-```
-
-## Choosing the right format
-
-| Goal | Format | Why |
-|------|--------|-----|
-| Read and understand a page | `markdown` | Clean structure, headings, links preserved |
-| Feed content to an AI model | `llm` | Optimized: includes title + URL header, clean link refs |
-| Search or index content | `text` | Plain text, no formatting noise |
-| Programmatic analysis | `json` | Full metadata, structured data, DOM statistics |
-
-## Tips
-
-- **Use `llm` format** when passing content to yourself or another AI — it's specifically optimized for LLM consumption with better context framing.
-- **Use `only_main_content: true`** to skip navigation, sidebars, and footers. Reduces noise significantly.
-- **Use `include_selectors`/`exclude_selectors`** for fine-grained control when `only_main_content` isn't enough.
-- **Batch over individual scrapes** when fetching multiple URLs — it's faster and more efficient.
-- **Use `map` before `crawl`** to discover the site structure first, then crawl specific sections.
-- **Use `extract` with a JSON schema** for reliable structured output (e.g., pricing tables, product specs, contact info).
-- **Antibot bypass is automatic** — no extra configuration needed. Works on Cloudflare, DataDome, AWS WAF, and JS-rendered SPAs.
-- **Use `search` with `scrape: true`** to get full page content for each search result in one call instead of searching then scraping separately.
-- **Use `research` for complex questions** that need multiple sources — it handles the search-read-synthesize loop automatically. Enable `deep: true` for thorough analysis.
-- **Use `agent-scrape` for interactive pages** where data is behind filters, pagination, or form submissions that a simple scrape cannot reach.
-- **Use `watch` for ongoing monitoring** — set up a cron schedule and a webhook to get notified when a page changes without polling manually.
-
-## Smart Fetch Architecture
-
-The webclaw MCP server uses a **local-first** approach:
-
-1. **Local fetch** — fast, free, no API credits used (~80% of sites)
-2. **Cloud API fallback** — automatic when bot protection or JS rendering is detected
-
-This means:
-- Most scrapes cost zero credits (local extraction)
-- Cloudflare, DataDome, AWS WAF sites automatically fall back to the cloud API
-- JS-rendered SPAs (React, Next.js, Vue) also fall back automatically
-- Set `WEBCLAW_API_KEY` to enable cloud fallback
-
-## vs web_fetch
-
-| | webclaw | web_fetch |
-|---|---------|-----------|
-| Cloudflare bypass | Automatic (cloud fallback) | Fails (403) |
-| JS-rendered pages | Automatic fallback | Readability only |
-| Output quality | 20-step optimization pipeline | Basic HTML parsing |
-| Structured extraction | LLM-powered, schema-based | None |
-| Crawling | Full site crawl with sitemap | Single page only |
-| Caching | Built-in, configurable TTL | Per-session |
-| Rate limiting | Managed server-side | Client responsibility |
-
-Use `web_fetch` for simple, fast lookups. Use webclaw when you need reliability, quality, or advanced features.

crates/webclaw-cli/src/main.rs

@@ -183,11 +183,11 @@ struct Cli {
     browser: Browser,
 
     /// Proxy URL (http://user:pass@host:port or socks5://host:port)
-    #[arg(short, long, env = "WEBCLAW_PROXY")]
+    #[arg(short, long, env = "WEBCLAW_PROXY", hide_env_values = true)]
     proxy: Option<String>,
 
     /// File with proxies (host:port:user:pass, one per line). Rotates per request.
-    #[arg(long, env = "WEBCLAW_PROXY_FILE")]
+    #[arg(long, env = "WEBCLAW_PROXY_FILE", hide_env_values = true)]
     proxy_file: Option<String>,
 
     /// Request timeout in seconds
@@ -256,7 +256,7 @@ struct Cli {
 
     /// Webhook URL: POST a JSON payload when an operation completes.
     /// Works with crawl, batch, watch (on change), and single URL modes.
-    #[arg(long, env = "WEBCLAW_WEBHOOK_URL")]
+    #[arg(long, env = "WEBCLAW_WEBHOOK_URL", hide_env_values = true)]
     webhook: Option<String>,
 
     /// Extract brand identity (colors, fonts, logo)
@@ -313,6 +313,18 @@ struct Cli {
     #[arg(long)]
     map: bool,
 
+    /// Max pages for --map's crawl fallback when the sitemap is thin [default: 150]
+    #[arg(long)]
+    map_pages: Option<usize>,
+
+    /// Disable --map's crawl fallback (sitemap-only discovery)
+    #[arg(long)]
+    no_map_crawl: bool,
+
+    /// Cap the number of URLs --map returns (default: uncapped)
+    #[arg(long)]
+    map_limit: Option<usize>,
+
     // -- LLM options --
     /// Extract structured JSON using LLM (pass a JSON schema string or @file)
     #[arg(long)]
@@ -326,7 +338,7 @@ struct Cli {
     #[arg(long, num_args = 0..=1, default_missing_value = "3")]
     summarize: Option<usize>,
 
-    /// Force a specific LLM provider (ollama, openai, anthropic)
+    /// Force a specific LLM provider (ollama, openai, atlascloud, anthropic)
     #[arg(long, env = "WEBCLAW_LLM_PROVIDER")]
     llm_provider: Option<String>,
 
@@ -335,12 +347,12 @@ struct Cli {
     llm_model: Option<String>,
 
     /// Override the LLM base URL (Ollama, OpenAI-compatible, or Anthropic-compatible)
-    #[arg(long, env = "WEBCLAW_LLM_BASE_URL")]
+    #[arg(long, env = "WEBCLAW_LLM_BASE_URL", hide_env_values = true)]
     llm_base_url: Option<String>,
 
     // -- Cloud API options --
     /// Webclaw Cloud API key for automatic fallback on bot-protected or JS-rendered sites
-    #[arg(long, env = "WEBCLAW_API_KEY")]
+    #[arg(long, env = "WEBCLAW_API_KEY", hide_env_values = true)]
     api_key: Option<String>,
 
     /// Force all requests through the cloud API (skip local extraction)
@@ -410,6 +422,43 @@ enum Commands {
         #[arg(long)]
         raw: bool,
     },
+
+    /// Web search via Serper.dev using YOUR OWN API key.
+    ///
+    /// Returns Google organic results (title, link, snippet). With
+    /// `--scrape`, each result page is fetched and extracted to markdown.
+    /// Get a free key at serper.dev, then pass `--serper-key` or set
+    /// `SERPER_API_KEY`.
+    ///
+    /// Example: `webclaw search "rust async runtime" --num 5 --scrape`.
+    Search {
+        /// Search query.
+        query: String,
+
+        /// Serper.dev API key. Falls back to the `SERPER_API_KEY` env var.
+        #[arg(long, env = "SERPER_API_KEY", hide_env_values = true)]
+        serper_key: Option<String>,
+
+        /// Number of results to return (1-10).
+        #[arg(long, default_value = "5")]
+        num: usize,
+
+        /// Country code for localization (e.g. "us", "gb", "it").
+        #[arg(long)]
+        country: Option<String>,
+
+        /// Language code for localization (e.g. "en", "it").
+        #[arg(long)]
+        lang: Option<String>,
+
+        /// Fetch + extract each result page and include its markdown.
+        #[arg(long)]
+        scrape: bool,
+
+        /// Output format: `markdown` (human-readable, default) or `json`.
+        #[arg(short, long, default_value = "markdown")]
+        format: OutputFormat,
+    },
 }
 
 #[derive(Clone, ValueEnum)]
@@ -471,7 +520,13 @@ fn init_logging(verbose: bool) {
         EnvFilter::try_from_env("WEBCLAW_LOG").unwrap_or_else(|_| EnvFilter::new(default))
     };
 
-    tracing_subscriber::fmt().with_env_filter(filter).init();
+    // Logs go to stderr, never stdout: stdout carries the actual result
+    // (markdown / JSON / URL list). A stray WARN on stdout corrupts
+    // machine-readable output — e.g. `--map --format json` piped to a parser.
+    tracing_subscriber::fmt()
+        .with_env_filter(filter)
+        .with_writer(std::io::stderr)
+        .init();
 }
 
 /// Build FetchConfig from CLI flags.
@@ -809,6 +864,18 @@ fn collect_urls(cli: &Cli) -> Result<Vec<(String, Option<String>)>, String> {
     Ok(entries)
 }
 
+/// When `--urls-file` supplies exactly one URL and no positional URL was given,
+/// treat it like a single positional URL so a one-line file behaves identically
+/// to `webclaw <url>` (stdout, `--output-dir`, cloud fallback, `--raw-html`,
+/// `--diff`, LLM paths). Without this the lone file URL gives `entries.len() == 1`,
+/// which the batch gates skip, and the single path then finds an empty `cli.urls`
+/// and errors "no input provided". See issue #86.
+fn backfill_single_file_url(urls: &mut Vec<String>, entries: &[(String, Option<String>)]) {
+    if urls.is_empty() && entries.len() == 1 {
+        urls.push(entries[0].0.clone());
+    }
+}
+
 /// Result that can be either a local extraction or a cloud API JSON response.
 enum FetchOutput {
     Local(Box<ExtractionResult>),
@@ -1548,7 +1615,7 @@ async fn run_crawl(cli: &Cli) -> Result<(), String> {
     // Fire webhook on crawl complete
     if let Some(ref webhook_url) = cli.webhook {
         let urls: Vec<&str> = result.pages.iter().map(|p| p.url.as_str()).collect();
-        fire_webhook(
+        let handle = fire_webhook(
             webhook_url,
             &serde_json::json!({
                 "event": "crawl_complete",
@@ -1559,8 +1626,8 @@ async fn run_crawl(cli: &Cli) -> Result<(), String> {
                 "urls": urls,
             }),
         );
-        // Brief pause so the async webhook has time to fire
-        tokio::time::sleep(std::time::Duration::from_millis(500)).await;
+        // Wait for the webhook to finish so the process doesn't exit mid-send.
+        let _ = handle.await;
     }
 
     if result.errors > 0 {
@@ -1573,6 +1640,73 @@ async fn run_crawl(cli: &Cli) -> Result<(), String> {
     }
 }
 
+/// Web search via Serper.dev with the caller's own API key.
+///
+/// The Serper key is resolved by the caller (flag or `SERPER_API_KEY`
+/// env, via clap's `env`) and passed in already-unwrapped. When `scrape`
+/// is set, each result page is fetched + extracted through a FetchClient
+/// (which carries the browser TLS profile) and its markdown is included.
+#[allow(clippy::too_many_arguments)]
+async fn run_search(
+    serper_key: &str,
+    query: &str,
+    num: usize,
+    country: Option<&str>,
+    lang: Option<&str>,
+    scrape: bool,
+    format: &OutputFormat,
+) -> Result<(), String> {
+    // Default fetch config is enough: search localization is handled by
+    // Serper's gl/hl, and the result-page scrape just needs a standard
+    // browser profile. Attach cloud fallback when WEBCLAW_API_KEY is set
+    // so scraped pages behind bot protection can still escalate.
+    let mut client = webclaw_fetch::FetchClient::new(webclaw_fetch::FetchConfig::default())
+        .map_err(|e| format!("client error: {e}"))?;
+    if let Some(cloud) = webclaw_fetch::cloud::CloudClient::from_env() {
+        client = client.with_cloud(cloud);
+    }
+
+    let opts = webclaw_fetch::SearchOptions {
+        num_results: num,
+        country: country.map(str::to_string),
+        lang: lang.map(str::to_string),
+        scrape,
+    };
+
+    let results = webclaw_fetch::search(&client, serper_key, query, &opts)
+        .await
+        .map_err(|e| format!("search error: {e}"))?;
+
+    if matches!(format, OutputFormat::Json) {
+        let json = serde_json::json!({ "query": query, "results": results });
+        match serde_json::to_string_pretty(&json) {
+            Ok(s) => println!("{s}"),
+            Err(e) => return Err(format!("JSON encode failed: {e}")),
+        }
+        return Ok(());
+    }
+
+    if results.is_empty() {
+        eprintln!("no results for \"{query}\"");
+        return Ok(());
+    }
+
+    for r in &results {
+        println!("{}. {}", r.position, r.title);
+        println!("   {}", r.link);
+        if !r.snippet.is_empty() {
+            println!("   {}", r.snippet);
+        }
+        if let Some(ref content) = r.content {
+            println!();
+            println!("{content}");
+        }
+        println!();
+    }
+
+    Ok(())
+}
+
 async fn run_map(cli: &Cli) -> Result<(), String> {
     let url = cli
         .urls
@@ -1584,12 +1718,22 @@ async fn run_map(cli: &Cli) -> Result<(), String> {
     let client =
         FetchClient::new(build_fetch_config(cli)).map_err(|e| format!("client error: {e}"))?;
 
-    let entries = webclaw_fetch::sitemap::discover(&client, url)
-        .await
-        .map_err(|e| format!("sitemap discovery failed: {e}"))?;
+    // Layered discovery: sitemaps first, bounded crawl fallback when thin.
+    let mut opts = webclaw_fetch::MapOptions::default();
+    if let Some(pages) = cli.map_pages {
+        opts.max_crawl_pages = pages;
+    }
+    if cli.no_map_crawl {
+        opts.crawl_fallback = false;
+    }
+    if let Some(limit) = cli.map_limit {
+        opts.max_urls = Some(limit);
+    }
+
+    let entries = webclaw_fetch::discover_urls(&client, url, &opts).await;
 
     if entries.is_empty() {
-        eprintln!("no sitemap URLs found for {url}");
+        eprintln!("no URLs found for {url}");
     } else {
         eprintln!("discovered {} URLs", entries.len());
     }
@@ -1658,7 +1802,7 @@ async fn run_batch(cli: &Cli, entries: &[(String, Option<String>)]) -> Result<()
     // Fire webhook on batch complete
     if let Some(ref webhook_url) = cli.webhook {
         let urls: Vec<&str> = results.iter().map(|r| r.url.as_str()).collect();
-        fire_webhook(
+        let handle = fire_webhook(
             webhook_url,
             &serde_json::json!({
                 "event": "batch_complete",
@@ -1668,7 +1812,7 @@ async fn run_batch(cli: &Cli, entries: &[(String, Option<String>)]) -> Result<()
                 "urls": urls,
             }),
         );
-        tokio::time::sleep(std::time::Duration::from_millis(500)).await;
+        let _ = handle.await;
     }
 
     if errors > 0 {
@@ -1742,9 +1886,12 @@ async fn spawn_on_change(cmd: &str, stdin_payload: &[u8]) {
     }
 }
 
-/// Fire a webhook POST with a JSON payload. Non-blocking — errors logged to stderr.
-/// Auto-detects Discord and Slack webhook URLs and wraps the payload accordingly.
-fn fire_webhook(url: &str, payload: &serde_json::Value) {
+/// Fire a webhook POST with a JSON payload. Spawns the send on a background task
+/// and returns its `JoinHandle` so callers that need delivery (e.g. one-shot
+/// crawl/batch runs that exit immediately after) can `.await` it; long-running
+/// loops can drop the handle and let it run fire-and-forget. Errors are logged
+/// to stderr. Auto-detects Discord and Slack webhook URLs and wraps the payload.
+fn fire_webhook(url: &str, payload: &serde_json::Value) -> tokio::task::JoinHandle<()> {
     let url = url.to_string();
     let is_discord = url.contains("discord.com/api/webhooks");
     let is_slack = url.contains("hooks.slack.com");
@@ -1806,7 +1953,7 @@ fn fire_webhook(url: &str, payload: &serde_json::Value) {
             },
             Err(e) => eprintln!("[webhook] client error: {e}"),
         }
-    });
+    })
 }
 
 async fn run_watch(cli: &Cli, urls: &[String]) -> Result<(), String> {
@@ -2104,6 +2251,15 @@ async fn build_llm_provider(cli: &Cli) -> Result<Box<dyn LlmProvider>, String> {
                 .ok_or("OPENAI_API_KEY not set")?;
                 Ok(Box::new(provider))
             }
+            "atlascloud" => {
+                let provider = webclaw_llm::providers::atlascloud::AtlasCloudProvider::new(
+                    None,
+                    cli.llm_base_url.clone(),
+                    cli.llm_model.clone(),
+                )
+                .ok_or("ATLASCLOUD_API_KEY not set")?;
+                Ok(Box::new(provider))
+            }
             "anthropic" => {
                 let provider = webclaw_llm::providers::anthropic::AnthropicProvider::with_base_url(
                     None,
@@ -2114,7 +2270,7 @@ async fn build_llm_provider(cli: &Cli) -> Result<Box<dyn LlmProvider>, String> {
                 Ok(Box::new(provider))
             }
             other => Err(format!(
-                "unknown LLM provider: {other} (use ollama, openai, or anthropic)"
+                "unknown LLM provider: {other} (use ollama, openai, atlascloud, or anthropic)"
             )),
         }
     } else {
@@ -2318,7 +2474,7 @@ async fn run_batch_llm(cli: &Cli, entries: &[(String, Option<String>)]) -> Resul
     eprintln!("Processed {total} URLs ({ok} ok, {errors} errors)");
 
     if let Some(ref webhook_url) = cli.webhook {
-        fire_webhook(
+        let handle = fire_webhook(
             webhook_url,
             &serde_json::json!({
                 "event": "batch_llm_complete",
@@ -2327,7 +2483,7 @@ async fn run_batch_llm(cli: &Cli, entries: &[(String, Option<String>)]) -> Resul
                 "errors": errors,
             }),
         );
-        tokio::time::sleep(std::time::Duration::from_millis(500)).await;
+        let _ = handle.await;
     }
 
     if errors > 0 {
@@ -2489,7 +2645,7 @@ async fn run_research(cli: &Cli, query: &str) -> Result<(), String> {
 async fn main() {
     dotenvy::dotenv().ok();
 
-    let cli = Cli::parse();
+    let mut cli = Cli::parse();
     init_logging(cli.verbose);
 
     // Subcommand path. Handled before the flag dispatch so a subcommand
@@ -2586,6 +2742,40 @@ async fn main() {
                 }
                 return;
             }
+            Commands::Search {
+                query,
+                serper_key,
+                num,
+                country,
+                lang,
+                scrape,
+                format,
+            } => {
+                let key = match serper_key {
+                    Some(k) if !k.trim().is_empty() => k.clone(),
+                    _ => {
+                        eprintln!(
+                            "error: search requires a Serper.dev API key: pass --serper-key or set SERPER_API_KEY (get one free at serper.dev)"
+                        );
+                        process::exit(1);
+                    }
+                };
+                if let Err(e) = run_search(
+                    &key,
+                    query,
+                    *num,
+                    country.as_deref(),
+                    lang.as_deref(),
+                    *scrape,
+                    format,
+                )
+                .await
+                {
+                    eprintln!("error: {e}");
+                    process::exit(1);
+                }
+                return;
+            }
         }
     }
 
@@ -2598,6 +2788,20 @@ async fn main() {
         return;
     }
 
+    // Collect URLs from args + --urls-file up front, and backfill a lone
+    // --urls-file URL into cli.urls so a one-line file behaves like a positional
+    // URL across ALL modes below (crawl / watch / diff / brand / batch / single).
+    // See issue #86 — the mode gates and single path read cli.urls, which is
+    // empty when the URL came only from --urls-file.
+    let entries = match collect_urls(&cli) {
+        Ok(u) => u,
+        Err(e) => {
+            eprintln!("error: {e}");
+            process::exit(1);
+        }
+    };
+    backfill_single_file_url(&mut cli.urls, &entries);
+
     // --crawl: recursive crawl mode
     if cli.crawl {
         if let Err(e) = run_crawl(&cli).await {
@@ -2609,13 +2813,7 @@ async fn main() {
 
     // --watch: poll URL(s) for changes
     if cli.watch {
-        let watch_urls: Vec<String> = match collect_urls(&cli) {
-            Ok(entries) => entries.into_iter().map(|(url, _)| url).collect(),
-            Err(e) => {
-                eprintln!("error: {e}");
-                process::exit(1);
-            }
-        };
+        let watch_urls: Vec<String> = entries.iter().map(|(url, _)| url.clone()).collect();
         if let Err(e) = run_watch(&cli, &watch_urls).await {
             eprintln!("error: {e}");
             process::exit(1);
@@ -2650,15 +2848,6 @@ async fn main() {
         return;
     }
 
-    // Collect all URLs from args + --urls-file
-    let entries = match collect_urls(&cli) {
-        Ok(u) => u,
-        Err(e) => {
-            eprintln!("error: {e}");
-            process::exit(1);
-        }
-    };
-
     // LLM modes: --extract-json, --extract-prompt, --summarize
     // When multiple URLs are provided, run batch LLM extraction over all of them.
     if has_llm_flags(&cli) {
@@ -2730,6 +2919,36 @@ mod tests {
     use super::*;
     use webclaw_core::Content;
 
+    // issue #86: a single URL sourced only from --urls-file must be promoted to
+    // a positional URL so it takes the single-scrape path (batch gates need >1).
+    #[test]
+    fn single_file_url_backfilled_into_positional() {
+        let mut urls: Vec<String> = Vec::new();
+        let entries = vec![("https://example.com".to_string(), None)];
+        backfill_single_file_url(&mut urls, &entries);
+        assert_eq!(urls, vec!["https://example.com".to_string()]);
+    }
+
+    #[test]
+    fn positional_urls_left_untouched() {
+        let mut urls = vec!["https://a.com".to_string()];
+        let entries = vec![("https://a.com".to_string(), None)];
+        backfill_single_file_url(&mut urls, &entries);
+        assert_eq!(urls, vec!["https://a.com".to_string()]);
+    }
+
+    #[test]
+    fn multiple_file_urls_not_backfilled() {
+        // 2+ URLs already route to run_batch, so cli.urls stays empty here.
+        let mut urls: Vec<String> = Vec::new();
+        let entries = vec![
+            ("https://a.com".to_string(), None),
+            ("https://b.com".to_string(), None),
+        ];
+        backfill_single_file_url(&mut urls, &entries);
+        assert!(urls.is_empty());
+    }
+
     fn empty_result(title: Option<&str>, url: Option<&str>, markdown: &str) -> ExtractionResult {
         ExtractionResult {
             metadata: Metadata {

crates/webclaw-core/src/endpoints.rs

@@ -233,7 +233,13 @@ pub fn extract_endpoints(
         }
         let slice = if text.len() > *budget {
             *truncated = true;
-            &text[..*budget]
+            // Snap the cut to a UTF-8 char boundary so non-ASCII content
+            // (multibyte codepoints straddling the budget) can't panic.
+            let mut cut = (*budget).min(text.len());
+            while cut > 0 && !text.is_char_boundary(cut) {
+                cut -= 1;
+            }
+            &text[..cut]
         } else {
             text
         };
@@ -512,4 +518,16 @@ mod tests {
         );
         assert!(r.hosts.iter().any(|h| h == "pubapi.ticketmaster.co.uk"));
     }
+
+    #[test]
+    fn scan_truncation_at_non_ascii_boundary_does_not_panic() {
+        // A bundle just over the scan budget, padded with a multibyte char
+        // ('é' is 2 bytes) so the cut lands mid-codepoint. The old
+        // `&text[..budget]` slice panicked here; the boundary snap must not.
+        let pad = "é".repeat(MAX_SCAN_BYTES); // ~2× budget in bytes
+        let bundle = format!("{pad} fetch(\"/api/x\")");
+        let bundles = vec![("big.js".to_string(), bundle)];
+        let r = extract_endpoints("<html></html>", "https://example.com/", &bundles);
+        assert!(r.truncated, "oversized bundle should mark truncated");
+    }
 }

crates/webclaw-core/src/js_eval.rs

@@ -16,6 +16,29 @@ static SCRIPT_SELECTOR: Lazy<Selector> = Lazy::new(|| Selector::parse("script").
 static HTML_TAG_RE: Lazy<Regex> = Lazy::new(|| Regex::new(r"<[^>]+>").unwrap());
 const JS_EVAL_TIMEOUT: Duration = Duration::from_millis(250);
 
+/// Markers that, if absent from the HTML, prove the QuickJS scan cannot find
+/// any data blob. The scan only ever surfaces `globalThis.__*` object/array
+/// properties, and the seeded `__next_f` only emits when non-empty. Every
+/// realistic way an inline script populates such a global goes through one of
+/// these substrings (`window.`/`self.__next` assignments, or the
+/// `__NEXT_DATA__`/`__NUXT__`/`application/json` payload conventions). If none
+/// are present, running the VM is guaranteed to return zero blobs, so skipping
+/// it is output-neutral. Conservative by design: any of these may appear in
+/// non-script HTML too, which only makes us skip *less* often, never more.
+const JS_CANDIDATE_MARKERS: [&str; 5] = [
+    "window.",
+    "__NEXT_DATA__",
+    "__NUXT__",
+    "application/json",
+    "self.__next",
+];
+
+/// Returns true if the HTML plausibly contains JS-assigned data the QuickJS
+/// scan could surface. When false, the VM is provably a no-op and is skipped.
+pub fn has_js_candidate_data(html: &str) -> bool {
+    JS_CANDIDATE_MARKERS.iter().any(|m| html.contains(m))
+}
+
 /// A blob of data extracted from JS execution.
 pub struct JsDataBlob {
     pub name: String,
@@ -24,9 +47,17 @@ pub struct JsDataBlob {
 }
 
 /// Execute inline `<script>` tags in a QuickJS sandbox and extract `window.__*` data.
+///
+/// Convenience wrapper that parses `html` first. Hot callers that already hold a
+/// parsed [`Html`] should use [`extract_js_data_from_doc`] to avoid a second parse.
 pub fn extract_js_data(html: &str) -> Vec<JsDataBlob> {
     let doc = Html::parse_document(html);
+    extract_js_data_from_doc(&doc)
+}
 
+/// Execute inline `<script>` tags in a QuickJS sandbox and extract `window.__*` data,
+/// reusing an already-parsed [`Html`] document instead of re-parsing the HTML.
+pub fn extract_js_data_from_doc(doc: &Html) -> Vec<JsDataBlob> {
     let scripts: Vec<String> = doc
         .select(&SCRIPT_SELECTOR)
         .filter(|el| {

crates/webclaw-core/src/lib.rs

@@ -222,8 +222,8 @@ fn extract_with_options_inner(
     // (e.g., window.__PRELOADED_STATE__, self.__next_f). This supplements the
     // static JSON data island extraction above with runtime-evaluated data.
     #[cfg(all(feature = "quickjs", not(target_arch = "wasm32")))]
-    {
-        let blobs = js_eval::extract_js_data(html);
+    if js_eval::has_js_candidate_data(html) {
+        let blobs = js_eval::extract_js_data_from_doc(&doc);
         if !blobs.is_empty() {
             let js_text = js_eval::extract_readable_text(&blobs);
             if !js_text.is_empty() {

crates/webclaw-core/src/markdown.rs

@@ -13,6 +13,8 @@ use crate::noise;
 use crate::types::{CodeBlock, Image, Link};
 
 static CODE_SELECTOR: Lazy<Selector> = Lazy::new(|| Selector::parse("code").unwrap());
+static IMG_ALT_SELECTOR: Lazy<Selector> = Lazy::new(|| Selector::parse("img[alt]").unwrap());
+static A_HREF_SELECTOR: Lazy<Selector> = Lazy::new(|| Selector::parse("a[href]").unwrap());
 
 /// Maximum recursion depth for DOM traversal.
 /// Express.co.uk live blogs and similar pages can nest 1000+ levels deep,
@@ -853,7 +855,7 @@ fn collect_assets_from_noise(
     assets: &mut ConvertedAssets,
 ) {
     // Collect images with alt text
-    for img in element.select(&Selector::parse("img[alt]").unwrap()) {
+    for img in element.select(&IMG_ALT_SELECTOR) {
         let alt = img.value().attr("alt").unwrap_or("").to_string();
         let src = img
             .value()
@@ -866,7 +868,7 @@ fn collect_assets_from_noise(
     }
 
     // Collect links
-    for link in element.select(&Selector::parse("a[href]").unwrap()) {
+    for link in element.select(&A_HREF_SELECTOR) {
         let href = link
             .value()
             .attr("href")

crates/webclaw-core/src/structured_data.rs

@@ -178,7 +178,12 @@ pub fn extract_sveltekit(html: &str) -> Vec<Value> {
 /// Preserves already-quoted keys and string values.
 fn js_literal_to_json(input: &str) -> String {
     let bytes = input.as_bytes();
-    let mut out = String::with_capacity(input.len() + input.len() / 10);
+    // Accumulate raw bytes, not `byte as char`. The input is valid UTF-8 and we
+    // only ever copy its bytes verbatim or insert ASCII quotes, so the result is
+    // guaranteed valid UTF-8 — copying byte-by-byte preserves multibyte
+    // codepoints (e.g. accented/CJK string values) instead of mangling them
+    // into Latin-1 mojibake.
+    let mut out: Vec<u8> = Vec::with_capacity(input.len() + input.len() / 10);
     let mut i = 0;
     let len = bytes.len();
 
@@ -187,14 +192,14 @@ fn js_literal_to_json(input: &str) -> String {
 
         // Skip through strings
         if b == b'"' {
-            out.push('"');
+            out.push(b'"');
             i += 1;
             while i < len {
                 let c = bytes[i];
-                out.push(c as char);
+                out.push(c);
                 i += 1;
                 if c == b'\\' && i < len {
-                    out.push(bytes[i] as char);
+                    out.push(bytes[i]);
                     i += 1;
                 } else if c == b'"' {
                     break;
@@ -205,11 +210,11 @@ fn js_literal_to_json(input: &str) -> String {
 
         // After { or , — look for unquoted key followed by :
         if (b == b'{' || b == b',' || b == b'[') && i + 1 < len {
-            out.push(b as char);
+            out.push(b);
             i += 1;
             // Skip whitespace
             while i < len && bytes[i].is_ascii_whitespace() {
-                out.push(bytes[i] as char);
+                out.push(bytes[i]);
                 i += 1;
             }
             // Check if next is an unquoted identifier (key)
@@ -218,29 +223,30 @@ fn js_literal_to_json(input: &str) -> String {
                 while i < len && (bytes[i].is_ascii_alphanumeric() || bytes[i] == b'_') {
                     i += 1;
                 }
-                let key = &input[key_start..i];
+                let key = &bytes[key_start..i];
                 // Skip whitespace after key
                 while i < len && bytes[i].is_ascii_whitespace() {
                     i += 1;
                 }
                 // If followed by :, it's an unquoted key — quote it
                 if i < len && bytes[i] == b':' {
-                    out.push('"');
-                    out.push_str(key);
-                    out.push('"');
+                    out.push(b'"');
+                    out.extend_from_slice(key);
+                    out.push(b'"');
                 } else {
                     // Not a key — might be a bare value like true/false/null
-                    out.push_str(key);
+                    out.extend_from_slice(key);
                 }
             }
             continue;
         }
 
-        out.push(b as char);
+        out.push(b);
         i += 1;
     }
 
-    out
+    // Safe: we only copied bytes from valid-UTF-8 `input` plus ASCII quotes.
+    String::from_utf8(out).unwrap_or_else(|e| String::from_utf8_lossy(e.as_bytes()).into_owned())
 }
 
 /// Replace raw newlines/tabs inside JSON string values with escape sequences.
@@ -440,4 +446,17 @@ newline"}"#;
         assert_eq!(parsed["text"], "line1\nline2");
         assert_eq!(parsed["raw"], "has\nnewline");
     }
+
+    #[test]
+    fn js_literal_to_json_preserves_multibyte_utf8() {
+        // Unquoted ASCII keys with accented and CJK string values (the shape
+        // SvelteKit emits). The old `byte as char` path turned the multibyte
+        // values into Latin-1 mojibake; they must now survive intact.
+        let input = r#"{name:"déjà vu", city:"東京", emoji:"🌱"}"#;
+        let json = js_literal_to_json(input);
+        let parsed: Value = serde_json::from_str(&json).unwrap();
+        assert_eq!(parsed["name"], "déjà vu");
+        assert_eq!(parsed["city"], "東京");
+        assert_eq!(parsed["emoji"], "🌱");
+    }
 }

crates/webclaw-fetch/Cargo.toml

@@ -14,13 +14,16 @@ tracing = { workspace = true }
 tokio = { workspace = true }
 async-trait = "0.1"
 # Pinned to exact pre-release versions: wreq/wreq-util are release candidates
-# with no semver stability between rc.N builds (rc.29 broke the TLS + Response
-# API). An exact pin keeps `cargo build`, `cargo install` (which ignores
-# Cargo.lock), and the release workflow all on the version that compiles.
-wreq = { version = "=6.0.0-rc.28", features = ["cookies", "gzip", "brotli", "zstd", "deflate"] }
-wreq-util = "=3.0.0-rc.10"
+# with no semver stability between rc.N builds. An exact pin keeps `cargo build`,
+# `cargo install` (which ignores Cargo.lock), and the release workflow all on the
+# version that compiles.
+wreq = { version = "=6.0.0-rc.29", features = ["cookies", "gzip", "brotli", "zstd", "deflate", "stream"] }
+wreq-util = "=3.0.0-rc.12"
 http = "1"
 bytes = "1"
+# Stream adapter for `wreq::Response::bytes_stream()` (wreq 6.0.0-rc.29 dropped
+# `Response::chunk()`); used to buffer bodies under the running size ceiling.
+futures-util = "0.3"
 url = "2"
 rand = "0.8"
 quick-xml = { version = "0.37", features = ["serde"] }
@@ -29,6 +32,7 @@ reqwest = { version = "0.12", default-features = false, features = ["json", "rus
 serde_json.workspace = true
 calamine = "0.34"
 zip = "2"
+flate2 = "1"
 
 [dev-dependencies]
 tempfile = "3"

crates/webclaw-fetch/src/client.rs

@@ -12,6 +12,7 @@ use std::hash::{Hash, Hasher};
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 
+use futures_util::StreamExt;
 use rand::seq::SliceRandom;
 use tokio::sync::Semaphore;
 use tracing::{debug, instrument, warn};
@@ -118,7 +119,7 @@ impl Response {
     /// negotiated), so a tiny compressed payload that inflates to
     /// gigabytes is aborted as soon as the accumulated size crosses the
     /// cap — it never gets fully buffered in memory.
-    async fn from_wreq(mut resp: wreq::Response) -> Result<Self, FetchError> {
+    async fn from_wreq(resp: wreq::Response) -> Result<Self, FetchError> {
         if let Some(len) = resp.content_length()
             && len > MAX_BODY_BYTES
         {
@@ -130,12 +131,13 @@ impl Response {
         let url = resp.uri().to_string();
         let headers = resp.headers().clone();
 
+        // wreq 6.0.0-rc.29 dropped `Response::chunk()`. Stream post-decompression
+        // bytes via `bytes_stream()` and keep enforcing the running ceiling so a
+        // compression bomb is aborted before it is fully buffered in memory.
         let mut buf = bytes::BytesMut::new();
-        while let Some(chunk) = resp
-            .chunk()
-            .await
-            .map_err(|e| FetchError::BodyDecode(e.to_string()))?
-        {
+        let mut stream = resp.bytes_stream();
+        while let Some(chunk) = stream.next().await {
+            let chunk = chunk.map_err(|e| FetchError::BodyDecode(e.to_string()))?;
             check_body_ceiling(buf.len(), chunk.len())?;
             buf.extend_from_slice(&chunk);
         }
@@ -168,6 +170,13 @@ impl Response {
     fn into_text(self) -> String {
         String::from_utf8_lossy(&self.body).into_owned()
     }
+
+    /// Consume the response and return the raw, undecoded body bytes.
+    /// Used by [`FetchClient::fetch_raw`] for binary payloads (e.g. gzipped
+    /// sitemaps) that must not be run through lossy UTF-8 decoding.
+    fn into_body(self) -> bytes::Bytes {
+        self.body
+    }
 }
 
 /// Internal representation of the client pool strategy.
@@ -456,6 +465,27 @@ impl FetchClient {
         Err(last_err.unwrap_or_else(|| FetchError::Build("all retries exhausted".into())))
     }
 
+    /// Fetch a URL and return the raw, undecoded response body as bytes.
+    ///
+    /// Unlike [`fetch`](Self::fetch), this does **not** run the body through
+    /// `String::from_utf8_lossy`, so binary payloads survive intact. This is
+    /// required for gzipped sitemaps (`.xml.gz`): such files are served with
+    /// `Content-Type: application/gzip` and *no* `Content-Encoding`, so wreq
+    /// never auto-inflates them — the bytes arrive as raw gzip and the lossy
+    /// String path would mangle them. Callers detect the gzip magic
+    /// (`0x1f 0x8b`) and gunzip before parsing.
+    ///
+    /// No retry wrapper: callers (sitemap discovery) already tolerate
+    /// per-URL failures by skipping. Returns `(status, body)`.
+    pub async fn fetch_raw(&self, url: &str) -> Result<(u16, bytes::Bytes), FetchError> {
+        let parsed_url = crate::url_security::validate_public_http_url(url).await?;
+        let url = parsed_url.as_str();
+        let client = self.pick_client(url);
+        let resp = client.get(url).send().await?;
+        let response = Response::from_wreq(resp).await?;
+        Ok((response.status(), response.into_body()))
+    }
+
     /// Fetch a URL then extract structured content.
     #[instrument(skip(self), fields(url = %url))]
     pub async fn fetch_and_extract(
@@ -799,11 +829,17 @@ fn is_challenge_html(html: &str) -> bool {
     false
 }
 
-/// Extract the homepage URL (scheme + host) from a full URL.
+/// Extract the homepage URL (scheme + host[:port]) from a full URL.
 fn extract_homepage(url: &str) -> Option<String> {
-    url::Url::parse(url)
-        .ok()
-        .map(|u| format!("{}://{}/", u.scheme(), u.host_str().unwrap_or("")))
+    url::Url::parse(url).ok().map(|u| {
+        let host = u.host_str().unwrap_or("");
+        // `port()` is `Some` only for a non-default port; include it so a
+        // host like example.com:8443 is warmed on the right port.
+        match u.port() {
+            Some(port) => format!("{}://{}:{}/", u.scheme(), host, port),
+            None => format!("{}://{}/", u.scheme(), host),
+        }
+    })
 }
 
 /// Convert a webclaw-pdf PdfResult into a webclaw-core ExtractionResult.

crates/webclaw-fetch/src/crawler.rs

@@ -528,7 +528,7 @@ impl Crawler {
 }
 
 /// Canonical origin string for comparing same-origin: "scheme://host[:port]".
-fn origin_key(url: &Url) -> String {
+pub(crate) fn origin_key(url: &Url) -> String {
     let port_suffix = match url.port() {
         Some(p) => format!(":{p}"),
         None => String::new(),
@@ -563,7 +563,7 @@ fn root_domain(url: &Url) -> String {
 
 /// Normalize a URL for dedup: strip fragment, remove trailing slash (except root "/"),
 /// lowercase scheme + host. Preserves query params and path case.
-fn normalize(url: &Url) -> String {
+pub(crate) fn normalize(url: &Url) -> String {
     let scheme = url.scheme();
     let host = url.host_str().unwrap_or("").to_ascii_lowercase();
     let port_suffix = match url.port() {

crates/webclaw-fetch/src/extractors/amazon_product.rs

@@ -33,6 +33,7 @@ use serde_json::{Value, json};
 use url::Url;
 
 use super::ExtractorInfo;
+use super::og::parse_og;
 use crate::cloud::{self, CloudError};
 use crate::error::FetchError;
 use crate::fetcher::Fetcher;
@@ -115,23 +116,25 @@ pub async fn extract(client: &dyn Fetcher, url: &str) -> Result<Value, FetchErro
 /// without carrying webclaw_fetch types.
 pub fn parse(html: &str, url: &str, asin: &str) -> Value {
     let jsonld = find_product_jsonld(html);
+    // Single scan for the og:* fallbacks read below.
+    let og_meta = parse_og(html);
     // Three-tier title: JSON-LD `name` > Amazon's `#productTitle` span
     // (only present on real static HTML) > cloud-synthesized og:title.
     let title = jsonld
         .as_ref()
         .and_then(|v| get_text(v, "name"))
         .or_else(|| dom_title(html))
-        .or_else(|| og(html, "title"));
+        .or_else(|| og_meta.unescaped("title"));
     let image = jsonld
         .as_ref()
         .and_then(get_first_image)
         .or_else(|| dom_image(html))
-        .or_else(|| og(html, "image"));
+        .or_else(|| og_meta.unescaped("image"));
     let brand = jsonld.as_ref().and_then(get_brand);
     let description = jsonld
         .as_ref()
         .and_then(|v| get_text(v, "description"))
-        .or_else(|| og(html, "description"));
+        .or_else(|| og_meta.unescaped("description"));
     let aggregate_rating = jsonld.as_ref().and_then(get_aggregate_rating);
     let offer = jsonld.as_ref().and_then(first_offer);
 
@@ -336,31 +339,6 @@ fn dom_image(html: &str) -> Option<String> {
         .map(|m| m.as_str().to_string())
 }
 
-/// OG meta tag lookup. Cloud-synthesized HTML ships these even when
-/// JSON-LD and Amazon-DOM-IDs are both absent, so they're the last
-/// line of defence for `title`, `image`, `description`.
-fn og(html: &str, prop: &str) -> Option<String> {
-    static RE: OnceLock<Regex> = OnceLock::new();
-    let re = RE.get_or_init(|| {
-        Regex::new(r#"(?i)<meta[^>]+property="og:([a-z_]+)"[^>]+content="([^"]+)""#).unwrap()
-    });
-    for c in re.captures_iter(html) {
-        if c.get(1).is_some_and(|m| m.as_str() == prop) {
-            return c.get(2).map(|m| html_unescape(m.as_str()));
-        }
-    }
-    None
-}
-
-/// Undo the synthesize_html attribute escaping for the few entities it
-/// emits. Keeps us off a heavier HTML-entity dep.
-fn html_unescape(s: &str) -> String {
-    s.replace("&quot;", "\"")
-        .replace("&amp;", "&")
-        .replace("&lt;", "<")
-        .replace("&gt;", ">")
-}
-
 fn cloud_to_fetch_err(e: CloudError) -> FetchError {
     FetchError::Build(e.to_string())
 }
@@ -477,7 +455,7 @@ mod tests {
     fn og_unescape_handles_quot_entity() {
         let html = r#"<meta property="og:title" content="Apple &quot;M2 Pro&quot; Laptop">"#;
         assert_eq!(
-            og(html, "title").as_deref(),
+            parse_og(html).unescaped("title").as_deref(),
             Some(r#"Apple "M2 Pro" Laptop"#)
         );
     }

crates/webclaw-fetch/src/extractors/ebay_listing.rs

@@ -15,6 +15,7 @@ use serde_json::{Value, json};
 use url::Url;
 
 use super::ExtractorInfo;
+use super::og::parse_og;
 use crate::cloud::{self, CloudError};
 use crate::error::FetchError;
 use crate::fetcher::Fetcher;
@@ -65,19 +66,21 @@ pub async fn extract(client: &dyn Fetcher, url: &str) -> Result<Value, FetchErro
 
 pub fn parse(html: &str, url: &str, item_id: &str) -> Value {
     let jsonld = find_product_jsonld(html);
+    // Single scan for the three og:* fields read as fallbacks below.
+    let og_meta = parse_og(html);
     let title = jsonld
         .as_ref()
         .and_then(|v| get_text(v, "name"))
-        .or_else(|| og(html, "title"));
+        .or_else(|| og_meta.raw("title"));
     let image = jsonld
         .as_ref()
         .and_then(get_first_image)
-        .or_else(|| og(html, "image"));
+        .or_else(|| og_meta.raw("image"));
     let brand = jsonld.as_ref().and_then(get_brand);
     let description = jsonld
         .as_ref()
         .and_then(|v| get_text(v, "description"))
-        .or_else(|| og(html, "description"));
+        .or_else(|| og_meta.raw("description"));
     let offer = jsonld.as_ref().and_then(first_offer);
 
     // eBay's AggregateOffer uses lowPrice/highPrice. Offer uses price.
@@ -268,19 +271,6 @@ fn get_aggregate_rating(v: &Value) -> Option<Value> {
     }))
 }
 
-fn og(html: &str, prop: &str) -> Option<String> {
-    static RE: OnceLock<Regex> = OnceLock::new();
-    let re = RE.get_or_init(|| {
-        Regex::new(r#"(?i)<meta[^>]+property="og:([a-z_]+)"[^>]+content="([^"]+)""#).unwrap()
-    });
-    for c in re.captures_iter(html) {
-        if c.get(1).is_some_and(|m| m.as_str() == prop) {
-            return c.get(2).map(|m| m.as_str().to_string());
-        }
-    }
-    None
-}
-
 fn cloud_to_fetch_err(e: CloudError) -> FetchError {
     FetchError::Build(e.to_string())
 }

crates/webclaw-fetch/src/extractors/ecommerce_product.rs

@@ -42,6 +42,7 @@ use regex::Regex;
 use serde_json::{Value, json};
 
 use super::ExtractorInfo;
+use super::og::{og, parse_og};
 use crate::error::FetchError;
 use crate::fetcher::Fetcher;
 
@@ -142,15 +143,17 @@ fn build_jsonld_payload(product: &Value, html: &str, url: &str) -> Value {
 /// Build a minimal payload from OG / product meta tags. Used when a
 /// page has no Product JSON-LD at all.
 fn build_og_payload(html: &str, url: &str) -> Value {
+    // Single scan for the three og:* fields this fallback reads.
+    let og_meta = parse_og(html);
     let offers = build_og_offer(html).map(|o| vec![o]).unwrap_or_default();
-    let image = og(html, "image");
+    let image = og_meta.raw("image");
     let images: Vec<Value> = image.map(|i| vec![Value::String(i)]).unwrap_or_default();
 
     json!({
         "url":                url,
         "data_source":        "og_fallback",
-        "name":               og(html, "title"),
-        "description":        og(html, "description"),
+        "name":               og_meta.raw("title"),
+        "description":        og_meta.raw("description"),
         "brand":              meta_property(html, "product:brand"),
         "sku":                None::<String>,
         "mpn":                None::<String>,
@@ -368,20 +371,6 @@ fn build_og_offer(html: &str) -> Option<Value> {
     }))
 }
 
-/// Pull the value of `<meta property="og:{prop}" content="...">`.
-fn og(html: &str, prop: &str) -> Option<String> {
-    static RE: OnceLock<Regex> = OnceLock::new();
-    let re = RE.get_or_init(|| {
-        Regex::new(r#"(?i)<meta[^>]+property="og:([a-z_]+)"[^>]+content="([^"]+)""#).unwrap()
-    });
-    for c in re.captures_iter(html) {
-        if c.get(1).is_some_and(|m| m.as_str() == prop) {
-            return c.get(2).map(|m| m.as_str().to_string());
-        }
-    }
-    None
-}
-
 /// Pull the value of any `<meta property="..." content="...">` tag.
 /// Needed for namespaced OG variants like `product:price:amount` that
 /// the simple `og:*` matcher above doesn't cover.

crates/webclaw-fetch/src/extractors/etsy_listing.rs

@@ -26,6 +26,7 @@ use regex::Regex;
 use serde_json::{Value, json};
 
 use super::ExtractorInfo;
+use super::og::parse_og;
 use crate::cloud::{self, CloudError};
 use crate::error::FetchError;
 use crate::fetcher::Fetcher;
@@ -74,19 +75,26 @@ pub fn parse(html: &str, url: &str, listing_id: &str) -> Value {
     let jsonld = find_product_jsonld(html);
     let slug_title = humanise_slug(parse_slug(url).as_deref());
 
+    // Single scan for the three og:* fields used as fallbacks below.
+    let og_meta = parse_og(html);
+
     let title = jsonld
         .as_ref()
         .and_then(|v| get_text(v, "name"))
-        .or_else(|| og(html, "title").filter(|t| !is_generic_title(t)))
+        .or_else(|| og_meta.raw("title").filter(|t| !is_generic_title(t)))
         .or(slug_title);
     let description = jsonld
         .as_ref()
         .and_then(|v| get_text(v, "description"))
-        .or_else(|| og(html, "description").filter(|d| !is_generic_description(d)));
+        .or_else(|| {
+            og_meta
+                .raw("description")
+                .filter(|d| !is_generic_description(d))
+        });
     let image = jsonld
         .as_ref()
         .and_then(get_first_image)
-        .or_else(|| og(html, "image"));
+        .or_else(|| og_meta.raw("image"));
     let brand = jsonld.as_ref().and_then(get_brand);
 
     // Etsy listings often ship either a single Offer or an
@@ -359,19 +367,6 @@ fn strip_schema_prefix(s: String) -> String {
         .replace("https://schema.org/", "")
 }
 
-fn og(html: &str, prop: &str) -> Option<String> {
-    static RE: OnceLock<Regex> = OnceLock::new();
-    let re = RE.get_or_init(|| {
-        Regex::new(r#"(?i)<meta[^>]+property="og:([a-z_]+)"[^>]+content="([^"]+)""#).unwrap()
-    });
-    for c in re.captures_iter(html) {
-        if c.get(1).is_some_and(|m| m.as_str() == prop) {
-            return c.get(2).map(|m| m.as_str().to_string());
-        }
-    }
-    None
-}
-
 /// Etsy links the owning shop with a canonical anchor like
 /// `<a href="/shop/ShopName" ...>`. Grab the first one after the
 /// breadcrumb boundary.

crates/webclaw-fetch/src/extractors/mod.rs

@@ -33,6 +33,7 @@ pub mod instagram_post;
 pub mod instagram_profile;
 pub mod linkedin_post;
 pub mod npm;
+pub(crate) mod og;
 pub mod pypi;
 pub mod reddit;
 pub mod shopify_collection;

crates/webclaw-fetch/src/extractors/og.rs

@@ -0,0 +1,79 @@
+//! Shared Open Graph (`og:*`) meta-tag parsing for the HTML vertical
+//! extractors.
+//!
+//! Several site extractors read a handful of `og:*` properties (title,
+//! description, image, ...) from the page `<head>`. Each used to carry a
+//! verbatim copy of the same regex + scan helper. This module centralises
+//! that logic and adds [`parse_og`], which collects every `og:*` pair in a
+//! single `captures_iter` pass so an extractor that needs multiple fields
+//! scans the document once instead of once per field.
+//!
+//! Values are stored raw. Callers that need HTML entity decoding apply
+//! [`html_unescape`] themselves — some extractors intentionally keep the
+//! raw value, so decoding is opt-in per call site to preserve output.
+
+use std::collections::HashMap;
+use std::sync::OnceLock;
+
+use regex::Regex;
+
+/// Matches `<meta property="og:<name>" content="<value>">`, case-insensitive.
+/// Capture 1 is the property suffix (after `og:`), capture 2 is the content.
+fn og_regex() -> &'static Regex {
+    static RE: OnceLock<Regex> = OnceLock::new();
+    RE.get_or_init(|| {
+        Regex::new(r#"(?i)<meta[^>]+property="og:([a-z_]+)"[^>]+content="([^"]+)""#).unwrap()
+    })
+}
+
+/// Return the raw content of the first `og:<prop>` meta tag, if present.
+///
+/// Single-pass per call. For extractors reading several properties, prefer
+/// [`parse_og`] to scan the document only once.
+pub(crate) fn og(html: &str, prop: &str) -> Option<String> {
+    for c in og_regex().captures_iter(html) {
+        if c.get(1).is_some_and(|m| m.as_str() == prop) {
+            return c.get(2).map(|m| m.as_str().to_string());
+        }
+    }
+    None
+}
+
+/// Parse every `og:*` meta tag in one pass into a `suffix -> content` map.
+///
+/// First occurrence wins, matching the short-circuit-on-first-match
+/// behaviour of [`og`] when called per property. Values are raw (not
+/// entity-decoded); use [`OgMeta::unescaped`] / [`OgMeta::raw`] to read.
+pub(crate) fn parse_og(html: &str) -> OgMeta {
+    let mut map: HashMap<String, String> = HashMap::new();
+    for c in og_regex().captures_iter(html) {
+        if let (Some(name), Some(content)) = (c.get(1), c.get(2)) {
+            map.entry(name.as_str().to_string())
+                .or_insert_with(|| content.as_str().to_string());
+        }
+    }
+    OgMeta(map)
+}
+
+/// Parsed `og:*` properties from a single document scan.
+pub(crate) struct OgMeta(HashMap<String, String>);
+
+impl OgMeta {
+    /// Raw content of `og:<prop>`, exactly as it appeared in the HTML.
+    pub(crate) fn raw(&self, prop: &str) -> Option<String> {
+        self.0.get(prop).cloned()
+    }
+
+    /// Content of `og:<prop>` with the common HTML entities decoded.
+    pub(crate) fn unescaped(&self, prop: &str) -> Option<String> {
+        self.0.get(prop).map(|v| html_unescape(v))
+    }
+}
+
+/// Decode the small set of HTML entities that show up in `og:*` content.
+pub(crate) fn html_unescape(s: &str) -> String {
+    s.replace("&quot;", "\"")
+        .replace("&amp;", "&")
+        .replace("&lt;", "<")
+        .replace("&gt;", ">")
+}

crates/webclaw-fetch/src/extractors/substack_post.rs

@@ -28,6 +28,7 @@ use serde::Deserialize;
 use serde_json::{Value, json};
 
 use super::ExtractorInfo;
+use super::og::parse_og;
 use crate::cloud::{self, CloudError};
 use crate::error::FetchError;
 use crate::fetcher::Fetcher;
@@ -181,24 +182,27 @@ async fn html_fallback(
 pub fn parse_html(html: &str, url: &str, api_url: &str, slug: &str) -> Value {
     let article = find_article_jsonld(html);
 
+    // Single scan for the four og:* fields read as fallbacks below.
+    let og_meta = parse_og(html);
+
     let title = article
         .as_ref()
         .and_then(|v| get_text(v, "headline"))
-        .or_else(|| og(html, "title"));
+        .or_else(|| og_meta.raw("title"));
     let description = article
         .as_ref()
         .and_then(|v| get_text(v, "description"))
-        .or_else(|| og(html, "description"));
+        .or_else(|| og_meta.raw("description"));
     let cover_image = article
         .as_ref()
         .and_then(get_first_image)
-        .or_else(|| og(html, "image"));
+        .or_else(|| og_meta.raw("image"));
     let post_date = article
         .as_ref()
         .and_then(|v| get_text(v, "datePublished"))
         .or_else(|| meta_property(html, "article:published_time"));
     let updated_at = article.as_ref().and_then(|v| get_text(v, "dateModified"));
-    let publication_name = og(html, "site_name");
+    let publication_name = og_meta.raw("site_name");
     let authors = article.as_ref().map(extract_authors).unwrap_or_default();
 
     json!({
@@ -302,19 +306,6 @@ fn handle_from_author_url(u: &str) -> Option<String> {
 // HTML tag helpers
 // ---------------------------------------------------------------------------
 
-fn og(html: &str, prop: &str) -> Option<String> {
-    static RE: OnceLock<Regex> = OnceLock::new();
-    let re = RE.get_or_init(|| {
-        Regex::new(r#"(?i)<meta[^>]+property="og:([a-z_]+)"[^>]+content="([^"]+)""#).unwrap()
-    });
-    for c in re.captures_iter(html) {
-        if c.get(1).is_some_and(|m| m.as_str() == prop) {
-            return c.get(2).map(|m| m.as_str().to_string());
-        }
-    }
-    None
-}
-
 /// Pull `<meta property="article:published_time" content="...">` and
 /// similar structured meta tags.
 fn meta_property(html: &str, prop: &str) -> Option<String> {

crates/webclaw-fetch/src/extractors/trustpilot_reviews.rs

@@ -32,6 +32,7 @@ use regex::Regex;
 use serde_json::{Value, json};
 
 use super::ExtractorInfo;
+use super::og::parse_og;
 use crate::cloud::{self, CloudError};
 use crate::error::FetchError;
 use crate::fetcher::Fetcher;
@@ -87,11 +88,17 @@ pub fn parse(html: &str, url: &str) -> Result<Value, FetchError> {
     // The aiSummary block: not typed (no `@type`), detect by key.
     let ai_block = find_ai_summary_block(&blocks);
 
+    // Single scan of the page's og:* meta tags; title + description feed
+    // the regex fallbacks below.
+    let og_meta = parse_og(html);
+    let og_title = og_meta.unescaped("title");
+    let og_description = og_meta.unescaped("description");
+
     // Business name: Dataset > metadata.title regex > URL domain.
     let business_name = dataset
         .as_ref()
         .and_then(|d| get_string(d, "name"))
-        .or_else(|| parse_name_from_og_title(html))
+        .or_else(|| parse_name_from_og_title(og_title.as_deref()))
         .or_else(|| Some(domain.clone()));
 
     // Rating distribution from the csvw:Table columns. Each column has
@@ -105,8 +112,8 @@ pub fn parse(html: &str, url: &str) -> Result<Value, FetchError> {
 
     // Page-title / page-description fallbacks. OG title format:
     // "Anthropic is rated \"Bad\" with 1.5 / 5 on Trustpilot"
-    let (rating_label, rating_from_og) = parse_rating_from_og_title(html);
-    let total_from_desc = parse_review_count_from_og_description(html);
+    let (rating_label, rating_from_og) = parse_rating_from_og_title(og_title.as_deref());
+    let total_from_desc = parse_review_count_from_og_description(og_description.as_deref());
 
     // Recent reviews carried by the aiSummary block.
     let recent_reviews: Vec<Value> = ai_block
@@ -336,20 +343,21 @@ fn compute_rating_stats(distribution: &Value) -> (Option<String>, Option<i64>) {
 
 /// Regex out the business name from the standard Trustpilot OG title
 /// shape: `"{name} is rated \"{label}\" with {rating} / 5 on Trustpilot"`.
-fn parse_name_from_og_title(html: &str) -> Option<String> {
-    let title = og(html, "title")?;
+/// `title` is the (entity-decoded) `og:title` content.
+fn parse_name_from_og_title(title: Option<&str>) -> Option<String> {
+    let title = title?;
     // "Anthropic is rated \"Bad\" with 1.5 / 5 on Trustpilot"
     static RE: OnceLock<Regex> = OnceLock::new();
     let re = RE.get_or_init(|| Regex::new(r"^(.+?)\s+is rated\b").unwrap());
-    re.captures(&title)
+    re.captures(title)
         .and_then(|c| c.get(1))
         .map(|m| m.as_str().to_string())
 }
 
 /// Pull the rating label (e.g. "Bad", "Excellent") and numeric value
-/// from the OG title.
-fn parse_rating_from_og_title(html: &str) -> (Option<String>, Option<String>) {
-    let Some(title) = og(html, "title") else {
+/// from the (entity-decoded) `og:title` content.
+fn parse_rating_from_og_title(title: Option<&str>) -> (Option<String>, Option<String>) {
+    let Some(title) = title else {
         return (None, None);
     };
     static RE: OnceLock<Regex> = OnceLock::new();
@@ -357,7 +365,7 @@ fn parse_rating_from_og_title(html: &str) -> (Option<String>, Option<String>) {
     let re = RE.get_or_init(|| {
         Regex::new(r#"is rated\s*[\\"]+([^"\\]+)[\\"]+\s*with\s*([\d.]+)\s*/\s*5"#).unwrap()
     });
-    let Some(caps) = re.captures(&title) else {
+    let Some(caps) = re.captures(title) else {
         return (None, None);
     };
     (
@@ -366,13 +374,13 @@ fn parse_rating_from_og_title(html: &str) -> (Option<String>, Option<String>) {
     )
 }
 
-/// Parse "hear what 226 customers have already said" from the OG
-/// description tag.
-fn parse_review_count_from_og_description(html: &str) -> Option<i64> {
-    let desc = og(html, "description")?;
+/// Parse "hear what 226 customers have already said" from the
+/// (entity-decoded) `og:description` content.
+fn parse_review_count_from_og_description(desc: Option<&str>) -> Option<i64> {
+    let desc = desc?;
     static RE: OnceLock<Regex> = OnceLock::new();
     let re = RE.get_or_init(|| Regex::new(r"(\d[\d,]*)\s+customers").unwrap());
-    re.captures(&desc)?
+    re.captures(desc)?
         .get(1)?
         .as_str()
         .replace(',', "")
@@ -380,29 +388,6 @@ fn parse_review_count_from_og_description(html: &str) -> Option<i64> {
         .ok()
 }
 
-fn og(html: &str, prop: &str) -> Option<String> {
-    static RE: OnceLock<Regex> = OnceLock::new();
-    let re = RE.get_or_init(|| {
-        Regex::new(r#"(?i)<meta[^>]+property="og:([a-z_]+)"[^>]+content="([^"]+)""#).unwrap()
-    });
-    for c in re.captures_iter(html) {
-        if c.get(1).is_some_and(|m| m.as_str() == prop) {
-            let raw = c.get(2).map(|m| m.as_str())?;
-            return Some(html_unescape(raw));
-        }
-    }
-    None
-}
-
-/// Minimal HTML entity unescaping for the three entities the
-/// synthesize_html escaper might produce. Keeps us off a heavier dep.
-fn html_unescape(s: &str) -> String {
-    s.replace("&quot;", "\"")
-        .replace("&amp;", "&")
-        .replace("&lt;", "<")
-        .replace("&gt;", ">")
-}
-
 fn get_string(v: &Value, key: &str) -> Option<String> {
     v.get(key).and_then(|x| x.as_str().map(String::from))
 }
@@ -488,8 +473,12 @@ mod tests {
     #[test]
     fn parse_og_title_extracts_name_and_rating() {
         let html = r#"<meta property="og:title" content="Anthropic is rated &quot;Bad&quot; with 1.5 / 5 on Trustpilot">"#;
-        assert_eq!(parse_name_from_og_title(html), Some("Anthropic".into()));
-        let (label, rating) = parse_rating_from_og_title(html);
+        let title = parse_og(html).unescaped("title");
+        assert_eq!(
+            parse_name_from_og_title(title.as_deref()),
+            Some("Anthropic".into())
+        );
+        let (label, rating) = parse_rating_from_og_title(title.as_deref());
         assert_eq!(label.as_deref(), Some("Bad"));
         assert_eq!(rating.as_deref(), Some("1.5"));
     }
@@ -497,7 +486,11 @@ mod tests {
     #[test]
     fn parse_review_count_from_og_description_picks_number() {
         let html = r#"<meta property="og:description" content="Do you agree? Voice your opinion today and hear what 226 customers have already said.">"#;
-        assert_eq!(parse_review_count_from_og_description(html), Some(226));
+        let desc = parse_og(html).unescaped("description");
+        assert_eq!(
+            parse_review_count_from_og_description(desc.as_deref()),
+            Some(226)
+        );
     }
 
     #[test]

crates/webclaw-fetch/src/extractors/youtube_video.rs

@@ -25,6 +25,7 @@ use regex::Regex;
 use serde_json::{Value, json};
 
 use super::ExtractorInfo;
+use super::og::parse_og;
 use crate::error::FetchError;
 use crate::fetcher::Fetcher;
 
@@ -143,9 +144,11 @@ fn build_player_payload(
 // ---------------------------------------------------------------------------
 
 fn build_og_fallback(html: &str, url: &str, canonical: &str, video_id: &str) -> Value {
-    let title = og(html, "title");
-    let description = og(html, "description");
-    let thumbnail = og(html, "image");
+    // Single scan for the three og:* fields read below.
+    let og_meta = parse_og(html);
+    let title = og_meta.raw("title");
+    let description = og_meta.raw("description");
+    let thumbnail = og_meta.raw("image");
     // YouTube sets `<meta name="channel_name" ...>` on some pages but
     // OG-only pages reliably carry `og:video:tag` and the channel in
     // `<link itemprop="name">`. We keep this lean: just what's stable.
@@ -248,19 +251,6 @@ fn extract_player_response(html: &str) -> Option<Value> {
 // Meta-tag helpers (for OG fallback)
 // ---------------------------------------------------------------------------
 
-fn og(html: &str, prop: &str) -> Option<String> {
-    static RE: OnceLock<Regex> = OnceLock::new();
-    let re = RE.get_or_init(|| {
-        Regex::new(r#"(?i)<meta[^>]+property="og:([a-z_]+)"[^>]+content="([^"]+)""#).unwrap()
-    });
-    for c in re.captures_iter(html) {
-        if c.get(1).is_some_and(|m| m.as_str() == prop) {
-            return c.get(2).map(|m| m.as_str().to_string());
-        }
-    }
-    None
-}
-
 fn meta_name(html: &str, name: &str) -> Option<String> {
     static RE: OnceLock<Regex> = OnceLock::new();
     let re = RE.get_or_init(|| {

crates/webclaw-fetch/src/lib.rs

@@ -11,9 +11,11 @@ pub mod extractors;
 pub mod fetcher;
 pub mod linkedin;
 pub mod locale;
+pub mod map;
 pub mod progress;
 pub mod proxy;
 pub mod reddit;
+pub mod search;
 pub mod sitemap;
 pub mod tls;
 pub mod url_security;
@@ -25,7 +27,9 @@ pub use error::FetchError;
 pub use fetcher::Fetcher;
 pub use http::HeaderMap;
 pub use locale::{accept_language_for_tld, accept_language_for_url};
+pub use map::{MapOptions, discover_urls};
 pub use progress::{PROGRESS_INTERVAL, with_progress};
 pub use proxy::{parse_proxy_file, parse_proxy_line};
+pub use search::{SearchOptions, SearchResult, parse_serper_organic, search};
 pub use sitemap::SitemapEntry;
 pub use webclaw_pdf::PdfMode;

crates/webclaw-fetch/src/map.rs

@@ -0,0 +1,326 @@
+//! Layered URL discovery for the `map` command.
+//!
+//! `sitemap::discover` only finds URLs a site explicitly advertises in its
+//! `sitemap.xml`. Plenty of sites have no sitemap (news.ycombinator.com), a
+//! stale one, or a thin one that lists a handful of section roots. For those,
+//! a sitemap-only map returns almost nothing.
+//!
+//! This module adds a second layer: when the sitemap yields fewer than a
+//! threshold of URLs, run a *bounded* same-origin crawl and harvest every URL
+//! it touches — fetched pages, the visited set, **and** the remaining frontier
+//! (links queued but never fetched because the page cap was hit). That last
+//! bucket is the gold: a 150-page crawl of a link-dense site surfaces several
+//! thousand frontier URLs, turning a useless map into a real one.
+//!
+//! Strategy (layered, sitemap-first):
+//! 1. Sitemaps via [`sitemap::discover`] — authoritative, carries metadata
+//!    (lastmod / priority / changefreq).
+//! 2. If sitemaps are thin (`< min_sitemap_urls`) and the fallback is enabled,
+//!    a bounded crawl fills in the rest. Crawl-discovered URLs carry no
+//!    metadata (`None` everywhere) since they come from link harvesting, not a
+//!    sitemap.
+//!
+//! Sitemap entries always come first in the returned vec; crawl-discovered
+//! URLs are appended, deduplicated against the sitemap set using the *same*
+//! normalization the crawler uses ([`crawler::normalize`]) so map output stays
+//! internally consistent.
+
+use std::collections::HashSet;
+use std::time::Duration;
+
+use url::Url;
+
+use crate::client::{FetchClient, FetchConfig};
+use crate::crawler::{self, CrawlConfig, Crawler};
+use crate::sitemap::{self, SitemapEntry};
+
+/// Tuning knobs for [`discover_urls`].
+#[derive(Debug, Clone)]
+pub struct MapOptions {
+    /// Hard cap on pages the fallback crawl will fetch. The crawl surfaces far
+    /// more URLs than this via the unfetched frontier, so a small number still
+    /// yields a large map while keeping the crawl fast and polite.
+    pub max_crawl_pages: usize,
+    /// How deep the fallback crawl follows links (1 = links off the seed only).
+    pub crawl_depth: usize,
+    /// Sitemap-URL count below which the crawl fallback kicks in. A site with a
+    /// rich sitemap (≥ this many URLs) skips the crawl entirely.
+    pub min_sitemap_urls: usize,
+    /// Master switch for the crawl fallback. When `false`, behaves exactly like
+    /// the old sitemap-only `discover`.
+    pub crawl_fallback: bool,
+    /// Optional cap on URLs returned. `None` (default) = uncapped: return every
+    /// URL discovered (the crawl is already bounded by `max_crawl_pages`, so the
+    /// uncapped set is the links harvested from the fetched pages). Set `Some(n)`
+    /// to truncate.
+    pub max_urls: Option<usize>,
+}
+
+impl Default for MapOptions {
+    fn default() -> Self {
+        Self {
+            max_crawl_pages: 150,
+            crawl_depth: 2,
+            min_sitemap_urls: 200,
+            crawl_fallback: true,
+            max_urls: None,
+        }
+    }
+}
+
+/// Discover URLs for a site using the layered strategy described in the module
+/// docs: sitemaps first, then a bounded crawl fallback when the sitemap is
+/// thin.
+///
+/// Never errors — sitemap and crawl failures are swallowed and simply yield
+/// fewer URLs (an empty vec in the worst case), matching `sitemap::discover`'s
+/// "absence is not an error" contract.
+pub async fn discover_urls(
+    client: &FetchClient,
+    base_url: &str,
+    opts: &MapOptions,
+) -> Vec<SitemapEntry> {
+    // Layer 1: sitemaps.
+    let mut entries = sitemap::discover(client, base_url)
+        .await
+        .unwrap_or_default();
+
+    // Track normalized URLs we've already emitted, for cross-layer dedup.
+    let mut seen: HashSet<String> = entries.iter().filter_map(normalize_str).collect();
+
+    // Layer 2: bounded crawl fallback, only when the sitemap is thin.
+    if !opts.crawl_fallback || entries.len() >= opts.min_sitemap_urls {
+        return entries;
+    }
+
+    let Some(base_origin) = Url::parse(base_url).ok().map(|u| crawler::origin_key(&u)) else {
+        // Unparseable base URL — nothing sensible to crawl against.
+        return entries;
+    };
+
+    let config = CrawlConfig {
+        fetch: FetchConfig::default(),
+        max_depth: opts.crawl_depth,
+        max_pages: opts.max_crawl_pages,
+        // Politeness + scope: same-origin only (crawler default), modest delay.
+        delay: Duration::from_millis(50),
+        ..CrawlConfig::default()
+    };
+
+    let crawler = match Crawler::new(base_url, config) {
+        Ok(c) => c,
+        Err(_) => return entries,
+    };
+
+    let result = crawler.crawl(base_url, None).await;
+
+    // Richest source first: every link harvested from each fetched page. A
+    // directory/index page holds hundreds of same-origin links, and this set is
+    // NOT bound by the crawler's internal frontier cap. Then the URLs the crawl
+    // itself touched (fetched, visited, queued-but-unfetched frontier).
+    let mut discovered: Vec<String> = Vec::new();
+    for p in &result.pages {
+        discovered.push(p.url.clone());
+        if let Some(ex) = p.extraction.as_ref() {
+            let page_base = Url::parse(&p.url).ok();
+            for link in &ex.content.links {
+                // Resolve relative/protocol-relative hrefs against the page URL
+                // so the same-origin filter and dedup see absolute URLs.
+                let abs = match &page_base {
+                    Some(b) => b.join(&link.href).ok(),
+                    None => Url::parse(&link.href).ok(),
+                };
+                if let Some(u) = abs {
+                    discovered.push(u.to_string());
+                }
+            }
+        }
+    }
+    discovered.extend(result.visited);
+    discovered.extend(result.remaining_frontier.into_iter().map(|(url, _)| url));
+
+    append_crawled(&mut entries, &mut seen, discovered, &base_origin);
+
+    // Uncapped by default; only truncate if the caller set an explicit limit
+    // (sitemap entries added first keep priority).
+    if let Some(cap) = opts.max_urls {
+        entries.truncate(cap);
+    }
+    entries
+}
+
+/// Normalize a raw URL string to the crawler's canonical form, returning `None`
+/// if it doesn't parse.
+fn normalize_url(raw: &str) -> Option<String> {
+    Url::parse(raw).ok().map(|u| crawler::normalize(&u))
+}
+
+/// Normalize a [`SitemapEntry`]'s URL for the dedup set.
+fn normalize_str(entry: &SitemapEntry) -> Option<String> {
+    normalize_url(&entry.url)
+}
+
+/// Append crawl-discovered URLs to `entries`, skipping any that are off-origin,
+/// unparseable, or already present (by normalized form).
+///
+/// Split out from [`discover_urls`] so the union/dedup/same-origin logic is
+/// unit-testable without touching the network. Mutates `entries` and `seen` in
+/// place; crawl URLs get empty metadata.
+fn append_crawled(
+    entries: &mut Vec<SitemapEntry>,
+    seen: &mut HashSet<String>,
+    discovered: impl IntoIterator<Item = String>,
+    base_origin: &str,
+) {
+    for raw in discovered {
+        let Ok(parsed) = Url::parse(&raw) else {
+            continue;
+        };
+        // Same-origin filter: drop anything whose origin differs from the seed.
+        if crawler::origin_key(&parsed) != base_origin {
+            continue;
+        }
+        let norm = crawler::normalize(&parsed);
+        if seen.insert(norm.clone()) {
+            entries.push(SitemapEntry {
+                url: norm,
+                last_modified: None,
+                priority: None,
+                change_freq: None,
+            });
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn entry(url: &str) -> SitemapEntry {
+        SitemapEntry {
+            url: url.to_string(),
+            last_modified: None,
+            priority: None,
+            change_freq: None,
+        }
+    }
+
+    fn origin_of(url: &str) -> String {
+        crawler::origin_key(&Url::parse(url).unwrap())
+    }
+
+    #[test]
+    fn append_adds_new_same_origin_urls() {
+        let mut entries = vec![entry("https://example.com/")];
+        let mut seen: HashSet<String> = entries.iter().filter_map(normalize_str).collect();
+
+        append_crawled(
+            &mut entries,
+            &mut seen,
+            vec![
+                "https://example.com/about".to_string(),
+                "https://example.com/contact".to_string(),
+            ],
+            &origin_of("https://example.com"),
+        );
+
+        let urls: Vec<&str> = entries.iter().map(|e| e.url.as_str()).collect();
+        assert_eq!(
+            urls,
+            vec![
+                "https://example.com/",
+                "https://example.com/about",
+                "https://example.com/contact",
+            ]
+        );
+    }
+
+    #[test]
+    fn append_dedups_against_sitemap_and_self() {
+        let mut entries = vec![entry("https://example.com/about")];
+        let mut seen: HashSet<String> = entries.iter().filter_map(normalize_str).collect();
+
+        append_crawled(
+            &mut entries,
+            &mut seen,
+            vec![
+                // Same as sitemap entry (trailing slash normalizes away).
+                "https://example.com/about/".to_string(),
+                // Fragment + duplicate -> only one new entry survives.
+                "https://example.com/new#frag".to_string(),
+                "https://example.com/new".to_string(),
+            ],
+            &origin_of("https://example.com"),
+        );
+
+        let urls: Vec<&str> = entries.iter().map(|e| e.url.as_str()).collect();
+        assert_eq!(
+            urls,
+            vec!["https://example.com/about", "https://example.com/new"]
+        );
+    }
+
+    #[test]
+    fn append_filters_off_origin() {
+        let mut entries = Vec::new();
+        let mut seen = HashSet::new();
+
+        append_crawled(
+            &mut entries,
+            &mut seen,
+            vec![
+                "https://example.com/keep".to_string(),
+                "https://evil.com/drop".to_string(),
+                "https://sub.example.com/drop".to_string(), // different origin
+                "ftp://example.com/drop".to_string(),       // unparseable as http origin match
+            ],
+            &origin_of("https://example.com"),
+        );
+
+        let urls: Vec<&str> = entries.iter().map(|e| e.url.as_str()).collect();
+        assert_eq!(urls, vec!["https://example.com/keep"]);
+    }
+
+    #[test]
+    fn append_treats_www_as_same_origin() {
+        // origin_key strips a leading `www.`, so www and apex collapse.
+        let mut entries = Vec::new();
+        let mut seen = HashSet::new();
+
+        append_crawled(
+            &mut entries,
+            &mut seen,
+            vec!["https://www.example.com/page".to_string()],
+            &origin_of("https://example.com"),
+        );
+
+        assert_eq!(entries.len(), 1);
+    }
+
+    #[test]
+    fn crawl_urls_carry_no_metadata() {
+        let mut entries = Vec::new();
+        let mut seen = HashSet::new();
+
+        append_crawled(
+            &mut entries,
+            &mut seen,
+            vec!["https://example.com/x".to_string()],
+            &origin_of("https://example.com"),
+        );
+
+        assert_eq!(entries.len(), 1);
+        assert!(entries[0].last_modified.is_none());
+        assert!(entries[0].priority.is_none());
+        assert!(entries[0].change_freq.is_none());
+    }
+
+    #[test]
+    fn map_options_defaults() {
+        let o = MapOptions::default();
+        assert_eq!(o.max_crawl_pages, 150);
+        assert_eq!(o.crawl_depth, 2);
+        assert_eq!(o.min_sitemap_urls, 200);
+        assert!(o.crawl_fallback);
+    }
+}

crates/webclaw-fetch/src/search.rs

@@ -0,0 +1,322 @@
+//! Web search via Serper.dev (Google results) with optional content scraping.
+//!
+//! This is the self-hosted search path: the caller supplies their own
+//! Serper.dev API key (free tier at serper.dev). The CLI, MCP server, and
+//! OSS REST server all route through [`search`] so search works without the
+//! hosted webclaw API.
+//!
+//! Serper returns a plain JSON API, so we hit it with a vanilla wreq client
+//! (10s timeout) — no browser TLS fingerprinting needed. When `scrape` is
+//! set, the top results are fetched through the caller's [`FetchClient`]
+//! (which *does* carry the fingerprinting) and extracted to markdown.
+use std::sync::Arc;
+use std::time::Duration;
+
+use serde::{Deserialize, Serialize};
+use serde_json::{Value, json};
+use tokio::sync::Semaphore;
+use tracing::warn;
+
+use crate::client::FetchClient;
+use crate::error::FetchError;
+
+/// Serper.dev search endpoint.
+const SERPER_URL: &str = "https://google.serper.dev/search";
+
+/// Bound on the number of result pages scraped concurrently when
+/// `scrape` is enabled. Keeps the fan-out from overwhelming the proxy
+/// pool / remote hosts on a large result set.
+const SCRAPE_CONCURRENCY: usize = 5;
+
+/// Options controlling a search request.
+#[derive(Debug, Clone)]
+pub struct SearchOptions {
+    /// Number of organic results to request (clamped to `1..=10`).
+    pub num_results: usize,
+    /// Country code for localization (Serper `gl`, e.g. `"us"`, `"gb"`).
+    pub country: Option<String>,
+    /// Language code for localization (Serper `hl`, e.g. `"en"`, `"it"`).
+    pub lang: Option<String>,
+    /// When true, fetch + extract the result pages and fill in `content`.
+    pub scrape: bool,
+}
+
+impl Default for SearchOptions {
+    fn default() -> Self {
+        Self {
+            num_results: 5,
+            country: None,
+            lang: None,
+            scrape: false,
+        }
+    }
+}
+
+/// A single organic search result. When `scrape` was requested and the
+/// fetch succeeded, `content` holds the extracted markdown; otherwise it
+/// is `None` (a per-result fetch failure never fails the whole search).
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct SearchResult {
+    pub title: String,
+    pub link: String,
+    pub snippet: String,
+    pub position: usize,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub content: Option<String>,
+}
+
+/// Run a web search through Serper.dev.
+///
+/// `client`     — the caller's [`FetchClient`], used only when `opts.scrape`
+///                is set (to fetch + extract the result pages).
+/// `serper_key` — the caller's Serper.dev API key.
+/// `query`      — the search query.
+/// `opts`       — result count, localization, and whether to scrape.
+///
+/// Returns the organic results in Serper's order. With `scrape` enabled,
+/// the top results are fetched concurrently (bounded) and their extracted
+/// markdown is attached to `content`.
+pub async fn search(
+    client: &FetchClient,
+    serper_key: &str,
+    query: &str,
+    opts: &SearchOptions,
+) -> Result<Vec<SearchResult>, FetchError> {
+    let num = opts.num_results.clamp(1, 10);
+
+    let response = call_serper(
+        serper_key,
+        query,
+        num,
+        opts.country.as_deref(),
+        opts.lang.as_deref(),
+    )
+    .await?;
+
+    let mut results = parse_serper_organic(&response);
+
+    if opts.scrape && !results.is_empty() {
+        scrape_results(client, &mut results).await;
+    }
+
+    Ok(results)
+}
+
+/// POST the query to Serper.dev and return the raw JSON response.
+///
+/// Builds a plain wreq client (no browser emulation — Serper is a JSON
+/// API, not a bot-protected page). Non-2xx responses are surfaced as a
+/// [`FetchError::Build`] carrying the status and body so the caller can
+/// show Serper's own error (bad key, quota exceeded, etc.).
+async fn call_serper(
+    api_key: &str,
+    query: &str,
+    num: usize,
+    country: Option<&str>,
+    lang: Option<&str>,
+) -> Result<Value, FetchError> {
+    let http = wreq::Client::builder()
+        .timeout(Duration::from_secs(10))
+        .build()
+        .map_err(|e| FetchError::Build(format!("failed to build serper client: {e}")))?;
+
+    let mut body = json!({ "q": query, "num": num });
+    if let Some(gl) = country {
+        body["gl"] = json!(gl);
+    }
+    if let Some(hl) = lang {
+        body["hl"] = json!(hl);
+    }
+    // Serialize ourselves rather than `.json()` — the wreq `json` feature
+    // is not enabled in this crate and isn't worth pulling in for one call.
+    let payload = serde_json::to_vec(&body)
+        .map_err(|e| FetchError::Build(format!("serper request encode error: {e}")))?;
+
+    let resp = http
+        .post(SERPER_URL)
+        .header("X-API-KEY", api_key)
+        .header("Content-Type", "application/json")
+        .body(payload)
+        .send()
+        .await?;
+
+    let status = resp.status();
+    if !status.is_success() {
+        let code = status.as_u16();
+        let text = resp.text().await.unwrap_or_default();
+        return Err(FetchError::Build(format!("serper returned {code}: {text}")));
+    }
+
+    let text = resp
+        .text()
+        .await
+        .map_err(|e| FetchError::BodyDecode(format!("serper response read error: {e}")))?;
+    serde_json::from_str::<Value>(&text)
+        .map_err(|e| FetchError::BodyDecode(format!("serper response parse error: {e}")))
+}
+
+/// Parse the `organic` array of a Serper response into [`SearchResult`]s.
+///
+/// Pure (no network), so it is unit-tested against a fixture. Entries
+/// missing `title` or `link` are skipped; `snippet` defaults to empty.
+/// `position` is 1-based over the kept entries.
+pub fn parse_serper_organic(response: &Value) -> Vec<SearchResult> {
+    let Some(organic) = response.get("organic").and_then(|v| v.as_array()) else {
+        return Vec::new();
+    };
+
+    organic
+        .iter()
+        .filter_map(|item| {
+            let title = item.get("title")?.as_str()?.to_string();
+            let link = item.get("link")?.as_str()?.to_string();
+            let snippet = item
+                .get("snippet")
+                .and_then(|v| v.as_str())
+                .unwrap_or("")
+                .to_string();
+            Some(SearchResult {
+                title,
+                link,
+                snippet,
+                // Filled in after collection so it tracks kept entries,
+                // not the raw array index (which may include skips).
+                position: 0,
+                content: None,
+            })
+        })
+        .enumerate()
+        .map(|(i, mut r)| {
+            r.position = i + 1;
+            r
+        })
+        .collect()
+}
+
+/// Fetch + extract the result pages and attach markdown to `content`.
+///
+/// Bounded by [`SCRAPE_CONCURRENCY`]. A per-result fetch or extraction
+/// failure leaves that result's `content` as `None` rather than failing
+/// the whole search.
+async fn scrape_results(client: &FetchClient, results: &mut [SearchResult]) {
+    let sem = Arc::new(Semaphore::new(SCRAPE_CONCURRENCY));
+
+    // Collect owned links first so the per-result futures don't borrow
+    // `results`. That keeps the future captures free of the slice's
+    // lifetime, which is what lets this compile inside the MCP `#[tool]`
+    // macro's stricter `Send`/lifetime bounds.
+    let links: Vec<String> = results.iter().map(|r| r.link.clone()).collect();
+
+    let scrapes = links.into_iter().map(|link| {
+        let sem = sem.clone();
+        async move {
+            // If the semaphore is closed (shutdown race), skip rather than panic.
+            let _permit = match sem.acquire().await {
+                Ok(p) => p,
+                Err(_) => return None,
+            };
+            match client.fetch(&link).await {
+                Ok(fetched) => match webclaw_core::extract(&fetched.html, Some(&fetched.url)) {
+                    Ok(extraction) => Some(extraction.content.markdown),
+                    Err(e) => {
+                        warn!(url = %link, error = %e, "search: extraction failed");
+                        None
+                    }
+                },
+                Err(e) => {
+                    warn!(url = %link, error = %e, "search: fetch failed");
+                    None
+                }
+            }
+        }
+    });
+
+    // `join_all` drives every scrape future concurrently and returns
+    // results in input order; the semaphore caps how many fetches run at
+    // once. Result set is tiny (≤10), so the all-at-once poll is fine.
+    let contents = futures_util::future::join_all(scrapes).await;
+    for (r, content) in results.iter_mut().zip(contents) {
+        r.content = content;
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn fixture() -> Value {
+        json!({
+            "searchParameters": { "q": "rust async", "type": "search" },
+            "organic": [
+                {
+                    "title": "Async Rust",
+                    "link": "https://example.com/async",
+                    "snippet": "Learn async in Rust.",
+                    "position": 1
+                },
+                {
+                    // snippet missing on purpose -> defaults to ""
+                    "title": "Tokio",
+                    "link": "https://tokio.rs"
+                },
+                {
+                    // no link -> skipped, must not shift positions of the rest
+                    "title": "No Link Here"
+                }
+            ]
+        })
+    }
+
+    #[test]
+    fn parses_organic_results() {
+        let results = parse_serper_organic(&fixture());
+        assert_eq!(results.len(), 2);
+
+        assert_eq!(results[0].title, "Async Rust");
+        assert_eq!(results[0].link, "https://example.com/async");
+        assert_eq!(results[0].snippet, "Learn async in Rust.");
+        assert_eq!(results[0].position, 1);
+        assert!(results[0].content.is_none());
+
+        // Missing snippet -> empty string, and position is 1-based over
+        // kept entries (the link-less entry is dropped, not counted).
+        assert_eq!(results[1].title, "Tokio");
+        assert_eq!(results[1].snippet, "");
+        assert_eq!(results[1].position, 2);
+    }
+
+    #[test]
+    fn missing_organic_key_yields_empty() {
+        assert!(parse_serper_organic(&json!({})).is_empty());
+        assert!(parse_serper_organic(&json!({ "organic": "not-an-array" })).is_empty());
+    }
+
+    #[test]
+    fn search_result_serializes_without_null_content() {
+        let r = SearchResult {
+            title: "T".into(),
+            link: "https://e.com".into(),
+            snippet: "s".into(),
+            position: 1,
+            content: None,
+        };
+        let v = serde_json::to_value(&r).unwrap();
+        assert!(v.get("content").is_none(), "None content should be skipped");
+
+        let r2 = SearchResult {
+            content: Some("# md".into()),
+            ..r
+        };
+        let v2 = serde_json::to_value(&r2).unwrap();
+        assert_eq!(v2.get("content").and_then(|c| c.as_str()), Some("# md"));
+    }
+
+    #[test]
+    fn default_options() {
+        let o = SearchOptions::default();
+        assert_eq!(o.num_results, 5);
+        assert!(!o.scrape);
+        assert!(o.country.is_none());
+        assert!(o.lang.is_none());
+    }
+}

crates/webclaw-fetch/src/sitemap.rs

@@ -18,12 +18,20 @@ use crate::error::FetchError;
 
 /// Maximum depth when recursively fetching sitemap index files.
 /// Prevents infinite loops from circular sitemap references.
-const MAX_RECURSION_DEPTH: usize = 3;
+///
+/// Raised 3→5: large sites (gov.uk, news publishers) nest sitemap indexes
+/// more than three levels deep — a top index → per-section index →
+/// per-month index → urlset is already four hops. Three cut those off.
+const MAX_RECURSION_DEPTH: usize = 5;
 
 /// Common sitemap paths to try when robots.txt doesn't list any.
 const FALLBACK_SITEMAP_PATHS: &[&str] = &[
     "/sitemap.xml",
     "/sitemap_index.xml",
+    "/sitemap-index.xml",
+    "/sitemap1.xml",
+    "/sitemaps.xml",
+    "/sitemap/index.xml",
     "/wp-sitemap.xml",
     "/sitemap/sitemap-index.xml",
 ];
@@ -105,10 +113,12 @@ async fn fetch_sitemaps(
     for sitemap_url in urls {
         debug!(url = %sitemap_url, depth, "fetching sitemap");
 
-        let xml = match client.fetch(sitemap_url).await {
-            Ok(result) if result.status == 200 => result.html,
-            Ok(result) => {
-                debug!(url = %sitemap_url, status = result.status, "sitemap not found");
+        // Fetch raw bytes so gzipped sitemaps survive intact. `fetch` runs
+        // the body through `from_utf8_lossy`, which corrupts binary gzip.
+        let body = match client.fetch_raw(sitemap_url).await {
+            Ok((200, body)) => body,
+            Ok((status, _)) => {
+                debug!(url = %sitemap_url, status, "sitemap not found");
                 continue;
             }
             Err(e) => {
@@ -117,6 +127,14 @@ async fn fetch_sitemaps(
             }
         };
 
+        let xml = match decode_sitemap_body(&body) {
+            Some(xml) => xml,
+            None => {
+                debug!(url = %sitemap_url, "failed to decode sitemap body, skipping");
+                continue;
+            }
+        };
+
         match detect_sitemap_type(&xml) {
             SitemapType::UrlSet => {
                 let parsed = parse_urlset(&xml);
@@ -147,6 +165,33 @@ async fn fetch_sitemaps(
     }
 }
 
+/// Decode a raw sitemap body into a UTF-8 XML string.
+///
+/// Sitemaps are commonly served gzipped (`.xml.gz`) with
+/// `Content-Type: application/gzip` and *no* `Content-Encoding`, so the HTTP
+/// layer never inflates them. We detect the gzip magic bytes (`0x1f 0x8b`)
+/// and gunzip in-process; otherwise the body is treated as plain XML.
+///
+/// Returns `None` if a gzip stream fails to inflate. Plain (non-gzip) bodies
+/// always succeed via lossy UTF-8 decode, mirroring the previous behaviour.
+pub(crate) fn decode_sitemap_body(body: &[u8]) -> Option<String> {
+    if body.starts_with(&[0x1f, 0x8b]) {
+        use std::io::Read;
+
+        let mut decoder = flate2::read::GzDecoder::new(body);
+        let mut out = String::new();
+        match decoder.read_to_string(&mut out) {
+            Ok(_) => Some(out),
+            Err(e) => {
+                warn!(error = %e, "failed to gunzip sitemap body");
+                None
+            }
+        }
+    } else {
+        Some(String::from_utf8_lossy(body).into_owned())
+    }
+}
+
 // ---------------------------------------------------------------------------
 // Pure parsing functions (no I/O, fully testable)
 // ---------------------------------------------------------------------------
@@ -669,5 +714,47 @@ mod tests {
         assert!(FALLBACK_SITEMAP_PATHS.contains(&"/sitemap_index.xml"));
         assert!(FALLBACK_SITEMAP_PATHS.contains(&"/wp-sitemap.xml"));
         assert!(FALLBACK_SITEMAP_PATHS.contains(&"/sitemap/sitemap-index.xml"));
+        // Paths added for robustness (item 3).
+        assert!(FALLBACK_SITEMAP_PATHS.contains(&"/sitemap-index.xml"));
+        assert!(FALLBACK_SITEMAP_PATHS.contains(&"/sitemap1.xml"));
+        assert!(FALLBACK_SITEMAP_PATHS.contains(&"/sitemaps.xml"));
+        assert!(FALLBACK_SITEMAP_PATHS.contains(&"/sitemap/index.xml"));
+    }
+
+    #[test]
+    fn decode_plain_xml_body() {
+        let xml = r#"<?xml version="1.0"?><urlset></urlset>"#;
+        let got = decode_sitemap_body(xml.as_bytes()).expect("plain body decodes");
+        assert_eq!(got, xml);
+    }
+
+    #[test]
+    fn decode_gzipped_body() {
+        use std::io::Write;
+
+        let xml = r#"<?xml version="1.0"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <url><loc>https://example.com/gz-page</loc></url>
+</urlset>"#;
+
+        // Gzip-compress the XML, then confirm decode_sitemap_body inflates it
+        // and the parser finds the URL.
+        let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
+        encoder.write_all(xml.as_bytes()).unwrap();
+        let gz = encoder.finish().unwrap();
+
+        assert_eq!(&gz[..2], &[0x1f, 0x8b], "gzip magic present");
+
+        let decoded = decode_sitemap_body(&gz).expect("gzip body inflates");
+        let entries = parse_urlset(&decoded);
+        assert_eq!(entries.len(), 1);
+        assert_eq!(entries[0].url, "https://example.com/gz-page");
+    }
+
+    #[test]
+    fn decode_corrupt_gzip_returns_none() {
+        // Starts with gzip magic but the rest is garbage -> inflate fails.
+        let bad = [0x1f, 0x8b, 0x08, 0x00, 0xde, 0xad, 0xbe, 0xef];
+        assert!(decode_sitemap_body(&bad).is_none());
     }
 }

crates/webclaw-fetch/src/tls.rs

@@ -10,15 +10,24 @@ use std::{borrow::Cow, io, time::Duration};
 use wreq::http2::{
     Http2Options, PseudoId, PseudoOrder, SettingId, SettingsOrder, StreamDependency, StreamId,
 };
-use wreq::tls::{
-    AlpnProtocol, AlpsProtocol, CertificateCompressionAlgorithm, ExtensionType, TlsOptions,
-    TlsVersion,
-};
-use wreq::{Client, Emulation};
+use wreq::tls::compress::CertificateCompressor;
+use wreq::tls::{AlpnProtocol, AlpsProtocol, ExtensionType, TlsOptions, TlsVersion};
+use wreq::{Client, Emulation, Group, IntoEmulation};
+use wreq_util::emulate::compress::{BrotliCompressor, ZlibCompressor};
 
 use crate::browser::BrowserVariant;
 use crate::error::FetchError;
 
+// Certificate-compression advertisement per profile. wreq 6.0.0-rc.29 replaced
+// the `CertificateCompressionAlgorithm` enum argument with `&dyn
+// CertificateCompressor` trait objects; wreq-util ships the concrete zlib/brotli
+// implementations. The advertised set (and order) is a TLS fingerprint signal,
+// so these mirror the previous enum lists exactly.
+static CHROME_CERT_COMPRESSORS: &[&'static dyn CertificateCompressor] = &[&BrotliCompressor];
+static FIREFOX_CERT_COMPRESSORS: &[&'static dyn CertificateCompressor] =
+    &[&ZlibCompressor, &BrotliCompressor];
+static SAFARI_CERT_COMPRESSORS: &[&'static dyn CertificateCompressor] = &[&ZlibCompressor];
+
 #[derive(Clone, Default)]
 struct PublicDnsResolver;
 
@@ -119,7 +128,7 @@ fn chrome_extensions() -> Vec<ExtensionType> {
         ExtensionType::PSK_KEY_EXCHANGE_MODES,                 // 45
         ExtensionType::EC_POINT_FORMATS,                       // 11
         ExtensionType::CERT_COMPRESSION,                       // 27
-        ExtensionType::APPLICATION_SETTINGS_NEW, // 17613 (new codepoint, matches alps_use_new_codepoint)
+        ExtensionType::APPLICATION_SETTINGS, // 17613 (new codepoint, matches alps_use_new_codepoint)
         ExtensionType::SUPPORTED_VERSIONS,   // 43
         ExtensionType::SIGNATURE_ALGORITHMS, // 13
         ExtensionType::SERVER_NAME,          // 0
@@ -287,7 +296,7 @@ fn chrome_tls() -> TlsOptions {
         .alps_protocols([AlpsProtocol::HTTP3, AlpsProtocol::HTTP2])
         .alps_use_new_codepoint(true)
         .aes_hw_override(true)
-        .certificate_compression_algorithms(&[CertificateCompressionAlgorithm::BROTLI])
+        .certificate_compressors(CHROME_CERT_COMPRESSORS)
         .build()
 }
 
@@ -304,10 +313,7 @@ fn firefox_tls() -> TlsOptions {
         .pre_shared_key(true)
         .enable_ocsp_stapling(true)
         .enable_signed_cert_timestamps(true)
-        .certificate_compression_algorithms(&[
-            CertificateCompressionAlgorithm::ZLIB,
-            CertificateCompressionAlgorithm::BROTLI,
-        ])
+        .certificate_compressors(FIREFOX_CERT_COMPRESSORS)
         .build()
 }
 
@@ -324,7 +330,7 @@ fn safari_tls() -> TlsOptions {
         .pre_shared_key(false)
         .enable_ocsp_stapling(true)
         .enable_signed_cert_timestamps(true)
-        .certificate_compression_algorithms(&[CertificateCompressionAlgorithm::ZLIB])
+        .certificate_compressors(SAFARI_CERT_COMPRESSORS)
         .build()
 }
 
@@ -345,21 +351,23 @@ fn safari_tls() -> TlsOptions {
 ///     `priority: u=0, i`, zstd), replace with the real iOS 26 set.
 ///  4. `accept-language` preserved from config.extra_headers for locale.
 fn safari_ios_emulation() -> wreq::Emulation {
-    use wreq::EmulationFactory;
-    let mut em = wreq_util::Emulation::SafariIos26.emulation();
+    // wreq 6.0.0-rc.29 exposes the `Emulation` fields directly (no `*_mut()`
+    // accessors) and wreq-util 3.0.0-rc.12 renamed the enum to `Profile` with
+    // `IntoEmulation::into_emulation` replacing `EmulationFactory::emulation`.
+    let mut em = wreq_util::Profile::SafariIos26.into_emulation();
 
-    if let Some(tls) = em.tls_options_mut().as_mut() {
+    if let Some(tls) = em.tls_options.as_mut() {
         tls.extension_permutation = Some(Cow::Owned(safari_ios_extensions()));
     }
 
     // Only override the priority flag — keep wreq-util's SETTINGS, WINDOW_UPDATE,
     // and pseudo-order intact. Replacing the whole Http2Options resets SETTINGS
     // to defaults, which sends only INITIAL_WINDOW_SIZE and fails DataDome.
-    if let Some(h2) = em.http2_options_mut().as_mut() {
+    if let Some(h2) = em.http2_options.as_mut() {
         h2.headers_stream_dependency = Some(StreamDependency::new(StreamId::zero(), 255, true));
     }
 
-    let hm = em.headers_mut();
+    let hm = &mut em.headers;
     hm.clear();
     for (k, v) in SAFARI_IOS_HEADERS {
         if let (Ok(n), Ok(val)) = (
@@ -508,12 +516,12 @@ pub fn build_client(
                 .tls_options(tls)
                 .http2_options(h2)
                 .headers(build_headers(headers))
-                .build()
+                .build(Group::default())
         }
     };
 
     // Append extra headers after profile defaults.
-    let hm = emulation.headers_mut();
+    let hm = &mut emulation.headers;
     for (k, v) in extra_headers {
         if let (Ok(n), Ok(val)) = (
             http::header::HeaderName::from_bytes(k.as_bytes()),
@@ -530,7 +538,11 @@ pub fn build_client(
             max_redirects as usize,
         ))
         .cookie_store(true)
-        .timeout(timeout);
+        .timeout(timeout)
+        .connect_timeout(Duration::from_secs(5))
+        .pool_idle_timeout(Duration::from_secs(90))
+        .pool_max_idle_per_host(8)
+        .tcp_keepalive(Duration::from_secs(60));
 
     if let Some(proxy_url) = proxy {
         let proxy = wreq::Proxy::all(proxy_url).map_err(|_| {

crates/webclaw-llm/src/chain.rs

@@ -1,5 +1,5 @@
 /// Provider chain — tries providers in order until one succeeds.
-/// Default order: Ollama (local, free) -> OpenAI -> Anthropic.
+/// Default order: Ollama (local, free) -> OpenAI -> Gemini -> Anthropic -> Atlas Cloud (opt-in).
 /// Only includes providers that are actually configured/available.
 use async_trait::async_trait;
 use tracing::{debug, warn};
@@ -7,7 +7,8 @@ use tracing::{debug, warn};
 use crate::error::LlmError;
 use crate::provider::{CompletionRequest, LlmProvider};
 use crate::providers::{
-    anthropic::AnthropicProvider, ollama::OllamaProvider, openai::OpenAiProvider,
+    anthropic::AnthropicProvider, atlascloud::AtlasCloudProvider, gemini::GeminiProvider,
+    ollama::OllamaProvider, openai::OpenAiProvider,
 };
 
 pub struct ProviderChain {
@@ -15,9 +16,13 @@ pub struct ProviderChain {
 }
 
 impl ProviderChain {
-    /// Build the default chain: Ollama -> OpenAI -> Anthropic.
+    /// Build the default chain: Ollama -> OpenAI -> Gemini -> Anthropic -> Atlas Cloud.
     /// Ollama is always added (availability checked at call time).
     /// Cloud providers are only added if their API keys are configured.
+    /// Gemini sits ahead of Anthropic so Google Cloud credits are preferred,
+    /// with Anthropic as the last-resort fallback. Atlas Cloud is opt-in and
+    /// added last (only when `ATLASCLOUD_API_KEY` is set), so it never preempts
+    /// an already-configured provider.
     pub async fn default() -> Self {
         let mut providers: Vec<Box<dyn LlmProvider>> = Vec::new();
 
@@ -34,11 +39,21 @@ impl ProviderChain {
             providers.push(Box::new(openai));
         }
 
+        if let Some(gemini) = GeminiProvider::new(None, None, None) {
+            debug!("gemini configured, adding to chain");
+            providers.push(Box::new(gemini));
+        }
+
         if let Some(anthropic) = AnthropicProvider::with_base_url(None, None, None) {
             debug!("anthropic configured, adding to chain");
             providers.push(Box::new(anthropic));
         }
 
+        if let Some(atlas) = AtlasCloudProvider::new(None, None, None) {
+            debug!("atlascloud configured, adding to chain");
+            providers.push(Box::new(atlas));
+        }
+
         Self { providers }
     }
 

crates/webclaw-llm/src/lib.rs

@@ -1,6 +1,6 @@
 /// webclaw-llm: LLM integration with local-first hybrid architecture.
 ///
-/// Provider chain tries Ollama (local) first, falls back to OpenAI, then Anthropic.
+/// Provider chain tries Ollama (local) first, falls back to OpenAI, then Gemini, then Anthropic.
 /// Provides schema-based extraction, prompt extraction, and summarization
 /// on top of webclaw-core's content pipeline.
 pub mod chain;

crates/webclaw-llm/src/providers/anthropic.rs

@@ -1,6 +1,8 @@
 /// Anthropic provider — Claude models via api.anthropic.com.
 /// Anthropic's API differs from OpenAI: system message is a top-level param,
 /// not part of the messages array.
+use std::time::Duration;
+
 use async_trait::async_trait;
 use serde_json::json;
 
@@ -35,14 +37,20 @@ impl AnthropicProvider {
         let key = load_api_key(key_override, "ANTHROPIC_API_KEY")?;
 
         Some(Self {
-            client: reqwest::Client::new(),
+            client: reqwest::Client::builder()
+                .timeout(Duration::from_secs(120))
+                .connect_timeout(Duration::from_secs(10))
+                .build()
+                .unwrap_or_else(|_| reqwest::Client::new()),
             key,
             base_url: base_url
                 .or_else(|| std::env::var("ANTHROPIC_BASE_URL").ok())
                 .unwrap_or_else(|| DEFAULT_ANTHROPIC_BASE_URL.into())
                 .trim_end_matches('/')
                 .to_string(),
-            default_model: model.unwrap_or_else(|| "claude-sonnet-4-20250514".into()),
+            default_model: model
+                .or_else(|| std::env::var("ANTHROPIC_MODEL").ok())
+                .unwrap_or_else(|| "claude-sonnet-4-6".into()),
         })
     }
 
@@ -108,11 +116,7 @@ impl LlmProvider for AnthropicProvider {
         if !resp.status().is_success() {
             let status = resp.status();
             let text = resp.text().await.unwrap_or_default();
-            let safe_text = if text.len() > 500 {
-                &text[..500]
-            } else {
-                &text
-            };
+            let safe_text = text.chars().take(500).collect::<String>();
             return Err(LlmError::ProviderError(format!(
                 "anthropic returned {status}: {safe_text}"
             )));
@@ -156,7 +160,7 @@ mod tests {
         let provider =
             AnthropicProvider::new(Some("sk-ant-test".into()), None).expect("should construct");
         assert_eq!(provider.name(), "anthropic");
-        assert_eq!(provider.default_model, "claude-sonnet-4-20250514");
+        assert_eq!(provider.default_model, "claude-sonnet-4-6");
         assert_eq!(provider.key, "sk-ant-test");
         assert_eq!(provider.base_url, "https://api.anthropic.com/v1");
         assert_eq!(
@@ -176,7 +180,7 @@ mod tests {
     #[test]
     fn default_model_accessor() {
         let provider = AnthropicProvider::new(Some("sk-ant-test".into()), None).unwrap();
-        assert_eq!(provider.default_model(), "claude-sonnet-4-20250514");
+        assert_eq!(provider.default_model(), "claude-sonnet-4-6");
     }
 
     #[test]

crates/webclaw-llm/src/providers/atlascloud.rs

@@ -0,0 +1,78 @@
+/// Atlas Cloud provider — OpenAI-compatible chat completions with Atlas defaults.
+use async_trait::async_trait;
+
+use crate::error::LlmError;
+use crate::provider::{CompletionRequest, LlmProvider};
+
+use super::openai::OpenAiProvider;
+
+pub struct AtlasCloudProvider {
+    inner: OpenAiProvider,
+}
+
+impl AtlasCloudProvider {
+    /// Returns `None` if no Atlas Cloud API key is available (param or env).
+    pub fn new(
+        key_override: Option<String>,
+        base_url: Option<String>,
+        model: Option<String>,
+    ) -> Option<Self> {
+        let key = super::load_api_key(key_override, "ATLASCLOUD_API_KEY")?;
+        let base_url = base_url
+            .or_else(|| std::env::var("ATLASCLOUD_BASE_URL").ok())
+            .unwrap_or_else(|| "https://api.atlascloud.ai/v1".into());
+        let model = model
+            .or_else(|| std::env::var("ATLASCLOUD_MODEL").ok())
+            .unwrap_or_else(|| "qwen/qwen3.5-flash".into());
+        let inner = OpenAiProvider::new(Some(key), Some(base_url), Some(model))?;
+        Some(Self { inner })
+    }
+
+    pub fn default_model(&self) -> &str {
+        self.inner.default_model()
+    }
+}
+
+#[async_trait]
+impl LlmProvider for AtlasCloudProvider {
+    async fn complete(&self, request: &CompletionRequest) -> Result<String, LlmError> {
+        self.inner.complete(request).await
+    }
+
+    async fn is_available(&self) -> bool {
+        self.inner.is_available().await
+    }
+
+    fn name(&self) -> &str {
+        "atlascloud"
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn empty_key_returns_none() {
+        assert!(AtlasCloudProvider::new(Some(String::new()), None, None).is_none());
+    }
+
+    #[test]
+    fn explicit_key_constructs_with_atlas_defaults() {
+        let provider =
+            AtlasCloudProvider::new(Some("test-key".into()), None, None).expect("should construct");
+        assert_eq!(provider.name(), "atlascloud");
+        assert_eq!(provider.default_model(), "qwen/qwen3.5-flash");
+    }
+
+    #[test]
+    fn explicit_model_override() {
+        let provider = AtlasCloudProvider::new(
+            Some("test-key".into()),
+            Some("https://proxy.example.com/v1".into()),
+            Some("deepseek-ai/deepseek-v4-pro".into()),
+        )
+        .expect("should construct");
+        assert_eq!(provider.default_model(), "deepseek-ai/deepseek-v4-pro");
+    }
+}

crates/webclaw-llm/src/providers/gemini.rs

@@ -0,0 +1,363 @@
+/// Google Gemini provider — Gemini models via the Generative Language API.
+/// Gemini's request shape differs from OpenAI/Anthropic: the system message is a
+/// top-level `systemInstruction`, conversation turns live in `contents` (with the
+/// assistant role renamed to `model`), and generation knobs sit under
+/// `generationConfig`. API-key auth is sent as an `x-goog-api-key` header.
+use std::time::Duration;
+
+use async_trait::async_trait;
+use serde_json::json;
+
+use crate::clean::strip_thinking_tags;
+use crate::error::LlmError;
+use crate::provider::{CompletionRequest, LlmProvider};
+
+use super::load_api_key;
+
+const DEFAULT_GEMINI_BASE_URL: &str = "https://generativelanguage.googleapis.com/v1beta";
+/// Default model. Gemini 2.5 Flash/Pro are "thinking" models: internal reasoning
+/// tokens count against `maxOutputTokens`, so the output budget must comfortably
+/// exceed the visible response (see `request_body`) or the model returns
+/// `finishReason=MAX_TOKENS` with no text. Set `GEMINI_MODEL` to a non-thinking
+/// model (e.g. `gemini-2.0-flash`) to avoid the reasoning overhead entirely.
+const DEFAULT_GEMINI_MODEL: &str = "gemini-2.5-flash";
+
+/// Gemini puts the model in the URL path, so only plain model identifiers are
+/// safe to interpolate. Real model names are ASCII alphanumerics plus `-`/`.`/`_`
+/// (e.g. `gemini-2.5-flash`, `gemini-2.0-flash-001`); anything else (`/`, `:`,
+/// `?`, `#`, whitespace) could redirect the request to a different path/method.
+fn is_safe_model_name(model: &str) -> bool {
+    !model.is_empty()
+        && model
+            .bytes()
+            .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'.' | b'_'))
+}
+
+pub struct GeminiProvider {
+    client: reqwest::Client,
+    key: String,
+    base_url: String,
+    default_model: String,
+}
+
+impl GeminiProvider {
+    /// Returns `None` if no API key is available (param or `GEMINI_API_KEY` env).
+    pub fn new(
+        key_override: Option<String>,
+        base_url: Option<String>,
+        model: Option<String>,
+    ) -> Option<Self> {
+        let key = load_api_key(key_override, "GEMINI_API_KEY")?;
+
+        Some(Self {
+            client: reqwest::Client::builder()
+                .timeout(Duration::from_secs(120))
+                .connect_timeout(Duration::from_secs(10))
+                .build()
+                .unwrap_or_else(|_| reqwest::Client::new()),
+            key,
+            base_url: base_url
+                .or_else(|| std::env::var("GEMINI_BASE_URL").ok())
+                .unwrap_or_else(|| DEFAULT_GEMINI_BASE_URL.into())
+                .trim_end_matches('/')
+                .to_string(),
+            default_model: model
+                .or_else(|| std::env::var("GEMINI_MODEL").ok())
+                .unwrap_or_else(|| DEFAULT_GEMINI_MODEL.into()),
+        })
+    }
+
+    pub fn default_model(&self) -> &str {
+        &self.default_model
+    }
+
+    /// Build the `generateContent` body from a generic completion request.
+    /// System messages become `systemInstruction`; user/assistant turns become
+    /// `contents` (assistant → `model`); `json_mode` constrains the model to
+    /// valid JSON via `responseMimeType`.
+    fn request_body(&self, request: &CompletionRequest) -> serde_json::Value {
+        let contents: Vec<serde_json::Value> = request
+            .messages
+            .iter()
+            .filter(|m| m.role != "system")
+            .map(|m| {
+                let role = if m.role == "assistant" {
+                    "model"
+                } else {
+                    "user"
+                };
+                json!({ "role": role, "parts": [{ "text": m.content }] })
+            })
+            .collect();
+
+        let system_parts: Vec<serde_json::Value> = request
+            .messages
+            .iter()
+            .filter(|m| m.role == "system")
+            .map(|m| json!({ "text": m.content }))
+            .collect();
+
+        // `maxOutputTokens` is a ceiling, not a reservation — you're billed per
+        // token actually produced — so default generously. Gemini 2.5 "thinking"
+        // models spend part of this budget on internal reasoning; too low a cap
+        // makes them return `finishReason=MAX_TOKENS` with no visible text.
+        let mut generation_config = json!({
+            "maxOutputTokens": request.max_tokens.unwrap_or(8192),
+        });
+        if let Some(temp) = request.temperature {
+            generation_config["temperature"] = json!(temp);
+        }
+        if request.json_mode {
+            generation_config["responseMimeType"] = json!("application/json");
+        }
+
+        let mut body = json!({
+            "contents": contents,
+            "generationConfig": generation_config,
+        });
+
+        // Gemini rejects an empty `systemInstruction`, so only attach it when a
+        // system message is actually present.
+        if !system_parts.is_empty() {
+            body["systemInstruction"] = json!({ "parts": system_parts });
+        }
+
+        body
+    }
+}
+
+#[async_trait]
+impl LlmProvider for GeminiProvider {
+    async fn complete(&self, request: &CompletionRequest) -> Result<String, LlmError> {
+        let model = if request.model.is_empty() {
+            &self.default_model
+        } else {
+            &request.model
+        };
+
+        // The model goes in the URL path (Gemini's API requires it there, unlike
+        // OpenAI/Anthropic which take it in the body), so reject anything that
+        // isn't a plain model identifier to prevent path/query injection from a
+        // caller-supplied `request.model`.
+        if !is_safe_model_name(model) {
+            return Err(LlmError::ProviderError(format!(
+                "invalid gemini model name: {model:?}"
+            )));
+        }
+
+        let body = self.request_body(request);
+
+        // API-key auth goes in the header, never the URL, so the key can't leak
+        // into request logs, proxies, or referrer headers.
+        let url = format!("{}/models/{model}:generateContent", self.base_url);
+        let resp = self
+            .client
+            .post(&url)
+            .header("x-goog-api-key", &self.key)
+            .header("content-type", "application/json")
+            .json(&body)
+            .send()
+            .await?;
+
+        if !resp.status().is_success() {
+            let status = resp.status();
+            let text = resp.text().await.unwrap_or_default();
+            let safe_text = text.chars().take(500).collect::<String>();
+            return Err(LlmError::ProviderError(format!(
+                "gemini returned {status}: {safe_text}"
+            )));
+        }
+
+        // Cap response body size to defend against adversarial payloads.
+        let json = super::response_json_capped(resp).await?;
+
+        // Gemini response: {"candidates":[{"content":{"parts":[{"text":"..."}]}}]}.
+        // A candidate may carry multiple text parts; concatenate them in order.
+        let text = json["candidates"][0]["content"]["parts"]
+            .as_array()
+            .map(|parts| {
+                parts
+                    .iter()
+                    .filter_map(|p| p["text"].as_str())
+                    .collect::<String>()
+            })
+            .unwrap_or_default();
+
+        if text.is_empty() {
+            // No usable text. Surface Gemini's finishReason (or a prompt-level
+            // block reason) so MAX_TOKENS — e.g. a "thinking" model that spent
+            // its whole maxOutputTokens budget on reasoning — and SAFETY blocks
+            // are visible in logs/telemetry instead of masquerading as a parse
+            // failure. The chain falls through to the next provider on any Err.
+            let reason = json["candidates"][0]["finishReason"]
+                .as_str()
+                .or_else(|| json["promptFeedback"]["blockReason"].as_str())
+                .unwrap_or("unknown");
+            return Err(LlmError::ProviderError(format!(
+                "gemini returned no text (finishReason={reason})"
+            )));
+        }
+
+        Ok(strip_thinking_tags(&text))
+    }
+
+    async fn is_available(&self) -> bool {
+        !self.key.is_empty()
+    }
+
+    fn name(&self) -> &str {
+        "gemini"
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::provider::Message;
+
+    fn provider() -> GeminiProvider {
+        GeminiProvider::new(Some("test-key".into()), None, None).expect("should construct")
+    }
+
+    fn msg(role: &str, content: &str) -> Message {
+        Message {
+            role: role.into(),
+            content: content.into(),
+        }
+    }
+
+    fn request(messages: Vec<Message>, json_mode: bool) -> CompletionRequest {
+        CompletionRequest {
+            model: String::new(),
+            messages,
+            temperature: None,
+            max_tokens: None,
+            json_mode,
+        }
+    }
+
+    #[test]
+    fn empty_key_returns_none() {
+        assert!(GeminiProvider::new(Some(String::new()), None, None).is_none());
+    }
+
+    #[test]
+    fn model_name_validation_blocks_path_injection() {
+        // Real model identifiers pass.
+        assert!(is_safe_model_name("gemini-2.5-flash"));
+        assert!(is_safe_model_name("gemini-2.0-flash-001"));
+        assert!(is_safe_model_name("gemini-1.5-pro-002"));
+        // Anything that could alter the request path/method is rejected.
+        assert!(!is_safe_model_name(""));
+        assert!(!is_safe_model_name(
+            "gemini-2.5-flash:streamGenerateContent"
+        ));
+        assert!(!is_safe_model_name("../../models/x"));
+        assert!(!is_safe_model_name("model?alt=sse"));
+        assert!(!is_safe_model_name("a b"));
+    }
+
+    #[test]
+    fn explicit_key_constructs_with_defaults() {
+        let p = provider();
+        assert_eq!(p.name(), "gemini");
+        assert_eq!(p.key, "test-key");
+        assert_eq!(p.default_model, DEFAULT_GEMINI_MODEL);
+        assert_eq!(p.default_model(), DEFAULT_GEMINI_MODEL);
+        assert_eq!(p.base_url, DEFAULT_GEMINI_BASE_URL);
+    }
+
+    #[test]
+    fn custom_base_url_trims_trailing_slash_and_model() {
+        let p = GeminiProvider::new(
+            Some("test-key".into()),
+            Some("https://example.test/v1beta/".into()),
+            Some("gemini-2.5-pro".into()),
+        )
+        .unwrap();
+        assert_eq!(p.base_url, "https://example.test/v1beta");
+        assert_eq!(p.default_model, "gemini-2.5-pro");
+    }
+
+    #[test]
+    fn maps_user_and_assistant_roles_into_contents() {
+        let p = provider();
+        let body = p.request_body(&request(
+            vec![msg("user", "hello"), msg("assistant", "hi there")],
+            false,
+        ));
+        let contents = body["contents"].as_array().unwrap();
+        assert_eq!(contents.len(), 2);
+        assert_eq!(contents[0]["role"], "user");
+        assert_eq!(contents[0]["parts"][0]["text"], "hello");
+        // assistant must be renamed to Gemini's "model" role.
+        assert_eq!(contents[1]["role"], "model");
+        assert_eq!(contents[1]["parts"][0]["text"], "hi there");
+        // No system message -> no systemInstruction key at all.
+        assert!(body.get("systemInstruction").is_none());
+    }
+
+    #[test]
+    fn system_message_becomes_system_instruction_not_contents() {
+        let p = provider();
+        let body = p.request_body(&request(
+            vec![msg("system", "be terse"), msg("user", "hello")],
+            false,
+        ));
+        let contents = body["contents"].as_array().unwrap();
+        assert_eq!(contents.len(), 1, "system message lifted out of contents");
+        assert_eq!(contents[0]["role"], "user");
+        assert_eq!(body["systemInstruction"]["parts"][0]["text"], "be terse");
+    }
+
+    #[test]
+    fn json_mode_toggles_response_mime_type() {
+        let p = provider();
+        let on = p.request_body(&request(vec![msg("user", "x")], true));
+        assert_eq!(
+            on["generationConfig"]["responseMimeType"],
+            "application/json"
+        );
+        let off = p.request_body(&request(vec![msg("user", "x")], false));
+        assert!(off["generationConfig"].get("responseMimeType").is_none());
+    }
+
+    #[test]
+    fn max_output_tokens_default_and_temperature_override() {
+        let p = provider();
+        let default_body = p.request_body(&request(vec![msg("user", "x")], false));
+        assert_eq!(default_body["generationConfig"]["maxOutputTokens"], 8192);
+        // No temperature set -> key omitted.
+        assert!(
+            default_body["generationConfig"]
+                .get("temperature")
+                .is_none()
+        );
+
+        let mut req = request(vec![msg("user", "x")], false);
+        req.max_tokens = Some(256);
+        req.temperature = Some(0.5); // 0.5 is exact in both f32 and f64
+        let body = p.request_body(&req);
+        assert_eq!(body["generationConfig"]["maxOutputTokens"], 256);
+        assert_eq!(body["generationConfig"]["temperature"], 0.5);
+    }
+
+    // Env var fallback tests mutate process-global state and race with parallel
+    // tests. Run in isolation if needed:
+    //   cargo test -p webclaw-llm env_var -- --ignored --test-threads=1
+    #[test]
+    #[ignore = "mutates process env; run with --test-threads=1"]
+    fn env_var_key_fallback() {
+        unsafe { std::env::set_var("GEMINI_API_KEY", "gemini-env-key") };
+        let p = GeminiProvider::new(None, None, None).expect("should construct from env");
+        assert_eq!(p.key, "gemini-env-key");
+        unsafe { std::env::remove_var("GEMINI_API_KEY") };
+    }
+
+    #[test]
+    #[ignore = "mutates process env; run with --test-threads=1"]
+    fn no_key_returns_none() {
+        unsafe { std::env::remove_var("GEMINI_API_KEY") };
+        assert!(GeminiProvider::new(None, None, None).is_none());
+    }
+}

crates/webclaw-llm/src/providers/mod.rs

@@ -1,4 +1,6 @@
 pub mod anthropic;
+pub mod atlascloud;
+pub mod gemini;
 pub mod ollama;
 pub mod openai;
 

crates/webclaw-llm/src/providers/ollama.rs

@@ -1,5 +1,7 @@
 /// Ollama provider — talks to a local Ollama instance (default localhost:11434).
 /// First choice in the provider chain: free, private, fast on Apple Silicon.
+use std::time::Duration;
+
 use async_trait::async_trait;
 use serde_json::json;
 
@@ -24,7 +26,11 @@ impl OllamaProvider {
             .unwrap_or_else(|| "qwen3:8b".into());
 
         Self {
-            client: reqwest::Client::new(),
+            client: reqwest::Client::builder()
+                .timeout(Duration::from_secs(120))
+                .connect_timeout(Duration::from_secs(10))
+                .build()
+                .unwrap_or_else(|_| reqwest::Client::new()),
             base_url,
             default_model,
         }
@@ -70,11 +76,7 @@ impl LlmProvider for OllamaProvider {
         if !resp.status().is_success() {
             let status = resp.status();
             let text = resp.text().await.unwrap_or_default();
-            let safe_text = if text.len() > 500 {
-                &text[..500]
-            } else {
-                &text
-            };
+            let safe_text = text.chars().take(500).collect::<String>();
             return Err(LlmError::ProviderError(format!(
                 "ollama returned {status}: {safe_text}"
             )));
@@ -98,7 +100,8 @@ impl LlmProvider for OllamaProvider {
 
     async fn is_available(&self) -> bool {
         let url = format!("{}/api/tags", self.base_url);
-        matches!(self.client.get(&url).send().await, Ok(r) if r.status().is_success())
+        let req = self.client.get(&url).timeout(Duration::from_secs(10));
+        matches!(req.send().await, Ok(r) if r.status().is_success())
     }
 
     fn name(&self) -> &str {

crates/webclaw-llm/src/providers/openai.rs

@@ -1,4 +1,6 @@
 /// OpenAI provider — works with api.openai.com and any OpenAI-compatible endpoint.
+use std::time::Duration;
+
 use async_trait::async_trait;
 use serde_json::json;
 
@@ -69,7 +71,11 @@ impl OpenAiProvider {
         let key = load_api_key(key_override, "OPENAI_API_KEY")?;
 
         Some(Self {
-            client: reqwest::Client::new(),
+            client: reqwest::Client::builder()
+                .timeout(Duration::from_secs(120))
+                .connect_timeout(Duration::from_secs(10))
+                .build()
+                .unwrap_or_else(|_| reqwest::Client::new()),
             key,
             base_url: base_url
                 .or_else(|| std::env::var("OPENAI_BASE_URL").ok())
@@ -132,11 +138,7 @@ impl LlmProvider for OpenAiProvider {
         if !resp.status().is_success() {
             let status = resp.status();
             let text = resp.text().await.unwrap_or_default();
-            let safe_text = if text.len() > 500 {
-                &text[..500]
-            } else {
-                &text
-            };
+            let safe_text = text.chars().take(500).collect::<String>();
             return Err(LlmError::ProviderError(format!(
                 "openai returned {status}: {safe_text}"
             )));

crates/webclaw-mcp/src/server.rs

@@ -65,6 +65,10 @@ const LOCAL_FETCH_TIMEOUT: Duration = Duration::from_secs(30);
 /// Maximum poll iterations for research jobs (~10 minutes at 3s intervals).
 const RESEARCH_MAX_POLLS: u32 = 200;
 
+/// Maximum poll iterations for lead batch jobs (~10 minutes at 3s intervals);
+/// enough for a full 25-URL batch at ~20-40s per lead, 4 in parallel.
+const LEAD_BATCH_MAX_POLLS: u32 = 200;
+
 #[tool_router]
 impl WebclawMcp {
     pub async fn new() -> Self {
@@ -323,9 +327,10 @@ impl WebclawMcp {
         if params.urls.len() > 100 {
             return Err("batch is limited to 100 URLs per request".into());
         }
-        for u in &params.urls {
-            validate_url(u).await?;
-        }
+        // No up-front DNS pre-validation: it aborted the whole batch on a
+        // single unresolvable URL. The fetch layer applies the same SSRF
+        // guard (validate_public_http_url) per URL, so bad entries surface
+        // as individual per-URL errors below instead of failing the batch.
 
         let format = params.format.as_deref().unwrap_or("markdown");
         let concurrency = params.concurrency.unwrap_or(5);
@@ -667,13 +672,144 @@ impl WebclawMcp {
         ))
     }
 
-    /// Search the web for a query and return structured results. Requires WEBCLAW_API_KEY.
+    /// Enrich a company into an outreach-ready lead: the founders and leadership
+    /// with their LinkedIn and X (recovered from open-web search), plus company
+    /// summary, socials, tech stack, pricing, and public emails. Requires
+    /// WEBCLAW_API_KEY. Flat 100 credits per lead.
     #[tool]
-    async fn search(&self, Parameters(params): Parameters<SearchParams>) -> Result<String, String> {
+    async fn lead(&self, Parameters(params): Parameters<LeadParams>) -> Result<String, String> {
         let cloud = self
             .cloud
             .as_ref()
-            .ok_or("Search requires WEBCLAW_API_KEY. Get a key at https://webclaw.io")?;
+            .ok_or("Lead enrichment requires WEBCLAW_API_KEY. Get a key at https://webclaw.io")?;
+
+        let mut body = json!({ "url": params.url });
+        if let Some(no_cache) = params.no_cache {
+            body["no_cache"] = json!(no_cache);
+        }
+
+        let resp = cloud.post("lead", body).await?;
+        Ok(serde_json::to_string_pretty(&resp).unwrap_or_default())
+    }
+
+    /// Enrich many companies at once (up to 25 URLs) into outreach-ready leads.
+    /// Async: starts a batch job, polls until it finishes, and returns the
+    /// per-URL results. Requires WEBCLAW_API_KEY. Billed 100 credits per
+    /// successfully enriched lead (unreachable/empty URLs are not charged).
+    #[tool]
+    async fn lead_batch(
+        &self,
+        Parameters(params): Parameters<LeadBatchParams>,
+    ) -> Result<String, String> {
+        let cloud = self
+            .cloud
+            .as_ref()
+            .ok_or("Lead enrichment requires WEBCLAW_API_KEY. Get a key at https://webclaw.io")?;
+
+        let url_count = params.urls.len();
+        if url_count == 0 {
+            return Err("lead_batch requires at least one URL".to_string());
+        }
+
+        let mut body = json!({ "urls": params.urls });
+        if let Some(no_cache) = params.no_cache {
+            body["no_cache"] = json!(no_cache);
+        }
+
+        // Start the async batch job.
+        let start_resp = cloud.post("lead/batch", body).await?;
+        let job_id = start_resp
+            .get("id")
+            .and_then(|v| v.as_str())
+            .ok_or("Lead batch API did not return a job ID")?
+            .to_string();
+
+        info!(job_id = %job_id, urls = url_count, "lead batch started, polling for completion");
+
+        // Poll until completed or failed.
+        for poll in 0..LEAD_BATCH_MAX_POLLS {
+            tokio::time::sleep(Duration::from_secs(3)).await;
+
+            let status_resp = cloud.get(&format!("lead/batch/{job_id}")).await?;
+            let status = status_resp
+                .get("status")
+                .and_then(|v| v.as_str())
+                .unwrap_or("unknown");
+
+            match status {
+                "completed" => {
+                    return Ok(serde_json::to_string_pretty(&status_resp).unwrap_or_default());
+                }
+                "failed" => {
+                    let error = status_resp
+                        .get("error")
+                        .and_then(|v| v.as_str())
+                        .unwrap_or("unknown error");
+                    return Err(format!("Lead batch job failed: {error}"));
+                }
+                _ => {
+                    if poll % 20 == 19 {
+                        info!(job_id = %job_id, poll, "lead batch still in progress...");
+                    }
+                }
+            }
+        }
+
+        Err(format!(
+            "Lead batch job {job_id} timed out after ~10 minutes of polling. \
+             Check status manually via the webclaw API: GET /v1/lead/batch/{job_id}"
+        ))
+    }
+
+    /// Search the web for a query and return structured results.
+    ///
+    /// Resolves the backend in priority order:
+    /// 1. `SERPER_API_KEY` set → local Serper.dev search with the user's
+    ///    own key (no hosted API needed). Supports `country`, `lang`, and
+    ///    `scrape` (fetch + extract each result page).
+    /// 2. else `WEBCLAW_API_KEY` set → the hosted webclaw search API.
+    /// 3. else → an error explaining both options.
+    #[tool]
+    async fn search(&self, Parameters(params): Parameters<SearchParams>) -> Result<String, String> {
+        // Local path: user's own Serper key. Preferred when present so the
+        // tool works without the hosted API and without spending credits.
+        if let Ok(serper_key) = std::env::var("SERPER_API_KEY")
+            && !serper_key.trim().is_empty()
+        {
+            let opts = webclaw_fetch::SearchOptions {
+                num_results: params.num_results.unwrap_or(5) as usize,
+                country: params.country.clone(),
+                lang: params.lang.clone(),
+                scrape: params.scrape.unwrap_or(false),
+            };
+            let results = webclaw_fetch::search(
+                self.fetch_client.as_ref(),
+                &serper_key,
+                &params.query,
+                &opts,
+            )
+            .await
+            .map_err(|e| format!("search error: {e}"))?;
+
+            let mut output = format!("Found {} results:\n\n", results.len());
+            for r in &results {
+                output.push_str(&format!("{}. {}\n   {}\n", r.position, r.title, r.link));
+                if !r.snippet.is_empty() {
+                    output.push_str(&format!("   {}\n", r.snippet));
+                }
+                if let Some(ref content) = r.content {
+                    output.push_str(&format!("\n{content}\n"));
+                }
+                output.push('\n');
+            }
+            return Ok(output);
+        }
+
+        // Hosted path: the webclaw cloud API.
+        let cloud = self.cloud.as_ref().ok_or(
+            "Search requires a search backend: set SERPER_API_KEY for local search \
+             (get one free at serper.dev), or WEBCLAW_API_KEY for the hosted API.",
+        )?;
 
         let mut body = json!({ "query": params.query });
         if let Some(num) = params.num_results {
@@ -767,7 +903,7 @@ impl ServerHandler for WebclawMcp {
             .with_instructions(String::from(
                 "Webclaw MCP server -- web content extraction for AI agents. \
                  Tools: scrape, crawl, map, batch, extract, summarize, diff, brand, research, search, \
-                 list_extractors, vertical_scrape.",
+                 list_extractors, vertical_scrape, lead, lead_batch.",
             ))
     }
 }

crates/webclaw-mcp/src/tools.rs

@@ -4,6 +4,89 @@
 use schemars::JsonSchema;
 use serde::Deserialize;
 
+// ── Coercion helpers ────────────────────────────────────────────────────────
+//
+// MCP clients (Claude Desktop, VS Code extension, etc.) sometimes pass numeric
+// parameters as JSON strings (e.g. `"depth": "3"` instead of `"depth": 3`).
+// serde's default u32/usize deserialisers reject strings with:
+//
+//   "invalid type: string \"3\", expected u32"
+//
+// These helpers accept both forms transparently so callers never see that
+// error regardless of which representation their client sends. The same
+// problem hits booleans: clients send `"true"`/`"false"` as JSON strings,
+// which serde's default bool deserialiser rejects — `deser_opt_bool_or_str`
+// covers that case.
+
+fn deser_opt_u32_or_str<'de, D>(d: D) -> Result<Option<u32>, D::Error>
+where
+    D: serde::Deserializer<'de>,
+{
+    #[derive(serde::Deserialize)]
+    #[serde(untagged)]
+    enum NumOrStr {
+        Num(u32),
+        Str(String),
+    }
+    match Option::<NumOrStr>::deserialize(d)? {
+        None => Ok(None),
+        Some(NumOrStr::Num(n)) => Ok(Some(n)),
+        Some(NumOrStr::Str(s)) => {
+            s.trim().parse::<u32>().map(Some).map_err(|_| {
+                serde::de::Error::custom(format!("expected a u32, got string \"{s}\""))
+            })
+        }
+    }
+}
+
+fn deser_opt_usize_or_str<'de, D>(d: D) -> Result<Option<usize>, D::Error>
+where
+    D: serde::Deserializer<'de>,
+{
+    #[derive(serde::Deserialize)]
+    #[serde(untagged)]
+    enum NumOrStr {
+        Num(usize),
+        Str(String),
+    }
+    match Option::<NumOrStr>::deserialize(d)? {
+        None => Ok(None),
+        Some(NumOrStr::Num(n)) => Ok(Some(n)),
+        Some(NumOrStr::Str(s)) => {
+            s.trim().parse::<usize>().map(Some).map_err(|_| {
+                serde::de::Error::custom(format!("expected a usize, got string \"{s}\""))
+            })
+        }
+    }
+}
+
+fn deser_opt_bool_or_str<'de, D>(d: D) -> Result<Option<bool>, D::Error>
+where
+    D: serde::Deserializer<'de>,
+{
+    #[derive(serde::Deserialize)]
+    #[serde(untagged)]
+    enum BoolOrStr {
+        Bool(bool),
+        Str(String),
+    }
+    match Option::<BoolOrStr>::deserialize(d)? {
+        None => Ok(None),
+        Some(BoolOrStr::Bool(b)) => Ok(Some(b)),
+        // Accept "true"/"false" case-insensitively (trimmed). Reject anything
+        // else with a clear message rather than silently coercing it.
+        Some(BoolOrStr::Str(s)) => match s.trim().to_ascii_lowercase().as_str() {
+            "true" => Ok(Some(true)),
+            "false" => Ok(Some(false)),
+            _ => Err(serde::de::Error::custom(format!(
+                "expected a bool, got string \"{s}\""
+            ))),
+        },
+    }
+}
+
+// ── Parameter structs ───────────────────────────────────────────────────────
+
 #[derive(Debug, Deserialize, JsonSchema)]
 pub struct ScrapeParams {
     /// URL to scrape
@@ -15,6 +98,7 @@ pub struct ScrapeParams {
     /// CSS selectors to exclude from output
     pub exclude_selectors: Option<Vec<String>>,
     /// If true, extract only the main content (article/main element)
+    #[serde(default, deserialize_with = "deser_opt_bool_or_str")]
     pub only_main_content: Option<bool>,
     /// Browser profile: "chrome" (default), "firefox", or "random"
     pub browser: Option<String>,
@@ -27,12 +111,16 @@ pub struct CrawlParams {
     /// Seed URL to start crawling from
     pub url: String,
     /// Maximum link depth to follow (default: 2)
+    #[serde(default, deserialize_with = "deser_opt_u32_or_str")]
     pub depth: Option<u32>,
     /// Maximum number of pages to crawl (default: 50)
+    #[serde(default, deserialize_with = "deser_opt_usize_or_str")]
     pub max_pages: Option<usize>,
     /// Number of concurrent requests (default: 5)
+    #[serde(default, deserialize_with = "deser_opt_usize_or_str")]
     pub concurrency: Option<usize>,
     /// Seed the frontier from sitemap discovery before crawling
+    #[serde(default, deserialize_with = "deser_opt_bool_or_str")]
     pub use_sitemap: Option<bool>,
     /// Output format for each page: "markdown" (default), "llm", "text"
     pub format: Option<String>,
@@ -51,6 +139,7 @@ pub struct BatchParams {
     /// Output format: "markdown" (default), "llm", "text"
     pub format: Option<String>,
     /// Number of concurrent requests (default: 5)
+    #[serde(default, deserialize_with = "deser_opt_usize_or_str")]
     pub concurrency: Option<usize>,
 }
 
@@ -69,6 +158,7 @@ pub struct SummarizeParams {
     /// URL to fetch and summarize
     pub url: String,
     /// Number of sentences in the summary (default: 3)
+    #[serde(default, deserialize_with = "deser_opt_usize_or_str")]
     pub max_sentences: Option<usize>,
 }
 
@@ -91,17 +181,47 @@ pub struct ResearchParams {
     /// Research query or question to investigate
     pub query: String,
     /// Enable deep research mode for more thorough investigation (default: false)
+    #[serde(default, deserialize_with = "deser_opt_bool_or_str")]
     pub deep: Option<bool>,
     /// Topic hint to guide research focus (e.g. "technology", "finance", "science")
     pub topic: Option<String>,
 }
 
+#[derive(Debug, Deserialize, JsonSchema)]
+pub struct LeadParams {
+    /// Company website URL to enrich into an outreach-ready lead
+    pub url: String,
+    /// Skip the cache and force a fresh enrichment (default: false)
+    #[serde(default, deserialize_with = "deser_opt_bool_or_str")]
+    pub no_cache: Option<bool>,
+}
+
+#[derive(Debug, Deserialize, JsonSchema)]
+pub struct LeadBatchParams {
+    /// Company website URLs to enrich (up to 25 per batch)
+    pub urls: Vec<String>,
+    /// Skip the cache and force a fresh enrichment (default: false)
+    #[serde(default, deserialize_with = "deser_opt_bool_or_str")]
+    pub no_cache: Option<bool>,
+}
+
 #[derive(Debug, Deserialize, JsonSchema)]
 pub struct SearchParams {
     /// Search query
     pub query: String,
-    /// Number of results to return (default: 10)
+    /// Number of results to return (default: 5, max: 10)
+    #[serde(default, deserialize_with = "deser_opt_u32_or_str")]
     pub num_results: Option<u32>,
+    /// Country code for localization (e.g. "us", "gb", "it").
+    /// Only used by the local Serper path (SERPER_API_KEY).
+    pub country: Option<String>,
+    /// Language code for localization (e.g. "en", "it").
+    /// Only used by the local Serper path (SERPER_API_KEY).
+    pub lang: Option<String>,
+    /// When true, fetch + extract each result page and include its
+    /// markdown. Only used by the local Serper path (SERPER_API_KEY).
+    #[serde(default, deserialize_with = "deser_opt_bool_or_str")]
+    pub scrape: Option<bool>,
 }
 
 /// Parameters for `vertical_scrape`: run a site-specific extractor by name.
@@ -120,3 +240,292 @@ pub struct VerticalParams {
 /// so rmcp can generate a schema and parse the (empty) JSON-RPC params.
 #[derive(Debug, Deserialize, JsonSchema)]
 pub struct ListExtractorsParams {}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    // ── CrawlParams.depth (u32) ──────────────────────────────────────────────
+
+    #[test]
+    fn crawl_depth_from_numeric_string() {
+        let v: CrawlParams =
+            serde_json::from_str(r#"{"url":"https://x.com","depth":"3"}"#).unwrap();
+        assert_eq!(v.depth, Some(3));
+    }
+
+    #[test]
+    fn crawl_depth_from_number() {
+        let v: CrawlParams = serde_json::from_str(r#"{"url":"https://x.com","depth":3}"#).unwrap();
+        assert_eq!(v.depth, Some(3));
+    }
+
+    #[test]
+    fn crawl_depth_absent_is_none() {
+        let v: CrawlParams = serde_json::from_str(r#"{"url":"https://x.com"}"#).unwrap();
+        assert_eq!(v.depth, None);
+    }
+
+    #[test]
+    fn crawl_depth_non_numeric_string_errors() {
+        let e = serde_json::from_str::<CrawlParams>(r#"{"url":"https://x.com","depth":"abc"}"#);
+        assert!(e.is_err(), "expected Err, got {e:?}");
+    }
+
+    // ── CrawlParams.max_pages (usize) ────────────────────────────────────────
+
+    #[test]
+    fn crawl_max_pages_from_numeric_string() {
+        let v: CrawlParams =
+            serde_json::from_str(r#"{"url":"https://x.com","max_pages":"50"}"#).unwrap();
+        assert_eq!(v.max_pages, Some(50));
+    }
+
+    #[test]
+    fn crawl_max_pages_from_number() {
+        let v: CrawlParams =
+            serde_json::from_str(r#"{"url":"https://x.com","max_pages":50}"#).unwrap();
+        assert_eq!(v.max_pages, Some(50));
+    }
+
+    #[test]
+    fn crawl_max_pages_absent_is_none() {
+        let v: CrawlParams = serde_json::from_str(r#"{"url":"https://x.com"}"#).unwrap();
+        assert_eq!(v.max_pages, None);
+    }
+
+    #[test]
+    fn crawl_max_pages_non_numeric_string_errors() {
+        let e = serde_json::from_str::<CrawlParams>(r#"{"url":"https://x.com","max_pages":"abc"}"#);
+        assert!(e.is_err(), "expected Err, got {e:?}");
+    }
+
+    // ── CrawlParams.concurrency (usize) ──────────────────────────────────────
+
+    #[test]
+    fn crawl_concurrency_from_numeric_string() {
+        let v: CrawlParams =
+            serde_json::from_str(r#"{"url":"https://x.com","concurrency":"5"}"#).unwrap();
+        assert_eq!(v.concurrency, Some(5));
+    }
+
+    #[test]
+    fn crawl_concurrency_from_number() {
+        let v: CrawlParams =
+            serde_json::from_str(r#"{"url":"https://x.com","concurrency":5}"#).unwrap();
+        assert_eq!(v.concurrency, Some(5));
+    }
+
+    #[test]
+    fn crawl_concurrency_absent_is_none() {
+        let v: CrawlParams = serde_json::from_str(r#"{"url":"https://x.com"}"#).unwrap();
+        assert_eq!(v.concurrency, None);
+    }
+
+    #[test]
+    fn crawl_concurrency_non_numeric_string_errors() {
+        let e =
+            serde_json::from_str::<CrawlParams>(r#"{"url":"https://x.com","concurrency":"abc"}"#);
+        assert!(e.is_err(), "expected Err, got {e:?}");
+    }
+
+    // ── BatchParams.concurrency (usize) ──────────────────────────────────────
+
+    #[test]
+    fn batch_concurrency_from_numeric_string() {
+        let v: BatchParams =
+            serde_json::from_str(r#"{"urls":["https://x.com"],"concurrency":"5"}"#).unwrap();
+        assert_eq!(v.concurrency, Some(5));
+    }
+
+    #[test]
+    fn batch_concurrency_from_number() {
+        let v: BatchParams =
+            serde_json::from_str(r#"{"urls":["https://x.com"],"concurrency":5}"#).unwrap();
+        assert_eq!(v.concurrency, Some(5));
+    }
+
+    #[test]
+    fn batch_concurrency_absent_is_none() {
+        let v: BatchParams = serde_json::from_str(r#"{"urls":["https://x.com"]}"#).unwrap();
+        assert_eq!(v.concurrency, None);
+    }
+
+    #[test]
+    fn batch_concurrency_non_numeric_string_errors() {
+        let e = serde_json::from_str::<BatchParams>(
+            r#"{"urls":["https://x.com"],"concurrency":"abc"}"#,
+        );
+        assert!(e.is_err(), "expected Err, got {e:?}");
+    }
+
+    // ── SearchParams.num_results (u32) ───────────────────────────────────────
+
+    #[test]
+    fn search_num_results_from_numeric_string() {
+        let v: SearchParams =
+            serde_json::from_str(r#"{"query":"rust","num_results":"10"}"#).unwrap();
+        assert_eq!(v.num_results, Some(10));
+    }
+
+    #[test]
+    fn search_num_results_from_number() {
+        let v: SearchParams = serde_json::from_str(r#"{"query":"rust","num_results":10}"#).unwrap();
+        assert_eq!(v.num_results, Some(10));
+    }
+
+    #[test]
+    fn search_num_results_absent_is_none() {
+        let v: SearchParams = serde_json::from_str(r#"{"query":"rust"}"#).unwrap();
+        assert_eq!(v.num_results, None);
+    }
+
+    #[test]
+    fn search_num_results_non_numeric_string_errors() {
+        let e = serde_json::from_str::<SearchParams>(r#"{"query":"rust","num_results":"abc"}"#);
+        assert!(e.is_err(), "expected Err, got {e:?}");
+    }
+
+    // ── SummarizeParams.max_sentences (usize) ────────────────────────────────
+
+    #[test]
+    fn summarize_max_sentences_from_numeric_string() {
+        let v: SummarizeParams =
+            serde_json::from_str(r#"{"url":"https://x.com","max_sentences":"3"}"#).unwrap();
+        assert_eq!(v.max_sentences, Some(3));
+    }
+
+    #[test]
+    fn summarize_max_sentences_from_number() {
+        let v: SummarizeParams =
+            serde_json::from_str(r#"{"url":"https://x.com","max_sentences":3}"#).unwrap();
+        assert_eq!(v.max_sentences, Some(3));
+    }
+
+    #[test]
+    fn summarize_max_sentences_absent_is_none() {
+        let v: SummarizeParams = serde_json::from_str(r#"{"url":"https://x.com"}"#).unwrap();
+        assert_eq!(v.max_sentences, None);
+    }
+
+    #[test]
+    fn summarize_max_sentences_non_numeric_string_errors() {
+        let e = serde_json::from_str::<SummarizeParams>(
+            r#"{"url":"https://x.com","max_sentences":"abc"}"#,
+        );
+        assert!(e.is_err(), "expected Err, got {e:?}");
+    }
+
+    // ── Boolean param string-coercion (issue #62) ───────────────────────────
+
+    // ScrapeParams.only_main_content
+    #[test]
+    fn scrape_only_main_content_from_bool() {
+        let v: ScrapeParams =
+            serde_json::from_str(r#"{"url":"https://x.com","only_main_content":true}"#).unwrap();
+        assert_eq!(v.only_main_content, Some(true));
+    }
+
+    #[test]
+    fn scrape_only_main_content_from_string() {
+        let t: ScrapeParams =
+            serde_json::from_str(r#"{"url":"https://x.com","only_main_content":"true"}"#).unwrap();
+        assert_eq!(t.only_main_content, Some(true));
+        let f: ScrapeParams =
+            serde_json::from_str(r#"{"url":"https://x.com","only_main_content":"false"}"#).unwrap();
+        assert_eq!(f.only_main_content, Some(false));
+    }
+
+    #[test]
+    fn scrape_only_main_content_absent_is_none() {
+        let v: ScrapeParams = serde_json::from_str(r#"{"url":"https://x.com"}"#).unwrap();
+        assert_eq!(v.only_main_content, None);
+    }
+
+    #[test]
+    fn scrape_only_main_content_non_bool_string_errors() {
+        let e = serde_json::from_str::<ScrapeParams>(
+            r#"{"url":"https://x.com","only_main_content":"yes"}"#,
+        );
+        assert!(e.is_err(), "expected Err, got {e:?}");
+    }
+
+    // CrawlParams.use_sitemap
+    #[test]
+    fn crawl_use_sitemap_from_bool() {
+        let v: CrawlParams =
+            serde_json::from_str(r#"{"url":"https://x.com","use_sitemap":false}"#).unwrap();
+        assert_eq!(v.use_sitemap, Some(false));
+    }
+
+    #[test]
+    fn crawl_use_sitemap_from_string() {
+        let v: CrawlParams =
+            serde_json::from_str(r#"{"url":"https://x.com","use_sitemap":"true"}"#).unwrap();
+        assert_eq!(v.use_sitemap, Some(true));
+    }
+
+    #[test]
+    fn crawl_use_sitemap_absent_is_none() {
+        let v: CrawlParams = serde_json::from_str(r#"{"url":"https://x.com"}"#).unwrap();
+        assert_eq!(v.use_sitemap, None);
+    }
+
+    #[test]
+    fn crawl_use_sitemap_non_bool_string_errors() {
+        let e =
+            serde_json::from_str::<CrawlParams>(r#"{"url":"https://x.com","use_sitemap":"nope"}"#);
+        assert!(e.is_err(), "expected Err, got {e:?}");
+    }
+
+    // ResearchParams.deep
+    #[test]
+    fn research_deep_from_bool() {
+        let v: ResearchParams = serde_json::from_str(r#"{"query":"rust","deep":true}"#).unwrap();
+        assert_eq!(v.deep, Some(true));
+    }
+
+    #[test]
+    fn research_deep_from_string() {
+        let v: ResearchParams = serde_json::from_str(r#"{"query":"rust","deep":"true"}"#).unwrap();
+        assert_eq!(v.deep, Some(true));
+    }
+
+    #[test]
+    fn research_deep_absent_is_none() {
+        let v: ResearchParams = serde_json::from_str(r#"{"query":"rust"}"#).unwrap();
+        assert_eq!(v.deep, None);
+    }
+
+    #[test]
+    fn research_deep_non_bool_string_errors() {
+        // Numeric-looking strings are NOT accepted for bools (avoids ambiguity).
+        let e = serde_json::from_str::<ResearchParams>(r#"{"query":"rust","deep":"1"}"#);
+        assert!(e.is_err(), "expected Err, got {e:?}");
+    }
+
+    // SearchParams.scrape
+    #[test]
+    fn search_scrape_from_bool() {
+        let v: SearchParams = serde_json::from_str(r#"{"query":"rust","scrape":true}"#).unwrap();
+        assert_eq!(v.scrape, Some(true));
+    }
+
+    #[test]
+    fn search_scrape_from_string_case_insensitive() {
+        let v: SearchParams = serde_json::from_str(r#"{"query":"rust","scrape":"True"}"#).unwrap();
+        assert_eq!(v.scrape, Some(true));
+    }
+
+    #[test]
+    fn search_scrape_absent_is_none() {
+        let v: SearchParams = serde_json::from_str(r#"{"query":"rust"}"#).unwrap();
+        assert_eq!(v.scrape, None);
+    }
+
+    #[test]
+    fn search_scrape_non_bool_string_errors() {
+        let e = serde_json::from_str::<SearchParams>(r#"{"query":"rust","scrape":"maybe"}"#);
+        assert!(e.is_err(), "expected Err, got {e:?}");
+    }
+}

crates/webclaw-server/src/error.rs

@@ -38,16 +38,24 @@ pub enum ApiError {
 
     #[error("internal: {0}")]
     Internal(String),
+
+    #[error("{0}")]
+    NotImplemented(String),
 }
 
 impl ApiError {
     pub fn bad_request(msg: impl Into<String>) -> Self {
         Self::BadRequest(msg.into())
     }
-    #[allow(dead_code)]
     pub fn internal(msg: impl Into<String>) -> Self {
         Self::Internal(msg.into())
     }
+    /// 501 — a capability the operator hasn't configured (e.g. search
+    /// without `SERPER_API_KEY`). Distinct from `BadRequest` (client's
+    /// fault) and `Internal` (our fault): it's a deployment-config gap.
+    pub fn not_implemented(msg: impl Into<String>) -> Self {
+        Self::NotImplemented(msg.into())
+    }
 
     fn status(&self) -> StatusCode {
         match self {
@@ -57,6 +65,7 @@ impl ApiError {
             Self::Fetch(_) => StatusCode::BAD_GATEWAY,
             Self::Extract(_) | Self::Llm(_) => StatusCode::UNPROCESSABLE_ENTITY,
             Self::Internal(_) => StatusCode::INTERNAL_SERVER_ERROR,
+            Self::NotImplemented(_) => StatusCode::NOT_IMPLEMENTED,
         }
     }
 }

crates/webclaw-server/src/main.rs

@@ -57,7 +57,7 @@ struct Args {
     /// `/v1/*` request must present `Authorization: Bearer <key>`.
     /// When unset, the server runs in open mode (no auth) — only
     /// safe on a local-bound interface or behind another auth layer.
-    #[arg(long, env = "WEBCLAW_API_KEY")]
+    #[arg(long, env = "WEBCLAW_API_KEY", hide_env_values = true)]
     api_key: Option<String>,
 
     /// Tracing filter. Env: RUST_LOG.
@@ -94,6 +94,7 @@ async fn main() -> anyhow::Result<()> {
         )
         .route("/crawl", post(routes::crawl::crawl))
         .route("/map", post(routes::map::map))
+        .route("/search", post(routes::search::search))
         .route("/batch", post(routes::batch::batch))
         .route("/extract", post(routes::extract::extract))
         .route("/extractors", get(routes::structured::list_extractors))

crates/webclaw-server/src/routes/mod.rs

@@ -6,6 +6,11 @@
 //! (anti-bot bypass with stealth Chrome, JS rendering at scale,
 //! per-user auth, billing, async job queues, agent loops) are
 //! intentionally not implemented here. Use api.webclaw.io for those.
+//!
+//! `POST /v1/search` is supported when the operator supplies their own
+//! Serper.dev API key via the `SERPER_API_KEY` env var (free key at
+//! serper.dev). Without it, the route returns 501. This is the
+//! bring-your-own-key path — no hosted webclaw account required.
 
 pub mod batch;
 pub mod brand;
@@ -15,5 +20,6 @@ pub mod extract;
 pub mod health;
 pub mod map;
 pub mod scrape;
+pub mod search;
 pub mod structured;
 pub mod summarize;

crates/webclaw-server/src/routes/search.rs

@@ -0,0 +1,68 @@
+//! POST /v1/search — web search via Serper.dev using the operator's own key.
+//!
+//! Enabled only when the server is started with `SERPER_API_KEY` set
+//! (get a free key at serper.dev). Without it, this route returns 501 so
+//! self-hosters know the capability exists but isn't configured.
+//!
+//! With `scrape: true`, each result page is fetched + extracted to
+//! markdown via the shared [`webclaw_fetch::FetchClient`]. A per-result
+//! fetch failure leaves that result's `content` null; it never fails the
+//! whole search.
+
+use axum::{Json, extract::State};
+use serde::Deserialize;
+use serde_json::{Value, json};
+
+use crate::{error::ApiError, state::AppState};
+
+#[derive(Debug, Deserialize)]
+pub struct SearchRequest {
+    pub query: String,
+    /// Max results to return (default 5, clamped to 1..=10).
+    #[serde(default = "default_num_results")]
+    pub num_results: usize,
+    /// Country code for localization (e.g. "us", "gb", "it").
+    pub country: Option<String>,
+    /// Language code for localization (e.g. "en", "it").
+    pub lang: Option<String>,
+    /// When true, fetch + extract each result page and include its markdown.
+    #[serde(default)]
+    pub scrape: bool,
+}
+
+fn default_num_results() -> usize {
+    5
+}
+
+pub async fn search(
+    State(state): State<AppState>,
+    Json(req): Json<SearchRequest>,
+) -> Result<Json<Value>, ApiError> {
+    if req.query.trim().is_empty() {
+        return Err(ApiError::bad_request("`query` is required"));
+    }
+
+    let serper_key = state.serper_api_key().ok_or_else(|| {
+        ApiError::not_implemented(
+            "search is not configured: start the server with SERPER_API_KEY set \
+             (get a free key at serper.dev)",
+        )
+    })?;
+
+    let opts = webclaw_fetch::SearchOptions {
+        num_results: req.num_results,
+        country: req.country.clone(),
+        lang: req.lang.clone(),
+        scrape: req.scrape,
+    };
+
+    let results = webclaw_fetch::search(state.fetch(), serper_key, &req.query, &opts)
+        .await
+        .map_err(|e| ApiError::internal(format!("search failed: {e}")))?;
+
+    Ok(Json(json!({
+        "query": req.query,
+        "count": results.len(),
+        "results": results,
+    })))
+}

crates/webclaw-server/src/state.rs

@@ -36,6 +36,9 @@ struct Inner {
     pub fetch: Arc<FetchClient>,
     /// Inbound bearer-auth token for this server's own `/v1/*` surface.
     pub api_key: Option<String>,
+    /// Operator's own Serper.dev API key, read from `SERPER_API_KEY`.
+    /// Enables `/v1/search`. Unset = `/v1/search` returns 501.
+    pub serper_api_key: Option<String>,
 }
 
 impl AppState {
@@ -66,10 +69,20 @@ impl AppState {
             fetch = fetch.with_cloud(cloud);
         }
 
+        // Operator's own Serper.dev key enables /v1/search. Empty/unset
+        // leaves search returning 501 with a setup hint.
+        let serper_api_key = std::env::var("SERPER_API_KEY")
+            .ok()
+            .filter(|k| !k.trim().is_empty());
+        if serper_api_key.is_some() {
+            info!("search enabled — using SERPER_API_KEY for /v1/search");
+        }
+
         Ok(Self {
             inner: Arc::new(Inner {
                 fetch: Arc::new(fetch),
                 api_key: inbound_api_key,
+                serper_api_key,
             }),
         })
     }
@@ -81,6 +94,11 @@ impl AppState {
     pub fn api_key(&self) -> Option<&str> {
         self.inner.api_key.as_deref()
     }
+
+    /// Operator's Serper.dev key for `/v1/search`, if configured.
+    pub fn serper_api_key(&self) -> Option<&str> {
+        self.inner.serper_api_key.as_deref()
+    }
 }
 
 /// Resolve the outbound cloud key. Prefers `WEBCLAW_CLOUD_API_KEY`;

deploy/hetzner.sh

@@ -196,7 +196,7 @@ build_cloud_init() {
     local env_content="# webclaw deployment — generated by hetzner.sh
 WEBCLAW_HOST=0.0.0.0
 WEBCLAW_PORT=3000
-WEBCLAW_AUTH_KEY=$auth_key
+WEBCLAW_API_KEY=$auth_key
 OLLAMA_HOST=http://ollama:11434
 OLLAMA_MODEL=$ollama_model
 WEBCLAW_LOG=info"
@@ -469,7 +469,7 @@ create_server() {
     printf "  ${BOLD}SSH:${RESET}          ssh root@%s\n" "$server_ip"
     printf "  ${BOLD}Auth key:${RESET}     %s\n" "$(mask_secret "$AUTH_KEY")"
     printf "  ${DIM}(full key stored in /opt/webclaw/.env on the server:\n"
-    printf "   ssh root@%s 'grep WEBCLAW_AUTH_KEY /opt/webclaw/.env')${RESET}\n" "$server_ip"
+    printf "   ssh root@%s 'grep WEBCLAW_API_KEY /opt/webclaw/.env')${RESET}\n" "$server_ip"
     echo
     printf "  ${BOLD}Monitor build progress:${RESET}\n"
     printf "    ssh root@%s 'cd /opt/webclaw && docker compose logs -f'\n" "$server_ip"

env.example

@@ -29,7 +29,7 @@ OLLAMA_MODEL=qwen3:8b
 # --- Server (webclaw-server only) ---
 # WEBCLAW_PORT=3000
 # WEBCLAW_HOST=0.0.0.0
-# WEBCLAW_AUTH_KEY=your-auth-key
+# WEBCLAW_API_KEY=your-auth-key
 # WEBCLAW_MAX_CONCURRENCY=50
 # WEBCLAW_JOB_TTL_SECS=3600
 # WEBCLAW_MAX_JOBS=100

examples/README.md

@@ -273,14 +273,15 @@ webclaw https://example.com  # Automatically detects and uses proxies.txt
 ## MCP Server (AI Agent Integration)
 
 ```bash
-# Start the MCP server (stdio transport)
-webclaw-mcp
+# Run the MCP server (stdio transport) — no install needed
+npx -y @webclaw/mcp
 
 # Configure in Claude Desktop (~/.config/claude/claude_desktop_config.json):
 # {
 #   "mcpServers": {
 #     "webclaw": {
-#       "command": "/path/to/webclaw-mcp",
+#       "command": "npx",
+#       "args": ["-y", "@webclaw/mcp"],
 #       "env": {
 #         "WEBCLAW_API_KEY": "wc_your_key"  // optional, enables cloud fallback
 #       }
@@ -288,7 +289,7 @@ webclaw-mcp
 #   }
 # }
 
-# Available tools: scrape, crawl, map, batch, extract, summarize, diff, brand, research, search
+# Available tools: scrape, search, crawl, map, batch, extract, summarize, diff, brand, research, lead, lead_batch, list_extractors, vertical_scrape
 ```
 
 ## Real-World Recipes

examples/mcp-web-scraping/README.md

@@ -16,7 +16,8 @@ The installer detects supported MCP clients and can write the config for you.
 {
   "mcpServers": {
     "webclaw": {
-      "command": "~/.webclaw/webclaw-mcp",
+      "command": "npx",
+      "args": ["-y", "@webclaw/mcp"],
       "env": {
         "WEBCLAW_API_KEY": "wc_your_key"
       }

examples/proxy-backed-crawling/README.md

@@ -1,6 +1,68 @@
 # Proxy-Backed Crawling
 
-Use proxy rotation when you need to distribute a crawl across a proxy pool. webclaw supports a single proxy or a proxy file.
+Use proxy rotation when you need to distribute a crawl across a proxy pool. webclaw supports a single proxy or a proxy file, and accepts any standard HTTP/HTTPS or SOCKS5 proxy URL.
+
+## Using ColdProxy
+
+[ColdProxy](https://coldproxy.com/?utm_source=github&utm_medium=sponsorship&utm_campaign=webclaw-sponsor) is webclaw's infrastructure partner, providing residential IPv4, residential IPv6, and datacenter IPv6 proxies across 195+ countries. Use a ColdProxy endpoint as a full URL with `--proxy` / `WEBCLAW_PROXY`, or list several in a `--proxy-file` pool.
+
+### 1. Get your endpoint
+
+Sign in to your [ColdProxy dashboard](https://coldproxy.com/?utm_source=github&utm_medium=sponsorship&utm_campaign=webclaw-sponsor) and copy your proxy host, port, and credentials. Assemble them into a standard proxy URL:
+
+```text
+http://USERNAME:PASSWORD@HOST:PORT
+```
+
+### 2. One ColdProxy endpoint
+
+```bash
+export WEBCLAW_PROXY="http://USERNAME:PASSWORD@HOST:PORT"
+webclaw https://example.com --format markdown
+```
+
+Or pass it inline:
+
+```bash
+webclaw https://example.com \
+  --proxy "http://USERNAME:PASSWORD@HOST:PORT" \
+  --format markdown
+```
+
+### 3. Rotate a ColdProxy pool
+
+List one ColdProxy endpoint per line in `coldproxy.txt`. Pool files use `host:port:user:pass` (one entry per line; lines starting with `#` are ignored). Mix product types and regions to match your workload:
+
+```text
+# residential IPv4
+HOST:PORT:USERNAME:PASSWORD
+# residential IPv6
+HOST:PORT:USERNAME:PASSWORD
+# datacenter IPv6
+HOST:PORT:USERNAME:PASSWORD
+```
+
+webclaw rotates across the pool per request:
+
+```bash
+webclaw https://docs.example.com \
+  --crawl \
+  --depth 2 \
+  --max-pages 200 \
+  --concurrency 10 \
+  --delay 200 \
+  --proxy-file coldproxy.txt \
+  --format markdown
+```
+
+### 4. Target a country
+
+ColdProxy offers access across 195+ countries. Use the country-specific endpoint from your ColdProxy dashboard for each region you want to collect from (for example, a France residential endpoint for fr-localized pages). Add one endpoint per country to your pool file to spread a single crawl across regions.
+
+### Choosing a product
+
+- **Residential IPv4 / IPv6** — suitable for region-specific testing, localized content validation, public data collection, market monitoring, and regional QA.
+- **Datacenter IPv6** — fastest and most cost-effective; best for high-volume crawling of tolerant endpoints.
 
 ## Single Proxy
 
@@ -20,12 +82,12 @@ webclaw https://example.com \
 
 ## Proxy Pool
 
-Create `proxies.txt` with one proxy per line:
+Create `proxies.txt` with one proxy per line in `host:port:user:pass` format (lines starting with `#` are ignored):
 
 ```text
-http://user:pass@proxy-1.example.com:8080
-http://user:pass@proxy-2.example.com:8080
-http://user:pass@proxy-3.example.com:8080
+proxy-1.example.com:8080:user:pass
+proxy-2.example.com:8080:user:pass
+proxy-3.example.com:8080:user:pass
 ```
 
 Run a crawl with controlled concurrency:

packages/create-webclaw/README.md

@@ -26,7 +26,7 @@
 npx create-webclaw
 ```
 
-That's it. Auto-detects your AI tools, downloads the MCP server, configures everything.
+That's it. Auto-detects your AI tools and writes the `npx @webclaw/mcp` config into each — nothing to install.
 
 Works with **Claude Desktop**, **Claude Code**, **Cursor**, **Windsurf**, **VS Code**, **OpenCode**, **Codex CLI**, and **Antigravity**.
 
@@ -40,7 +40,7 @@ When it does work, you get 100KB+ of raw HTML — navigation, ads, cookie banner
 
 ## The Fix
 
-webclaw impersonates Chrome 146 at the TLS protocol level. Perfect JA4 fingerprint. Perfect HTTP/2 Akamai hash. 99% bypass rate on 102 tested sites.
+webclaw impersonates the latest Chrome at the TLS protocol level. Perfect JA4 fingerprint. Perfect HTTP/2 Akamai hash. 99% bypass rate on 102 tested sites.
 
 Then it extracts just the content — clean markdown, 67% fewer tokens.
 
@@ -67,13 +67,29 @@ npx create-webclaw
 ```
 
 1. Detects installed AI tools (Claude, Cursor, Windsurf, VS Code, OpenCode, Codex, Antigravity)
-2. Downloads the `webclaw-mcp` binary for your platform (macOS arm64/x86, Linux x86/arm64)
-3. Asks for your API key (optional — **works locally without one**)
-4. Writes the MCP config for each detected tool
+2. Asks for your API key (optional — **works locally without one**)
+3. Writes the `npx @webclaw/mcp` config into each detected tool
 
-## 10 MCP Tools
+The server itself runs via [`@webclaw/mcp`](https://www.npmjs.com/package/@webclaw/mcp) — `npx` fetches it on first launch and caches it. `create-webclaw` is just the convenience that writes that config for you.
 
-After setup, your AI agent has access to:
+### Prefer to configure by hand?
+
+Add this one block to your client's `mcpServers` config — it's identical to what `create-webclaw` writes:
+
+```json
+{
+  "mcpServers": {
+    "webclaw": {
+      "command": "npx",
+      "args": ["-y", "@webclaw/mcp"]
+    }
+  }
+}
+```
+
+## MCP Tools
+
+After setup, your AI agent has access to these 14 tools:
 
 | Tool | What it does | API key needed? |
 |------|-------------|-----------------|
@@ -87,8 +103,12 @@ After setup, your AI agent has access to:
 | **diff** | Track content changes | No |
 | **brand** | Extract brand identity | No |
 | **research** | Deep multi-page research | Yes |
+| **list_extractors** | List the built-in vertical extractors | No |
+| **vertical_scrape** | Scrape with a built-in vertical extractor | No |
+| **lead** | Enrich a company URL into an outreach-ready lead (leadership + socials, tech, pricing, public emails) | Yes (webclaw) |
+| **lead_batch** | Enrich up to 25 company URLs at once | Yes (webclaw) |
 
-**8 of 10 tools work fully offline.** No API key, no cloud, no tracking.
+**8 of the 14 tools work fully offline** — no API key, no cloud, no tracking.
 
 ## Supported Tools
 
@@ -99,9 +119,9 @@ After setup, your AI agent has access to:
 | Cursor | `.cursor/mcp.json` |
 | Windsurf | `~/.codeium/windsurf/mcp_config.json` |
 | VS Code (Continue) | `~/.continue/config.json` |
-| OpenCode | `~/.opencode/config.json` |
-| Codex CLI | `~/.codex/config.json` |
-| Antigravity | `~/.antigravity/mcp.json` |
+| OpenCode | `~/.config/opencode/opencode.json` (or `./opencode.json`) |
+| Codex CLI | `~/.codex/config.toml` |
+| Antigravity | `~/.config/antigravity/mcp.json` (or `~/.antigravity/mcp.json`) |
 
 ## Sites That Work
 
@@ -140,7 +160,6 @@ Download from [GitHub Releases](https://github.com/0xMassi/webclaw/releases) for
 - [Website](https://webclaw.io)
 - [Documentation](https://webclaw.io/docs)
 - [GitHub](https://github.com/0xMassi/webclaw)
-- [TLS Library](https://github.com/0xMassi/webclaw-tls)
 - [Discord](https://discord.gg/KDfd48EpnW)
 - [Status](https://status.webclaw.io)
 

packages/create-webclaw/index.mjs

@@ -1,22 +1,22 @@
 #!/usr/bin/env node
 
+// create-webclaw — optional convenience installer for the webclaw MCP server.
+//
+// It auto-detects your AI tools (Claude Desktop, Claude Code, Cursor, Windsurf,
+// OpenCode, Codex, Antigravity, ...) and writes the canonical `npx @webclaw/mcp`
+// config into each. It does NOT download a binary: the runtime is always the
+// `@webclaw/mcp` launcher, so a scaffolded config is byte-identical to what
+// webclaw.io/docs and the MCP registries document. If you'd rather not run this,
+// just add the one config block below by hand.
+
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
 import { createInterface } from "readline";
-import { homedir, platform, arch } from "os";
+import { homedir, platform } from "os";
 import { join, dirname } from "path";
-import { execSync } from "child_process";
-import { createWriteStream } from "fs";
-import { chmod } from "fs/promises";
-import https from "https";
-import http from "http";
 
 // ── Constants ──
 
-const REPO = "0xMassi/webclaw";
-const BINARY_NAME = "webclaw-mcp";
-const INSTALL_DIR = join(homedir(), ".webclaw");
-const BINARY_PATH = join(INSTALL_DIR, BINARY_NAME);
-const VERSION = "latest";
+const MCP_PACKAGE = "@webclaw/mcp";
 
 const COLORS = {
   reset: "\x1b[0m",
@@ -75,7 +75,6 @@ const AI_TOOLS = [
     id: "cursor",
     name: "Cursor",
     detect: () => {
-      // Check for .cursor directory in home or current project
       return (
         existsSync(join(homedir(), ".cursor")) ||
         existsSync(join(process.cwd(), ".cursor"))
@@ -166,78 +165,6 @@ function ask(question) {
   });
 }
 
-function download(url) {
-  return new Promise((resolve, reject) => {
-    const client = url.startsWith("https") ? https : http;
-    client
-      .get(url, { headers: { "User-Agent": "create-webclaw" } }, (res) => {
-        // Follow redirects
-        if (
-          res.statusCode >= 300 &&
-          res.statusCode < 400 &&
-          res.headers.location
-        ) {
-          return download(res.headers.location).then(resolve).catch(reject);
-        }
-        if (res.statusCode !== 200) {
-          return reject(new Error(`HTTP ${res.statusCode}`));
-        }
-        const chunks = [];
-        res.on("data", (chunk) => chunks.push(chunk));
-        res.on("end", () => resolve(Buffer.concat(chunks)));
-        res.on("error", reject);
-      })
-      .on("error", reject);
-  });
-}
-
-async function downloadFile(url, dest) {
-  return new Promise((resolve, reject) => {
-    const client = url.startsWith("https") ? https : http;
-    client
-      .get(url, { headers: { "User-Agent": "create-webclaw" } }, (res) => {
-        if (
-          res.statusCode >= 300 &&
-          res.statusCode < 400 &&
-          res.headers.location
-        ) {
-          return downloadFile(res.headers.location, dest)
-            .then(resolve)
-            .catch(reject);
-        }
-        if (res.statusCode !== 200) {
-          return reject(new Error(`HTTP ${res.statusCode}`));
-        }
-        const file = createWriteStream(dest);
-        res.pipe(file);
-        file.on("finish", () => {
-          file.close();
-          resolve();
-        });
-        file.on("error", reject);
-      })
-      .on("error", reject);
-  });
-}
-
-function getAssetName() {
-  const os = platform();
-  const a = arch();
-
-  if (os === "darwin" && a === "arm64")
-    return `webclaw-mcp-aarch64-apple-darwin.tar.gz`;
-  if (os === "darwin" && a === "x64")
-    return `webclaw-mcp-x86_64-apple-darwin.tar.gz`;
-  if (os === "linux" && a === "x64")
-    return `webclaw-mcp-x86_64-unknown-linux-gnu.tar.gz`;
-  if (os === "linux" && a === "arm64")
-    return `webclaw-mcp-aarch64-unknown-linux-gnu.tar.gz`;
-  if (os === "win32" && a === "x64")
-    return `webclaw-mcp-x86_64-pc-windows-msvc.zip`;
-
-  return null;
-}
-
 function readJsonFile(path) {
   try {
     return JSON.parse(readFileSync(path, "utf-8"));
@@ -252,43 +179,20 @@ function writeJsonFile(path, data) {
   writeFileSync(path, JSON.stringify(data, null, 2) + "\n");
 }
 
+// The single canonical way to run the webclaw MCP server: the npx launcher.
+// `@webclaw/mcp` fetches the prebuilt binary on first run and speaks MCP over
+// stdio, so this config matches webclaw.io/docs and the MCP registries exactly.
 function buildMcpEntry(apiKey) {
-  const entry = {
-    command: BINARY_PATH,
-  };
-  if (apiKey) {
-    entry.env = { WEBCLAW_API_KEY: apiKey };
-  }
+  const entry = { command: "npx", args: ["-y", MCP_PACKAGE] };
+  if (apiKey) entry.env = { WEBCLAW_API_KEY: apiKey };
   return entry;
 }
 
+const MANUAL_CONFIG = `{ "mcpServers": { "webclaw": { "command": "npx", "args": ["-y", "${MCP_PACKAGE}"] } } }`;
+
 // ── MCP Config Writers ──
 
-function addToClaudeDesktop(configPath, apiKey) {
-  const config = readJsonFile(configPath);
-  if (!config.mcpServers) config.mcpServers = {};
-  config.mcpServers.webclaw = buildMcpEntry(apiKey);
-  writeJsonFile(configPath, config);
-}
-
-function addToClaudeCode(configPath, apiKey) {
-  const config = readJsonFile(configPath);
-  if (!config.mcpServers) config.mcpServers = {};
-  config.mcpServers.webclaw = buildMcpEntry(apiKey);
-  writeJsonFile(configPath, config);
-}
-
-function addToCursor(configPath, apiKey) {
-  const config = readJsonFile(configPath);
-  if (!config.mcpServers) config.mcpServers = {};
-  config.mcpServers.webclaw = {
-    command: BINARY_PATH,
-    ...(apiKey ? { env: { WEBCLAW_API_KEY: apiKey } } : {}),
-  };
-  writeJsonFile(configPath, config);
-}
-
-function addToWindsurf(configPath, apiKey) {
+function addToMcpServers(configPath, apiKey) {
   const config = readJsonFile(configPath);
   if (!config.mcpServers) config.mcpServers = {};
   config.mcpServers.webclaw = buildMcpEntry(apiKey);
@@ -298,13 +202,9 @@ function addToWindsurf(configPath, apiKey) {
 function addToVSCodeContinue(configPath, apiKey) {
   const config = readJsonFile(configPath);
   if (!config.mcpServers) config.mcpServers = [];
-  // Continue uses array format
+  // Continue uses array format.
   const existing = config.mcpServers.findIndex?.((s) => s.name === "webclaw");
-  const entry = {
-    name: "webclaw",
-    command: BINARY_PATH,
-    ...(apiKey ? { env: { WEBCLAW_API_KEY: apiKey } } : {}),
-  };
+  const entry = { name: "webclaw", ...buildMcpEntry(apiKey) };
   if (existing >= 0) {
     config.mcpServers[existing] = entry;
   } else if (Array.isArray(config.mcpServers)) {
@@ -318,7 +218,7 @@ function addToOpenCode(configPath, apiKey) {
   if (!config.mcp) config.mcp = {};
   config.mcp.webclaw = {
     type: "local",
-    command: [BINARY_PATH],
+    command: ["npx", "-y", MCP_PACKAGE],
     enabled: true,
   };
   if (apiKey) {
@@ -327,15 +227,8 @@ function addToOpenCode(configPath, apiKey) {
   writeJsonFile(configPath, config);
 }
 
-function addToAntigravity(configPath, apiKey) {
-  const config = readJsonFile(configPath);
-  if (!config.mcpServers) config.mcpServers = {};
-  config.mcpServers.webclaw = buildMcpEntry(apiKey);
-  writeJsonFile(configPath, config);
-}
-
 function addToCodex(configPath, apiKey) {
-  // Codex uses TOML format, not JSON. Append MCP server config section.
+  // Codex uses TOML, not JSON. Replace any existing webclaw section.
   const dir = dirname(configPath);
   if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
 
@@ -343,31 +236,28 @@ function addToCodex(configPath, apiKey) {
   try {
     existing = readFileSync(configPath, "utf-8");
   } catch {
-    // File doesn't exist yet
+    // File doesn't exist yet.
   }
-
-  // Remove any existing webclaw MCP section
   existing = existing.replace(
     /\n?\[mcp_servers\.webclaw\][^\[]*(?=\[|$)/gs,
     "",
   );
 
-  let section = `\n[mcp_servers.webclaw]\ncommand = "${BINARY_PATH}"\nargs = []\nenabled = true\n`;
+  let section = `\n[mcp_servers.webclaw]\ncommand = "npx"\nargs = ["-y", "${MCP_PACKAGE}"]\nenabled = true\n`;
   if (apiKey) {
     section += `env = { WEBCLAW_API_KEY = "${apiKey}" }\n`;
   }
-
   writeFileSync(configPath, existing.trimEnd() + "\n" + section);
 }
 
 const CONFIG_WRITERS = {
-  "claude-desktop": addToClaudeDesktop,
-  "claude-code": addToClaudeCode,
-  cursor: addToCursor,
-  windsurf: addToWindsurf,
+  "claude-desktop": addToMcpServers,
+  "claude-code": addToMcpServers,
+  cursor: addToMcpServers,
+  windsurf: addToMcpServers,
   "vscode-continue": addToVSCodeContinue,
   opencode: addToOpenCode,
-  antigravity: addToAntigravity,
+  antigravity: addToMcpServers,
   codex: addToCodex,
 };
 
@@ -385,7 +275,7 @@ async function main() {
   console.log(c("bold", "  └─────────────────────────────────────┘"));
   console.log();
 
-  // 1. Detect installed AI tools
+  // 1. Detect installed AI tools.
   console.log(c("bold", "  Detecting AI tools..."));
   console.log();
 
@@ -400,20 +290,10 @@ async function main() {
   if (detected.length === 0) {
     console.log(c("yellow", "  No supported AI tools detected."));
     console.log();
-    console.log(c("dim", "  Supported tools:"));
-    for (const tool of AI_TOOLS) {
-      console.log(c("dim", `    • ${tool.name}`));
-    }
+    console.log(c("dim", "  Add this to your MCP client config by hand:"));
+    console.log(c("cyan", `    ${MANUAL_CONFIG}`));
     console.log();
-    console.log(
-      c("dim", "  Install one of these tools and run this command again."),
-    );
-    console.log(c("dim", "  Or use --manual to configure manually."));
-    console.log();
-
-    if (process.argv.includes("--manual")) {
-      // Continue anyway for manual setup
-    } else {
+    if (!process.argv.includes("--manual")) {
       process.exit(0);
     }
   }
@@ -423,10 +303,12 @@ async function main() {
   }
   console.log();
 
-  // 2. Ask for API key
-  console.log(c("dim", "  An API key enables cloud features."));
+  // 2. Ask for an optional API key.
   console.log(
-    c("dim", "  Without one, webclaw runs locally (free, no account needed)."),
+    c("dim", "  An API key unlocks the cloud tools (bot bypass, JS rendering,"),
+  );
+  console.log(
+    c("dim", "  search, research, leads). Without one, webclaw runs locally."),
   );
   console.log();
 
@@ -436,112 +318,15 @@ async function main() {
   );
   console.log();
 
-  // 3. Download binary
-  console.log(c("bold", "  Downloading webclaw-mcp..."));
-
-  const assetName = getAssetName();
-  if (!assetName) {
-    console.log(c("red", `  Unsupported platform: ${platform()}-${arch()}`));
-    console.log(
-      c(
-        "dim",
-        "  Build from source: cargo install --git https://github.com/0xMassi/webclaw webclaw-mcp",
-      ),
-    );
-    process.exit(1);
-  }
-
-  if (!existsSync(INSTALL_DIR)) {
-    mkdirSync(INSTALL_DIR, { recursive: true });
-  }
-
-  let downloaded = false;
-
-  try {
-    // Get latest release URL
-    const releaseData = await download(
-      `https://api.github.com/repos/${REPO}/releases/latest`,
-    );
-    const release = JSON.parse(releaseData.toString());
-    const asset = release.assets?.find((a) => a.name === assetName);
-
-    if (asset) {
-      const tarPath = join(INSTALL_DIR, assetName);
-      await downloadFile(asset.browser_download_url, tarPath);
-
-      // Extract
-      if (assetName.endsWith(".tar.gz")) {
-        execSync(`tar xzf "${tarPath}" -C "${INSTALL_DIR}"`, {
-          stdio: "ignore",
-        });
-      } else if (assetName.endsWith(".zip")) {
-        execSync(`unzip -o "${tarPath}" -d "${INSTALL_DIR}"`, {
-          stdio: "ignore",
-        });
-      }
-
-      // Make executable
-      await chmod(BINARY_PATH, 0o755);
-
-      // Cleanup archive
-      try {
-        execSync(`rm "${tarPath}"`, { stdio: "ignore" });
-      } catch {}
-
-      console.log(c("green", `  ✓ Installed to ${BINARY_PATH}`));
-      downloaded = true;
-    }
-  } catch (e) {
-    // Release not available yet — expected before first release
-  }
-
-  if (!downloaded) {
-    // Try cargo install as fallback
-    console.log(
-      c("yellow", "  No pre-built binary found. Trying cargo install..."),
-    );
-    try {
-      execSync(
-        `cargo install --git https://github.com/${REPO} webclaw-mcp --root "${INSTALL_DIR}"`,
-        { stdio: "inherit" },
-      );
-      // cargo install puts binary in INSTALL_DIR/bin/
-      const cargoPath = join(INSTALL_DIR, "bin", BINARY_NAME);
-      if (existsSync(cargoPath)) {
-        // Move to expected location
-        execSync(`mv "${cargoPath}" "${BINARY_PATH}"`, {
-          stdio: "ignore",
-        });
-        console.log(c("green", `  ✓ Built and installed to ${BINARY_PATH}`));
-        downloaded = true;
-      }
-    } catch {
-      console.log(
-        c("red", "  Failed to install. Make sure Rust is installed:"),
-      );
-      console.log(
-        c(
-          "dim",
-          "  curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh",
-        ),
-      );
-      process.exit(1);
-    }
-  }
-
-  console.log();
-
-  // 4. Configure each detected tool
-  console.log(c("bold", "  Configuring MCP servers..."));
+  // 3. Write the npx config into each detected tool.
+  console.log(c("bold", "  Writing MCP config..."));
   console.log();
 
   for (const tool of detected) {
     const configPath = tool.configPath();
     if (!configPath) continue;
-
     const writer = CONFIG_WRITERS[tool.id];
     if (!writer) continue;
-
     try {
       writer(configPath, apiKey || null);
       console.log(
@@ -551,31 +336,27 @@ async function main() {
       console.log(c("red", `  ✗ ${tool.name}: ${e.message}`));
     }
   }
-
   console.log();
 
-  // 5. Verify
-  if (downloaded) {
-    try {
-      const version = execSync(`"${BINARY_PATH}" --version`, {
-        encoding: "utf-8",
-      }).trim();
-      console.log(c("green", `  ✓ ${version}`));
-    } catch {
-      console.log(c("green", `  ✓ webclaw-mcp installed`));
-    }
-  }
-
-  // 6. Summary
+  // 4. Summary.
+  console.log(c("bold", "  Done! webclaw is configured."));
   console.log();
-  console.log(c("bold", "  Done! webclaw is ready."));
+  console.log(
+    c("dim", "  Your client runs the server on demand via npx — nothing to"),
+  );
+  console.log(
+    c("dim", "  install. First launch fetches @webclaw/mcp, then it's cached."),
+  );
   console.log();
-  console.log(c("dim", "  Your AI agent now has these tools:"));
-  console.log(c("dim", "    • scrape — extract content from any URL"));
-  console.log(c("dim", "    • crawl  — recursively crawl a website"));
-  console.log(c("dim", "    • search — web search + parallel scrape"));
-  console.log(c("dim", "    • map    — discover URLs from sitemaps"));
-  console.log(c("dim", "    • batch  — extract multiple URLs in parallel"));
+  console.log(
+    c("dim", "  Tools: scrape, search, crawl, map, batch, extract, summarize,"),
+  );
+  console.log(
+    c(
+      "dim",
+      "  diff, brand, research, lead, lead_batch, + 30 site extractors.",
+    ),
+  );
   console.log();
 
   if (!apiKey) {
@@ -583,7 +364,7 @@ async function main() {
     console.log(
       c(
         "dim",
-        "  Get an API key at https://webclaw.io/dashboard for cloud features.",
+        "  Get a key at https://webclaw.io/dashboard for cloud features.",
       ),
     );
     console.log();

packages/create-webclaw/package.json

@@ -1,12 +1,15 @@
 {
   "name": "create-webclaw",
-  "version": "0.1.4",
-  "mcpName": "io.github.0xMassi/webclaw",
+  "version": "0.1.7",
   "description": "Set up webclaw MCP server for AI agents (Claude, Cursor, Windsurf, OpenCode, Codex, Antigravity)",
   "bin": {
     "create-webclaw": "./index.mjs"
   },
   "type": "module",
+  "files": [
+    "index.mjs",
+    "README.md"
+  ],
   "keywords": [
     "webclaw",
     "mcp",

packages/create-webclaw/server.json

@@ -2,13 +2,13 @@
   "$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json",
   "name": "io.github.0xMassi/webclaw",
   "title": "webclaw",
-  "description": "Web extraction MCP server. Scrape, crawl, extract, summarize any URL to clean markdown.",
-  "version": "0.1.4",
+  "description": "Turn any URL into clean markdown/JSON for AI agents. Self-hostable web content extraction.",
+  "version": "0.6.17",
   "packages": [
     {
       "registryType": "npm",
-      "identifier": "create-webclaw",
-      "version": "0.1.4",
+      "identifier": "@webclaw/mcp",
+      "version": "0.6.16",
       "transport": {
         "type": "stdio"
       }

packages/webclaw-mcp/README.md

@@ -0,0 +1,71 @@
+# @webclaw/mcp
+
+**Clean web access for AI agents, over MCP.** Turn any URL into markdown, JSON, or LLM-ready context — including pages that block bots or need JavaScript — straight from Claude, Cursor, and any MCP client.
+
+Zero-install launcher for the [webclaw](https://webclaw.io) MCP server. `npx @webclaw/mcp` downloads a prebuilt `webclaw-mcp` binary once (verified against the release `SHA256SUMS`), caches it, and runs it as an MCP stdio server. No Rust build, no global install. It runs on your machine; the hosted webclaw cloud is used only for the tools that need it, and only when you set `WEBCLAW_API_KEY`.
+
+## Add to your MCP client
+
+Point any stdio MCP client at the npx command — Claude Desktop, Cursor, Windsurf, Antigravity (`mcpServers` JSON):
+
+```json
+{
+  "mcpServers": {
+    "webclaw": {
+      "command": "npx",
+      "args": ["-y", "@webclaw/mcp"]
+    }
+  }
+}
+```
+
+Add a key to unlock the cloud-backed tools (bot-protection bypass, JS rendering, web search, research, lead enrichment):
+
+```json
+{
+  "mcpServers": {
+    "webclaw": {
+      "command": "npx",
+      "args": ["-y", "@webclaw/mcp"],
+      "env": { "WEBCLAW_API_KEY": "wc_your_key" }
+    }
+  }
+}
+```
+
+Claude Code:
+
+```bash
+claude mcp add webclaw -- npx -y @webclaw/mcp
+```
+
+Or run `npx create-webclaw` to auto-detect your AI tools and write their configs for you.
+
+## Tools (14)
+
+scrape, search, crawl, map, batch, extract, summarize, diff, brand, research, lead, lead_batch, plus `list_extractors` / `vertical_scrape` for 30+ site-specific extractors (Amazon, GitHub, Reddit, YouTube, npm, PyPI, and more).
+
+- **No key needed:** scrape, crawl, map, batch, diff, brand, list_extractors, vertical_scrape.
+- **Needs an LLM** (local Ollama or a provider key): extract, summarize.
+- **Needs `WEBCLAW_API_KEY`:** search, research, lead, lead_batch — plus automatic bot-protection bypass and JS rendering for the fetch tools.
+
+Get a key at [webclaw.io](https://webclaw.io).
+
+## Environment
+
+| Variable | Purpose |
+|---|---|
+| `WEBCLAW_API_KEY` | Enables cloud-backed tools (optional). |
+| `WEBCLAW_MCP_BIN` | Absolute path to a `webclaw-mcp` binary; skips the download. |
+| `WEBCLAW_MCP_VERSION` | Release tag to install (default: the pinned release). |
+| `WEBCLAW_MCP_CACHE` | Cache directory (default: `~/.cache/webclaw`). |
+
+## Links
+
+- **Docs:** [webclaw.io/docs/mcp](https://webclaw.io/docs/mcp)
+- **Source** (CLI, REST API, SDKs, extraction engine): [github.com/0xMassi/webclaw](https://github.com/0xMassi/webclaw)
+- **Hosted API & keys:** [webclaw.io](https://webclaw.io)
+
+## License
+
+AGPL-3.0-only

packages/webclaw-mcp/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@webclaw/mcp",
+  "version": "0.6.16",
+  "description": "Zero-install launcher for the webclaw MCP server: web extraction and anti-bot access for AI agents. Runs `npx @webclaw/mcp`.",
+  "type": "module",
+  "bin": {
+    "webclaw-mcp-launcher": "./webclaw-mcp.mjs"
+  },
+  "mcpName": "io.github.0xMassi/webclaw",
+  "files": [
+    "webclaw-mcp.mjs",
+    "README.md"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "license": "AGPL-3.0-only",
+  "author": "0xMassi",
+  "homepage": "https://webclaw.io",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/0xMassi/webclaw.git",
+    "directory": "packages/webclaw-mcp"
+  },
+  "bugs": {
+    "url": "https://github.com/0xMassi/webclaw/issues"
+  },
+  "keywords": [
+    "mcp",
+    "model-context-protocol",
+    "webclaw",
+    "web-scraping",
+    "web-extraction",
+    "ai-agents",
+    "cloudflare",
+    "anti-bot",
+    "llm"
+  ]
+}

packages/webclaw-mcp/webclaw-mcp.mjs

@@ -0,0 +1,266 @@
+#!/usr/bin/env node
+/**
+ * @webclaw/mcp — zero-install launcher for the webclaw MCP server.
+ *
+ * `npx -y @webclaw/mcp` resolves a prebuilt `webclaw-mcp` binary (downloaded
+ * once from the pinned GitHub release, then cached) and execs it as an MCP
+ * stdio server. This is what makes webclaw introspectable/installable in MCP
+ * clients and registries (Claude Desktop, Cursor, Glama, Smithery, ...) without
+ * a Rust build.
+ *
+ * HARD RULE: stdout carries the MCP JSON-RPC stream. Every diagnostic this
+ * launcher emits MUST go to stderr, or it corrupts the protocol. Never
+ * console.log here — only logErr().
+ *
+ * Overrides (env):
+ *   WEBCLAW_MCP_BIN      absolute path to a webclaw-mcp binary; skip download
+ *   WEBCLAW_MCP_VERSION  release tag to install (default: pinned RELEASE_TAG)
+ *   WEBCLAW_MCP_CACHE    cache root (default: ~/.cache/webclaw)
+ *   GITHUB_TOKEN         only used to fetch SHA256SUMS if rate-limited
+ */
+
+import {
+  existsSync,
+  mkdirSync,
+  createWriteStream,
+  readFileSync,
+  renameSync,
+  copyFileSync,
+  rmSync,
+  chmodSync,
+} from "node:fs";
+import { homedir, platform, arch } from "node:os";
+import { join } from "node:path";
+import { spawn, execFileSync } from "node:child_process";
+import { createHash } from "node:crypto";
+import https from "node:https";
+
+const REPO = "0xMassi/webclaw";
+// Release the wrapper installs. Bump this (and the package version) on each
+// core release, or override at runtime with WEBCLAW_MCP_VERSION.
+const RELEASE_TAG = process.env.WEBCLAW_MCP_VERSION || "v0.6.15";
+
+const IS_WINDOWS = platform() === "win32";
+const BIN_NAME = IS_WINDOWS ? "webclaw-mcp.exe" : "webclaw-mcp";
+const CACHE_ROOT =
+  process.env.WEBCLAW_MCP_CACHE || join(homedir(), ".cache", "webclaw");
+const CACHE_DIR = join(CACHE_ROOT, RELEASE_TAG);
+const CACHED_BIN = join(CACHE_DIR, BIN_NAME);
+
+function logErr(msg) {
+  process.stderr.write(`[@webclaw/mcp] ${msg}\n`);
+}
+
+function target() {
+  const map = {
+    "darwin-arm64": "aarch64-apple-darwin",
+    "darwin-x64": "x86_64-apple-darwin",
+    "linux-x64": "x86_64-unknown-linux-gnu",
+    "linux-arm64": "aarch64-unknown-linux-gnu",
+    "win32-x64": "x86_64-pc-windows-msvc",
+  };
+  return map[`${platform()}-${arch()}`] || null;
+}
+
+// GET a URL to a Buffer, following redirects. Auth headers are dropped on
+// redirect so a token never reaches the release CDN (its signed URLs reject it).
+function getBuffer(url, headers = {}) {
+  return new Promise((resolve, reject) => {
+    https
+      .get(
+        url,
+        { headers: { "User-Agent": "@webclaw/mcp", ...headers } },
+        (res) => {
+          if (
+            res.statusCode >= 300 &&
+            res.statusCode < 400 &&
+            res.headers.location
+          ) {
+            res.resume();
+            return getBuffer(res.headers.location).then(resolve, reject);
+          }
+          if (res.statusCode !== 200) {
+            res.resume();
+            return reject(new Error(`HTTP ${res.statusCode} for ${url}`));
+          }
+          const chunks = [];
+          res.on("data", (c) => chunks.push(c));
+          res.on("end", () => resolve(Buffer.concat(chunks)));
+          res.on("error", reject);
+        },
+      )
+      .on("error", reject);
+  });
+}
+
+// Stream a URL to a file, following redirects.
+function getFile(url, dest) {
+  return new Promise((resolve, reject) => {
+    https
+      .get(url, { headers: { "User-Agent": "@webclaw/mcp" } }, (res) => {
+        if (
+          res.statusCode >= 300 &&
+          res.statusCode < 400 &&
+          res.headers.location
+        ) {
+          res.resume();
+          return getFile(res.headers.location, dest).then(resolve, reject);
+        }
+        if (res.statusCode !== 200) {
+          res.resume();
+          return reject(new Error(`HTTP ${res.statusCode} for ${url}`));
+        }
+        const out = createWriteStream(dest);
+        res.pipe(out);
+        out.on("finish", () => out.close(() => resolve()));
+        out.on("error", reject);
+      })
+      .on("error", reject);
+  });
+}
+
+function sha256(path) {
+  return createHash("sha256").update(readFileSync(path)).digest("hex");
+}
+
+function extract(archivePath, destDir) {
+  if (IS_WINDOWS) {
+    execFileSync(
+      "powershell",
+      [
+        "-NoProfile",
+        "-Command",
+        `Expand-Archive -Path '${archivePath}' -DestinationPath '${destDir}' -Force`,
+      ],
+      { stdio: "ignore" },
+    );
+  } else {
+    execFileSync("tar", ["xzf", archivePath, "-C", destDir], {
+      stdio: "ignore",
+    });
+  }
+}
+
+async function ensureBinary() {
+  // 0. Explicit local binary (dev / offline / CI).
+  const override = process.env.WEBCLAW_MCP_BIN;
+  if (override) {
+    if (!existsSync(override)) {
+      throw new Error(`WEBCLAW_MCP_BIN points at a missing file: ${override}`);
+    }
+    return override;
+  }
+
+  // 1. Cache hit for this release.
+  if (existsSync(CACHED_BIN)) return CACHED_BIN;
+
+  // 2. Download, verify, extract, cache.
+  const tgt = target();
+  if (!tgt) {
+    throw new Error(
+      `unsupported platform ${platform()}-${arch()} — install the webclaw-mcp binary manually ` +
+        `(https://github.com/${REPO}/releases) and set WEBCLAW_MCP_BIN`,
+    );
+  }
+  const ext = IS_WINDOWS ? "zip" : "tar.gz";
+  const assetName = `webclaw-${RELEASE_TAG}-${tgt}.${ext}`;
+  const base = `https://github.com/${REPO}/releases/download/${RELEASE_TAG}`;
+
+  mkdirSync(CACHE_DIR, { recursive: true });
+  const archivePath = join(CACHE_DIR, assetName);
+  const tmpPath = `${archivePath}.download`;
+
+  logErr(`fetching ${assetName} (first run for ${RELEASE_TAG}) ...`);
+  await getFile(`${base}/${assetName}`, tmpPath);
+
+  // Verify against the release SHA256SUMS. If the sums file can't be fetched we
+  // proceed with a warning rather than hard-failing an otherwise-good install.
+  try {
+    const apiHeaders = process.env.GITHUB_TOKEN
+      ? { Authorization: `Bearer ${process.env.GITHUB_TOKEN}` }
+      : {};
+    const sums = (await getBuffer(`${base}/SHA256SUMS`, apiHeaders)).toString(
+      "utf8",
+    );
+    const line = sums.split("\n").find((l) => l.trim().endsWith(assetName));
+    const expected = line ? line.trim().split(/\s+/)[0] : null;
+    if (expected) {
+      const actual = sha256(tmpPath);
+      if (actual !== expected) {
+        rmSync(tmpPath, { force: true });
+        throw new Error(
+          `checksum mismatch for ${assetName}: expected ${expected}, got ${actual}`,
+        );
+      }
+    } else {
+      logErr(
+        `warning: ${assetName} not found in SHA256SUMS — skipping verification`,
+      );
+    }
+  } catch (e) {
+    if (/checksum mismatch/.test(e.message)) throw e;
+    logErr(`warning: could not verify checksum (${e.message}) — proceeding`);
+  }
+
+  renameSync(tmpPath, archivePath);
+  extract(archivePath, CACHE_DIR);
+
+  // Archives hold a top-level `webclaw-<tag>-<target>/` dir with all binaries.
+  const extractedDir = join(CACHE_DIR, `webclaw-${RELEASE_TAG}-${tgt}`);
+  const extractedBin = join(extractedDir, BIN_NAME);
+  if (!existsSync(extractedBin)) {
+    throw new Error(`binary missing after extract: ${extractedBin}`);
+  }
+  copyFileSync(extractedBin, CACHED_BIN);
+  if (!IS_WINDOWS) chmodSync(CACHED_BIN, 0o755);
+
+  // Drop the archive and the extra binaries; keep only the cached webclaw-mcp.
+  try {
+    rmSync(extractedDir, { recursive: true, force: true });
+    rmSync(archivePath, { force: true });
+  } catch {
+    /* non-fatal */
+  }
+
+  logErr(`installed ${BIN_NAME} for ${RELEASE_TAG} → ${CACHED_BIN}`);
+  return CACHED_BIN;
+}
+
+async function main() {
+  let bin;
+  try {
+    bin = await ensureBinary();
+  } catch (e) {
+    logErr(`error: ${e.message}`);
+    process.exit(1);
+  }
+
+  // Hand off to the real server. stdio:inherit wires the MCP JSON-RPC stream
+  // straight through; env passes WEBCLAW_API_KEY and friends unchanged.
+  const child = spawn(bin, process.argv.slice(2), {
+    stdio: "inherit",
+    env: process.env,
+  });
+
+  child.on("error", (e) => {
+    logErr(`failed to start webclaw-mcp: ${e.message}`);
+    process.exit(1);
+  });
+  child.on("exit", (code, signal) => {
+    if (signal) {
+      // Re-raise the signal so the parent's exit reflects it.
+      process.kill(process.pid, signal);
+    } else {
+      process.exit(code ?? 0);
+    }
+  });
+
+  // Forward termination signals to the child so clients can stop the server.
+  for (const sig of ["SIGINT", "SIGTERM", "SIGHUP"]) {
+    process.on(sig, () => {
+      if (!child.killed) child.kill(sig);
+    });
+  }
+}
+
+main();

setup.sh

@@ -244,7 +244,7 @@ EOF
 # --- Server ---
 WEBCLAW_PORT=$server_port
 WEBCLAW_HOST=0.0.0.0
-WEBCLAW_AUTH_KEY=$auth_key
+WEBCLAW_API_KEY=$auth_key
 
 # --- Logging ---
 WEBCLAW_LOG=info

skill/SKILL.md

@@ -1,634 +0,0 @@
----
-name: webclaw
-description: Web extraction engine with antibot bypass. Scrape, crawl, extract, summarize, search, map, diff, monitor, research, and analyze any URL — including Cloudflare-protected sites. Use when you need reliable web content, the built-in web_fetch fails, or you need structured data extraction from web pages.
-homepage: https://webclaw.io
-user-invocable: true
-metadata: {"openclaw":{"emoji":"🦀","requires":{"env":["WEBCLAW_API_KEY"]},"primaryEnv":"WEBCLAW_API_KEY","homepage":"https://webclaw.io","install":[{"id":"npx","kind":"node","bins":["webclaw-mcp"],"label":"npx create-webclaw"}]}}
----
-
-# webclaw
-
-High-quality web extraction with automatic antibot bypass. Beats Firecrawl on extraction quality and handles Cloudflare, DataDome, and JS-rendered pages automatically.
-
-## When to use this skill
-
-- **Always** when you need to fetch web content and want reliable results
-- When `web_fetch` returns empty/blocked content (403, Cloudflare challenges)
-- When you need structured data extraction (pricing tables, product info)
-- When you need to crawl an entire site or discover all URLs
-- When you need LLM-optimized content (cleaner than raw markdown)
-- When you need to summarize a page without reading the full content
-- When you need to detect content changes between visits
-- When you need brand identity analysis (colors, fonts, logos)
-- When you need web search results with optional page scraping
-- When you need deep multi-source research on a topic
-- When you need AI-guided scraping to accomplish a goal on a page
-- When you need to monitor a URL for changes over time
-
-## API base
-
-All requests go to `https://api.webclaw.io/v1/`.
-
-Authentication: `Authorization: Bearer $WEBCLAW_API_KEY`
-
-## Endpoints
-
-### 1. Scrape — extract content from a single URL
-
-```bash
-curl -X POST https://api.webclaw.io/v1/scrape \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "url": "https://example.com",
-    "formats": ["markdown"],
-    "only_main_content": true
-  }'
-```
-
-**Request fields:**
-
-| Field | Type | Default | Description |
-|-------|------|---------|-------------|
-| `url` | string | required | URL to scrape |
-| `formats` | string[] | `["markdown"]` | Output formats: `markdown`, `text`, `llm`, `json` |
-| `include_selectors` | string[] | `[]` | CSS selectors to keep (e.g. `["article", ".content"]`) |
-| `exclude_selectors` | string[] | `[]` | CSS selectors to remove (e.g. `["nav", "footer", ".ads"]`) |
-| `only_main_content` | bool | `false` | Extract only the main article/content area |
-| `no_cache` | bool | `false` | Skip cache, fetch fresh |
-| `max_cache_age` | int | server default | Max acceptable cache age in seconds |
-
-**Response:**
-
-```json
-{
-  "url": "https://example.com",
-  "metadata": {
-    "title": "Example",
-    "description": "...",
-    "language": "en",
-    "word_count": 1234
-  },
-  "markdown": "# Page Title\n\nContent here...",
-  "cache": { "status": "miss" }
-}
-```
-
-**Format options:**
-- `markdown` — clean markdown, best for general use
-- `text` — plain text without formatting
-- `llm` — optimized for LLM consumption: includes page title, URL, and cleaned content with link references. Best for feeding to AI models.
-- `json` — full extraction result with all metadata
-
-**When antibot bypass activates** (automatic, no extra config):
-```json
-{
-  "antibot": {
-    "bypass": true,
-    "elapsed_ms": 3200
-  }
-}
-```
-
-### 2. Crawl — scrape an entire website
-
-Starts an async job. Poll for results.
-
-**Start crawl:**
-```bash
-curl -X POST https://api.webclaw.io/v1/crawl \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "url": "https://docs.example.com",
-    "max_depth": 3,
-    "max_pages": 50,
-    "use_sitemap": true
-  }'
-```
-
-Response: `{ "job_id": "abc-123", "status": "running" }`
-
-**Poll status:**
-```bash
-curl https://api.webclaw.io/v1/crawl/abc-123 \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY"
-```
-
-Response when complete:
-```json
-{
-  "job_id": "abc-123",
-  "status": "completed",
-  "total": 47,
-  "completed": 45,
-  "errors": 2,
-  "pages": [
-    {
-      "url": "https://docs.example.com/intro",
-      "markdown": "# Introduction\n...",
-      "metadata": { "title": "Intro", "word_count": 500 }
-    }
-  ]
-}
-```
-
-**Request fields:**
-
-| Field | Type | Default | Description |
-|-------|------|---------|-------------|
-| `url` | string | required | Starting URL |
-| `max_depth` | int | `3` | How many links deep to follow |
-| `max_pages` | int | `100` | Maximum pages to crawl |
-| `use_sitemap` | bool | `false` | Seed URLs from sitemap.xml |
-| `formats` | string[] | `["markdown"]` | Output formats per page |
-| `include_selectors` | string[] | `[]` | CSS selectors to keep |
-| `exclude_selectors` | string[] | `[]` | CSS selectors to remove |
-| `only_main_content` | bool | `false` | Main content only |
-
-### 3. Map — discover all URLs on a site
-
-Fast URL discovery without full content extraction.
-
-```bash
-curl -X POST https://api.webclaw.io/v1/map \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY" \
-  -H "Content-Type: application/json" \
-  -d '{"url": "https://example.com"}'
-```
-
-Response:
-```json
-{
-  "url": "https://example.com",
-  "count": 142,
-  "urls": [
-    "https://example.com/about",
-    "https://example.com/pricing",
-    "https://example.com/docs/intro"
-  ]
-}
-```
-
-### 4. Batch — scrape multiple URLs in parallel
-
-```bash
-curl -X POST https://api.webclaw.io/v1/batch \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "urls": [
-      "https://a.com",
-      "https://b.com",
-      "https://c.com"
-    ],
-    "formats": ["markdown"],
-    "concurrency": 5
-  }'
-```
-
-Response:
-```json
-{
-  "total": 3,
-  "completed": 3,
-  "errors": 0,
-  "results": [
-    { "url": "https://a.com", "markdown": "...", "metadata": {} },
-    { "url": "https://b.com", "markdown": "...", "metadata": {} },
-    { "url": "https://c.com", "error": "timeout" }
-  ]
-}
-```
-
-### 5. Extract — LLM-powered structured extraction
-
-Pull structured data from any page using a JSON schema or plain-text prompt.
-
-**With JSON schema:**
-```bash
-curl -X POST https://api.webclaw.io/v1/extract \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "url": "https://example.com/pricing",
-    "schema": {
-      "type": "object",
-      "properties": {
-        "plans": {
-          "type": "array",
-          "items": {
-            "type": "object",
-            "properties": {
-              "name": { "type": "string" },
-              "price": { "type": "string" },
-              "features": { "type": "array", "items": { "type": "string" } }
-            }
-          }
-        }
-      }
-    }
-  }'
-```
-
-**With prompt:**
-```bash
-curl -X POST https://api.webclaw.io/v1/extract \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "url": "https://example.com/pricing",
-    "prompt": "Extract all pricing tiers with names, monthly prices, and key features"
-  }'
-```
-
-Response:
-```json
-{
-  "url": "https://example.com/pricing",
-  "data": {
-    "plans": [
-      { "name": "Starter", "price": "$49/mo", "features": ["10k pages", "Email support"] },
-      { "name": "Pro", "price": "$99/mo", "features": ["100k pages", "Priority support", "API access"] }
-    ]
-  }
-}
-```
-
-### 6. Summarize — get a quick summary of any page
-
-```bash
-curl -X POST https://api.webclaw.io/v1/summarize \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "url": "https://example.com/long-article",
-    "max_sentences": 3
-  }'
-```
-
-Response:
-```json
-{
-  "url": "https://example.com/long-article",
-  "summary": "The article discusses... Key findings include... The author concludes that..."
-}
-```
-
-### 7. Diff — detect content changes
-
-Compare current page content against a previous snapshot.
-
-```bash
-curl -X POST https://api.webclaw.io/v1/diff \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "url": "https://example.com",
-    "previous": {
-      "markdown": "# Old content...",
-      "metadata": { "title": "Old Title" }
-    }
-  }'
-```
-
-Response:
-```json
-{
-  "url": "https://example.com",
-  "status": "changed",
-  "diff": "--- previous\n+++ current\n@@ -1 +1 @@\n-# Old content\n+# New content",
-  "metadata_changes": [
-    { "field": "title", "old": "Old Title", "new": "New Title" }
-  ]
-}
-```
-
-### 8. Brand — extract brand identity
-
-Analyze a website's visual identity: colors, fonts, logo.
-
-```bash
-curl -X POST https://api.webclaw.io/v1/brand \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY" \
-  -H "Content-Type: application/json" \
-  -d '{"url": "https://example.com"}'
-```
-
-Response:
-```json
-{
-  "url": "https://example.com",
-  "brand": {
-    "colors": [
-      { "hex": "#FF6B35", "usage": "primary" },
-      { "hex": "#1A1A2E", "usage": "background" }
-    ],
-    "fonts": ["Inter", "JetBrains Mono"],
-    "logo_url": "https://example.com/logo.svg",
-    "favicon_url": "https://example.com/favicon.ico"
-  }
-}
-```
-
-### 9. Search — web search with optional scraping
-
-Search the web and optionally scrape each result page.
-
-```bash
-curl -X POST https://api.webclaw.io/v1/search \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "query": "best rust web frameworks 2026",
-    "num_results": 5,
-    "scrape": true,
-    "formats": ["markdown"]
-  }'
-```
-
-**Request fields:**
-
-| Field | Type | Default | Description |
-|-------|------|---------|-------------|
-| `query` | string | required | Search query |
-| `num_results` | int | `10` | Number of search results to return |
-| `scrape` | bool | `false` | Also scrape each result page for full content |
-| `formats` | string[] | `["markdown"]` | Output formats when `scrape` is true |
-| `country` | string | none | Country code for localized results (e.g. `"us"`, `"de"`) |
-| `lang` | string | none | Language code for results (e.g. `"en"`, `"fr"`) |
-
-**Response:**
-
-```json
-{
-  "query": "best rust web frameworks 2026",
-  "results": [
-    {
-      "title": "Top Rust Web Frameworks in 2026",
-      "url": "https://blog.example.com/rust-frameworks",
-      "snippet": "A comprehensive comparison of Axum, Actix, and Rocket...",
-      "position": 1,
-      "markdown": "# Top Rust Web Frameworks\n\n..."
-    },
-    {
-      "title": "Choosing a Rust Backend Framework",
-      "url": "https://dev.to/rust-backends",
-      "snippet": "When starting a new Rust web project...",
-      "position": 2,
-      "markdown": "# Choosing a Rust Backend\n\n..."
-    }
-  ]
-}
-```
-
-The `markdown` field on each result is only present when `scrape: true`. Without it, you get titles, URLs, snippets, and positions only.
-
-### 10. Research — deep multi-source research
-
-Starts an async research job that searches, scrapes, and synthesizes information across multiple sources. Poll for results.
-
-**Start research:**
-```bash
-curl -X POST https://api.webclaw.io/v1/research \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "query": "How does Cloudflare Turnstile work and what are its known bypass methods?",
-    "max_iterations": 5,
-    "max_sources": 10,
-    "topic": "security",
-    "deep": true
-  }'
-```
-
-**Request fields:**
-
-| Field | Type | Default | Description |
-|-------|------|---------|-------------|
-| `query` | string | required | Research question or topic |
-| `max_iterations` | int | server default | Maximum research iterations (search-read-analyze cycles) |
-| `max_sources` | int | server default | Maximum number of sources to consult |
-| `topic` | string | none | Topic hint to guide search strategy (e.g. `"security"`, `"finance"`, `"engineering"`) |
-| `deep` | bool | `false` | Enable deep research mode for more thorough analysis (costs 10 credits instead of 1) |
-
-Response: `{ "id": "res-abc-123", "status": "running" }`
-
-**Poll results:**
-```bash
-curl https://api.webclaw.io/v1/research/res-abc-123 \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY"
-```
-
-Response when complete:
-```json
-{
-  "id": "res-abc-123",
-  "status": "completed",
-  "query": "How does Cloudflare Turnstile work and what are its known bypass methods?",
-  "report": "# Cloudflare Turnstile Analysis\n\n## Overview\nCloudflare Turnstile is a CAPTCHA replacement that...\n\n## How It Works\n...\n\n## Known Bypass Methods\n...",
-  "sources": [
-    { "url": "https://developers.cloudflare.com/turnstile/", "title": "Turnstile Documentation" },
-    { "url": "https://blog.cloudflare.com/turnstile-ga/", "title": "Turnstile GA Announcement" }
-  ],
-  "findings": [
-    "Turnstile uses browser environment signals and proof-of-work challenges",
-    "Managed mode auto-selects challenge difficulty based on visitor risk score",
-    "Known bypass approaches include instrumented browser automation"
-  ],
-  "iterations": 5,
-  "elapsed_ms": 34200
-}
-```
-
-**Status values:** `running`, `completed`, `failed`
-
-### 11. Agent Scrape — AI-guided scraping
-
-Use an AI agent to navigate and interact with a page to accomplish a specific goal. The agent can click, scroll, fill forms, and extract data across multiple steps.
-
-```bash
-curl -X POST https://api.webclaw.io/v1/agent-scrape \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "url": "https://example.com/products",
-    "goal": "Find the cheapest laptop with at least 16GB RAM and extract its full specs",
-    "max_steps": 10
-  }'
-```
-
-**Request fields:**
-
-| Field | Type | Default | Description |
-|-------|------|---------|-------------|
-| `url` | string | required | Starting URL |
-| `goal` | string | required | What the agent should accomplish |
-| `max_steps` | int | server default | Maximum number of actions the agent can take |
-
-**Response:**
-
-```json
-{
-  "url": "https://example.com/products",
-  "result": "The cheapest laptop with 16GB+ RAM is the ThinkPad E14 Gen 6 at $649. Specs: AMD Ryzen 5 7535U, 16GB DDR4, 512GB SSD, 14\" FHD IPS display, 57Wh battery.",
-  "steps": [
-    { "action": "navigate", "detail": "Loaded products page" },
-    { "action": "click", "detail": "Clicked 'Laptops' category filter" },
-    { "action": "click", "detail": "Applied '16GB+' RAM filter" },
-    { "action": "click", "detail": "Sorted by price: low to high" },
-    { "action": "extract", "detail": "Extracted specs from first matching product" }
-  ]
-}
-```
-
-### 12. Watch — monitor a URL for changes
-
-Create persistent monitors that check a URL on a schedule and notify via webhook when content changes.
-
-**Create a monitor:**
-```bash
-curl -X POST https://api.webclaw.io/v1/watch \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "url": "https://example.com/pricing",
-    "interval": "0 */6 * * *",
-    "webhook_url": "https://hooks.example.com/pricing-changed",
-    "formats": ["markdown"]
-  }'
-```
-
-**Request fields:**
-
-| Field | Type | Default | Description |
-|-------|------|---------|-------------|
-| `url` | string | required | URL to monitor |
-| `interval` | string | required | Check frequency as cron expression or seconds (e.g. `"0 */6 * * *"` or `"3600"`) |
-| `webhook_url` | string | none | URL to POST when changes are detected |
-| `formats` | string[] | `["markdown"]` | Output formats for snapshots |
-
-Response:
-```json
-{
-  "id": "watch-abc-123",
-  "url": "https://example.com/pricing",
-  "interval": "0 */6 * * *",
-  "webhook_url": "https://hooks.example.com/pricing-changed",
-  "formats": ["markdown"],
-  "created_at": "2026-03-20T10:00:00Z",
-  "last_check": null,
-  "status": "active"
-}
-```
-
-**List all monitors:**
-```bash
-curl https://api.webclaw.io/v1/watch \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY"
-```
-
-Response:
-```json
-{
-  "monitors": [
-    {
-      "id": "watch-abc-123",
-      "url": "https://example.com/pricing",
-      "interval": "0 */6 * * *",
-      "status": "active",
-      "last_check": "2026-03-20T16:00:00Z",
-      "checks": 4
-    }
-  ]
-}
-```
-
-**Get a monitor with snapshots:**
-```bash
-curl https://api.webclaw.io/v1/watch/watch-abc-123 \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY"
-```
-
-Response:
-```json
-{
-  "id": "watch-abc-123",
-  "url": "https://example.com/pricing",
-  "interval": "0 */6 * * *",
-  "status": "active",
-  "snapshots": [
-    {
-      "checked_at": "2026-03-20T16:00:00Z",
-      "status": "changed",
-      "diff": "--- previous\n+++ current\n@@ -5 +5 @@\n-Pro: $99/mo\n+Pro: $119/mo"
-    },
-    {
-      "checked_at": "2026-03-20T10:00:00Z",
-      "status": "baseline"
-    }
-  ]
-}
-```
-
-**Trigger an immediate check:**
-```bash
-curl -X POST https://api.webclaw.io/v1/watch/watch-abc-123/check \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY"
-```
-
-**Delete a monitor:**
-```bash
-curl -X DELETE https://api.webclaw.io/v1/watch/watch-abc-123 \
-  -H "Authorization: Bearer $WEBCLAW_API_KEY"
-```
-
-## Choosing the right format
-
-| Goal | Format | Why |
-|------|--------|-----|
-| Read and understand a page | `markdown` | Clean structure, headings, links preserved |
-| Feed content to an AI model | `llm` | Optimized: includes title + URL header, clean link refs |
-| Search or index content | `text` | Plain text, no formatting noise |
-| Programmatic analysis | `json` | Full metadata, structured data, DOM statistics |
-
-## Tips
-
-- **Use `llm` format** when passing content to yourself or another AI — it's specifically optimized for LLM consumption with better context framing.
-- **Use `only_main_content: true`** to skip navigation, sidebars, and footers. Reduces noise significantly.
-- **Use `include_selectors`/`exclude_selectors`** for fine-grained control when `only_main_content` isn't enough.
-- **Batch over individual scrapes** when fetching multiple URLs — it's faster and more efficient.
-- **Use `map` before `crawl`** to discover the site structure first, then crawl specific sections.
-- **Use `extract` with a JSON schema** for reliable structured output (e.g., pricing tables, product specs, contact info).
-- **Antibot bypass is automatic** — no extra configuration needed. Works on Cloudflare, DataDome, AWS WAF, and JS-rendered SPAs.
-- **Use `search` with `scrape: true`** to get full page content for each search result in one call instead of searching then scraping separately.
-- **Use `research` for complex questions** that need multiple sources — it handles the search-read-synthesize loop automatically. Enable `deep: true` for thorough analysis.
-- **Use `agent-scrape` for interactive pages** where data is behind filters, pagination, or form submissions that a simple scrape cannot reach.
-- **Use `watch` for ongoing monitoring** — set up a cron schedule and a webhook to get notified when a page changes without polling manually.
-
-## Smart Fetch Architecture
-
-The webclaw MCP server uses a **local-first** approach:
-
-1. **Local fetch** — fast, free, no API credits used (~80% of sites)
-2. **Cloud API fallback** — automatic when bot protection or JS rendering is detected
-
-This means:
-- Most scrapes cost zero credits (local extraction)
-- Cloudflare, DataDome, AWS WAF sites automatically fall back to the cloud API
-- JS-rendered SPAs (React, Next.js, Vue) also fall back automatically
-- Set `WEBCLAW_API_KEY` to enable cloud fallback
-
-## vs web_fetch
-
-| | webclaw | web_fetch |
-|---|---------|-----------|
-| Cloudflare bypass | Automatic (cloud fallback) | Fails (403) |
-| JS-rendered pages | Automatic fallback | Readability only |
-| Output quality | 20-step optimization pipeline | Basic HTML parsing |
-| Structured extraction | LLM-powered, schema-based | None |
-| Crawling | Full site crawl with sitemap | Single page only |
-| Caching | Built-in, configurable TTL | Per-session |
-| Rate limiting | Managed server-side | Client responsibility |
-
-Use `web_fetch` for simple, fast lookups. Use webclaw when you need reliability, quality, or advanced features.

smithery.yaml

@@ -16,8 +16,8 @@ startCommand:
         secret: true
   commandFunction: |
     (config) => ({
-      command: 'webclaw-mcp',
-      args: [],
+      command: 'npx',
+      args: ['-y', '@webclaw/mcp'],
       env: config.apiKey ? { WEBCLAW_API_KEY: config.apiKey } : {}
     })
   exampleConfig: