diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index 650984e..1d603b9 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -10,4 +10,4 @@ lfx_crowdfunding: polar: buy_me_a_coffee: thanks_dev: -custom: +custom: ["https://webclaw.io/sponsor"] diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d79c80f..ce28330 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -188,6 +188,40 @@ jobs: $prerelease \ --generate-notes + npm: + name: npm (@webclaw/mcp) + needs: release + # Publish the zero-install MCP launcher on STABLE tags only (prerelease + # tags contain a hyphen and are skipped). Requires the NPM_TOKEN repo + # secret: an npm automation token with publish rights on the @webclaw scope. + if: ${{ github.event_name == 'push' && !contains(github.ref_name, '-') }} + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-node@v4 + with: + node-version: 22 + registry-url: https://registry.npmjs.org + - name: Pin launcher to this release and publish + working-directory: packages/webclaw-mcp + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: | + tag="${GITHUB_REF#refs/tags/}" # e.g. v0.6.17 (core release) + # Pin which prebuilt binary this launcher downloads to the just-released tag. + sed -i -E "/const RELEASE_TAG =/ s|\"v[0-9.]+\"|\"${tag}\"|" webclaw-mcp.mjs + # The launcher is versioned INDEPENDENTLY of the core release (a + # launcher-only fix must be shippable without a core bump). Publish the + # next patch above whatever is currently on npm — no commit-back needed. + cur="$(npm view @webclaw/mcp version 2>/dev/null || echo 0.0.0)" + npm version "$cur" --no-git-tag-version --allow-same-version >/dev/null + npm version patch --no-git-tag-version >/dev/null + echo "publishing @webclaw/mcp@$(node -p "require('./package.json').version") pinned to ${tag}:" + grep -n "RELEASE_TAG =" webclaw-mcp.mjs + npm publish --access public + docker: name: Docker needs: release diff --git a/.gitignore b/.gitignore index 7a5a785..490e243 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,6 @@ local-test-results.json # not code and shouldn't live in git. Track deliberately-saved research # output under a different name. research-*.json +# MCP registry publisher credentials (mcp-publisher writes these into +# packages/create-webclaw/). They are OAuth/API tokens — never commit them. +.mcpregistry_*_token diff --git a/.mcp.json b/.mcp.json index fabcd4f..a1eca74 100644 --- a/.mcp.json +++ b/.mcp.json @@ -1,7 +1,8 @@ { "mcpServers": { "webclaw": { - "command": "~/.webclaw/webclaw-mcp" + "command": "npx", + "args": ["-y", "@webclaw/mcp"] } } } diff --git a/CHANGELOG.md b/CHANGELOG.md index 003d53d..f2662ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,22 @@ All notable changes to webclaw are documented here. Format follows [Keep a Changelog](https://keepachangelog.com/). -## [Unreleased] +## [0.6.16] - 2026-07-22 + +### Added +- **`@webclaw/mcp` — zero-install MCP launcher.** `npx -y @webclaw/mcp` downloads the prebuilt `webclaw-mcp` binary once, verifies it against the release `SHA256SUMS`, caches it, and runs it as an MCP stdio server. This is now the canonical way to add webclaw to an MCP client (`{"command": "npx", "args": ["-y", "@webclaw/mcp"]}`) — no Rust build — and it makes the server introspectable in MCP registries (Glama, Smithery, the MCP registry). `create-webclaw` stays as the auto-detect installer. +- **Atlas Cloud provider in the LLM chain (opt-in).** Set `ATLASCLOUD_API_KEY` to add [Atlas Cloud](https://atlascloud.ai) as a provider for the LLM features (extraction, summarization). It's added at the end of the chain — Ollama → OpenAI → Gemini → Anthropic → Atlas Cloud — so it only runs when you configure it and never preempts an already-configured provider. Override the model with `ATLASCLOUD_MODEL` (default `qwen/qwen3.5-flash`) and the endpoint with `ATLASCLOUD_BASE_URL`. + +### Fixed +- **`--urls-file` with a single URL now works.** A file containing exactly one URL (with no positional URL on the command line) was rejected with *"no input provided"*: it fell between the batch path, which skips single-entry files, and the single-URL path, which only reads positional arguments. A lone file URL is now treated exactly like `webclaw `, so it works with stdout, `--output-dir`, `--brand`, `--diff-with`, `--crawl`, `--watch`, and the LLM paths. + +## [0.6.15] - 2026-07-18 + +### Added +- **`lead` and `lead_batch` MCP tools.** Turn a company URL into an outreach-ready lead — founders and leadership with their LinkedIn and X, plus company summary, socials, tech stack, pricing, and public emails — straight from any MCP client (Claude Desktop, Cursor, Claude Code, …). `lead` enriches one company; `lead_batch` enriches up to 25 at once and blocks until the batch finishes. Both require `WEBCLAW_API_KEY`. + +### Fixed +- **`webclaw --help` no longer prints secret environment values.** Secret env vars are masked in help output instead of being shown in the clear. ## [0.6.14] - 2026-06-27 diff --git a/CLAUDE.md b/CLAUDE.md index 959e49c..c34155d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -63,7 +63,7 @@ Three binaries: `webclaw` (CLI), `webclaw-mcp` (MCP server), `webclaw-server` (R - `url_security.rs` — SSRF guards + SSRF-safe redirect policy ### LLM Modules (`webclaw-llm`) -- Provider chain (`chain.rs`): Ollama (local-first, always added; availability checked at call time) -> OpenAI -> Gemini -> Anthropic. Gemini sits ahead of Anthropic so Google Cloud credits are preferred; Anthropic is the last-resort fallback. Each provider lives in `providers/` (`ollama.rs`, `openai.rs`, `gemini.rs`, `anthropic.rs`). +- Provider chain (`chain.rs`): Ollama (local-first, always added; availability checked at call time) -> OpenAI -> Gemini -> Anthropic -> Atlas Cloud. Gemini sits ahead of Anthropic so Google Cloud credits are preferred; Anthropic is the last-resort fallback. Atlas Cloud is opt-in and added last — only when `ATLASCLOUD_API_KEY` is set — so it never preempts an already-configured provider (`ATLASCLOUD_MODEL` / `ATLASCLOUD_BASE_URL` override its model/endpoint). Each provider lives in `providers/` (`ollama.rs`, `openai.rs`, `gemini.rs`, `anthropic.rs`, `atlascloud.rs`). - JSON schema extraction, prompt-based extraction, summarization ### PDF Modules (`webclaw-pdf`) @@ -71,7 +71,7 @@ Three binaries: `webclaw` (CLI), `webclaw-mcp` (MCP server), `webclaw-server` (R ### MCP Server (`webclaw-mcp`) - Model Context Protocol server over stdio transport -- 12 tools: scrape, crawl, map, batch, extract, summarize, diff, brand, research, search, list_extractors, vertical_scrape. `search` is local-first via the caller's `SERPER_API_KEY` (falls back to the hosted API when unset); `research` uses the hosted deep-research API. The rest run locally. +- 14 tools: scrape, crawl, map, batch, extract, summarize, diff, brand, research, search, list_extractors, vertical_scrape, lead, lead_batch. `search` is local-first via the caller's `SERPER_API_KEY` (falls back to the hosted API when unset); `research`, `lead`, and `lead_batch` are hosted-only cloud proxies (require `WEBCLAW_API_KEY`) — `lead`/`lead_batch` call `/v1/lead` and the async `/v1/lead/batch`. The rest run locally. - Works with Claude Desktop, Claude Code, and any MCP client - Uses `rmcp` crate (official Rust MCP SDK) @@ -117,13 +117,14 @@ CI (`.github/workflows/ci.yml`, with `RUSTFLAGS=--cfg reqwest_unstable`) runs fo ## Repo Layout & Packaging -Workspace is version **0.6.13**, edition **2024**, license **AGPL-3.0** (matters for the public-OSS scrubbing rules). No crate declares `rust-version`, so MSRV is implicit — edition 2024 floors it at Rust 1.85+; CI pins `dtolnay/rust-toolchain@stable`. +Workspace is version **0.6.15**, edition **2024**, license **AGPL-3.0** (matters for the public-OSS scrubbing rules). No crate declares `rust-version`, so MSRV is implicit — edition 2024 floors it at Rust 1.85+; CI pins `dtolnay/rust-toolchain@stable`. Artifacts outside `crates/` that need separate attention: - `packages/create-webclaw/` — `npx create-webclaw` Node scaffolder that installs/configures the MCP server for AI agents (Claude, Cursor, Windsurf, ...). Versioned independently (own `package.json`) — bump it separately when MCP setup changes. - `smithery.yaml` + `glama.json` — MCP-registry manifests (Smithery stdio config spawning `webclaw-mcp` with optional `WEBCLAW_API_KEY`; Glama). Update when the MCP launch command or env changes. - `examples/` — runnable demos (cloudflare-diagnostics, firecrawl-compatible-api, html-to-markdown-rag, mcp-web-scraping, proxy-backed-crawling). -- `Dockerfile` / `Dockerfile.ci` / `docker-compose.yml`, `benchmarks/` (`/benchmark` skill), `SKILL.md` + `skill/` (Claude Code skill). +- `Dockerfile` / `Dockerfile.ci` / `docker-compose.yml`, `benchmarks/` (`/benchmark` skill). +- The Claude Code / agent skill is NOT in this repo — it lives in its own skills.sh repo `github.com/0xMassi/webclaw-skill` (`skills/webclaw` + `skills/lead-enrichment`), installed via `npx skills add 0xMassi/webclaw-skill`. ## CLI @@ -190,7 +191,8 @@ Add to Claude Desktop config (`~/Library/Application Support/Claude/claude_deskt { "mcpServers": { "webclaw": { - "command": "/path/to/webclaw-mcp" + "command": "npx", + "args": ["-y", "@webclaw/mcp"] } } } diff --git a/Cargo.lock b/Cargo.lock index 3e72c87..e09d11c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3221,7 +3221,7 @@ dependencies = [ [[package]] name = "webclaw-cli" -version = "0.6.14" +version = "0.6.16" dependencies = [ "clap", "dotenvy", @@ -3242,7 +3242,7 @@ dependencies = [ [[package]] name = "webclaw-core" -version = "0.6.14" +version = "0.6.16" dependencies = [ "ego-tree", "once_cell", @@ -3260,7 +3260,7 @@ dependencies = [ [[package]] name = "webclaw-fetch" -version = "0.6.14" +version = "0.6.16" dependencies = [ "async-trait", "bytes", @@ -3288,7 +3288,7 @@ dependencies = [ [[package]] name = "webclaw-llm" -version = "0.6.14" +version = "0.6.16" dependencies = [ "async-trait", "reqwest", @@ -3301,7 +3301,7 @@ dependencies = [ [[package]] name = "webclaw-mcp" -version = "0.6.14" +version = "0.6.16" dependencies = [ "dirs", "dotenvy", @@ -3321,7 +3321,7 @@ dependencies = [ [[package]] name = "webclaw-pdf" -version = "0.6.14" +version = "0.6.16" dependencies = [ "pdf-extract", "thiserror", @@ -3330,7 +3330,7 @@ dependencies = [ [[package]] name = "webclaw-server" -version = "0.6.14" +version = "0.6.16" dependencies = [ "anyhow", "axum", diff --git a/Cargo.toml b/Cargo.toml index aa757c0..295f22f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ resolver = "2" members = ["crates/*"] [workspace.package] -version = "0.6.14" +version = "0.6.16" edition = "2024" license = "AGPL-3.0" repository = "https://github.com/0xMassi/webclaw" diff --git a/README.md b/README.md index b4f59f6..9fe5c3e 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +

English | 简体中文

+

webclaw @@ -25,6 +27,10 @@ Docs

+

+ 0xMassi/webclaw | Trendshift +

+

webclaw extracting clean markdown from a page

@@ -164,22 +170,21 @@ webclaw https://example.com/pricing --diff-with pricing-old.json webclaw ships with an MCP server for AI agents. -```bash -npx create-webclaw -``` - -Manual config: +Zero-install — point any MCP client at the npx launcher: ```json { "mcpServers": { "webclaw": { - "command": "~/.webclaw/webclaw-mcp" + "command": "npx", + "args": ["-y", "@webclaw/mcp"] } } } ``` +Or run `npx create-webclaw` to auto-detect your AI tools and write their configs for you. + Then ask your agent things like: ```text @@ -196,6 +201,22 @@ Extract the brand colors, fonts, and logos from this company website. --- +## Use as an agent skill + +Add webclaw to Claude Code, Cursor, Windsurf, and other MCP agents in one command: + +```bash +npx skills add 0xMassi/webclaw-skill +``` + +Your agent gets scrape, crawl, map, extract, summarize, diff, brand, and search +as native tools. Most sites extract locally with no API key. Set `WEBCLAW_API_KEY` +to handle bot-protected and JavaScript-rendered pages. + +Find it on [skills.sh](https://www.skills.sh/0xMassi/webclaw-skill/webclaw). + +--- + ## Tools | Tool | What it does | Local | @@ -390,7 +411,7 @@ Please remove secrets, cookies, private tokens, and customer data from logs befo @@ -400,7 +421,8 @@ Please remove secrets, cookies, private tokens, and customer data from logs befo ColdProxy supports webclaw as an Infrastructure Partner, providing residential IPv4, residential IPv6, and datacenter IPv6 proxy infrastructure across 195+ countries for public data collection, regional testing, monitoring, and web scraping workflows. Explore - ColdProxy's latest plans and available offers directly on the website. + ColdProxy's latest plans and available offers directly on the website. + Use code webclaw8Off for 8% off your first payment. See the proxy-backed crawling guide for a hands-on walkthrough of wiring ColdProxy into webclaw. @@ -414,7 +436,7 @@ Please remove secrets, cookies, private tokens, and customer data from logs befo
- + ColdProxy
@@ -424,33 +446,7 @@ Please remove secrets, cookies, private tokens, and customer data from logs befo sticky sessions up to 7 days, IP filtering (all proxies under a 97% fraud score), no KYC, and cashback up to 10% on traffic. Use WEBCLAW35 for 35% off Mobile and Residential proxies, or WEBCLAW40 for 40% off ISP (Static) proxies at - NodeMaven. - - - - - - - - - @@ -494,11 +490,11 @@ Thanks to everyone improving webclaw through issues, examples, docs, bug reports ## Star History - + - - - Star History Chart + + + Star History Chart diff --git a/README_zh-CN.md b/README_zh-CN.md new file mode 100644 index 0000000..1321ad5 --- /dev/null +++ b/README_zh-CN.md @@ -0,0 +1,495 @@ +

English | 简体中文

+ +

+ + webclaw + +

+ +

webclaw

+ +

+ 将任意网页转化为干净的 Markdown、JSON 与 LLM 就绪的上下文。
+ 面向 AI 智能体与 RAG 流水线的 CLI、MCP 服务器、REST API 与多语言 SDK。 +

+ +

+ Stars + Version + License + npm installs +

+ +

+ Discord + X / Twitter + Hosted webclaw + Docs +

+ +

+ webclaw 从网页中提取干净的 Markdown +

+ +--- + +大多数网页抓取工具，只会给你的智能体两种糟糕的结果之一： + +- 被拦截的页面、登录墙，或空壳应用（app shell） +- 塞满导航栏、脚本、样式、广告和重复模板的原始 HTML + +[webclaw.io](https://webclaw.io) 是 webclaw 的托管式网页提取 API。本仓库包含开源的 CLI、MCP 服务器、提取引擎，以及可自托管的服务端。 + +webclaw 用 Rust 编写，把一个 URL 变成你的工具真正用得上的干净内容——它是一个**开源、可自托管的 Firecrawl 替代方案**。 + +```bash +webclaw https://example.com --format markdown +``` + +```md +# Example Domain + +This domain is for use in illustrative examples in documents. + +You may use this domain in literature without prior coordination or asking for permission. +``` + +在终端里直接用，通过 MCP 接入 Claude / Cursor，从你的应用调用托管 API，或者自托管这套开源服务端——任你选择。 + +--- + +## 安装 + +### 智能体一键配置 + +把 webclaw 接入 Claude Code、Claude Desktop、Cursor、Windsurf、OpenCode、Codex CLI 以及其他兼容 MCP 的工具，最快的方式是： + +```bash +npx create-webclaw +``` + +安装器会自动检测已支持的客户端，并为你写好 MCP 服务器的配置。 + +### Homebrew + +```bash +brew tap 0xMassi/webclaw +brew install webclaw +``` + +### 预编译二进制 + +从 [GitHub Releases](https://github.com/0xMassi/webclaw/releases) 下载 macOS、Linux 和 Windows 二进制文件。 + +### Docker + +```bash +docker run --rm ghcr.io/0xmassi/webclaw https://example.com +``` + +### Cargo + +```bash +cargo install --git https://github.com/0xMassi/webclaw.git webclaw-cli +cargo install --git https://github.com/0xMassi/webclaw.git webclaw-mcp +``` + +如果从源码构建时因缺少本机构建工具而失败，请先安装对应平台的依赖： + +| 操作系统 | 命令 | +| --- | --- | +| Debian / Ubuntu | `sudo apt install -y pkg-config libssl-dev cmake clang git build-essential` | +| Fedora / RHEL | `sudo dnf install -y pkg-config openssl-devel cmake clang git make gcc` | +| Arch | `sudo pacman -S pkg-config openssl cmake clang git base-devel` | +| macOS | `xcode-select --install` | + +--- + +## 快速开始 + +### 抓取单个页面 + +```bash +webclaw https://stripe.com --format markdown +``` + +### 返回 LLM 优化文本 + +```bash +webclaw https://docs.anthropic.com --format llm +``` + +### 只保留正文内容 + +```bash +webclaw https://example.com/blog/post --only-main-content +``` + +### 包含或排除选择器 + +```bash +webclaw https://example.com \ + --include "article, main, .content" \ + --exclude "nav, footer, .sidebar, .ad" +``` + +### 爬取一个文档站点 + +```bash +webclaw https://docs.rust-lang.org --crawl --depth 2 --max-pages 50 +``` + +### 工作流示例 + +- [HTML 转 Markdown 用于 RAG](examples/html-to-markdown-rag/) +- [兼容 Firecrawl 的 API](examples/firecrawl-compatible-api/) +- [基于 MCP 的网页抓取](examples/mcp-web-scraping/) +- [使用 ColdProxy 的代理支持爬取](examples/proxy-backed-crawling/) + +### 提取品牌资产 + +```bash +webclaw https://github.com --brand +``` + +### 对比页面随时间的变化 + +```bash +webclaw https://example.com/pricing --format json > pricing-old.json +webclaw https://example.com/pricing --diff-with pricing-old.json +``` + +--- + +## MCP 服务器 + +webclaw 内置了面向 AI 智能体的 MCP 服务器。 + +无需安装——把任意 MCP 客户端指向 npx 启动器即可： + +```json +{ + "mcpServers": { + "webclaw": { + "command": "npx", + "args": ["-y", "@webclaw/mcp"] + } + } +} +``` + +或者运行 `npx create-webclaw`，让它自动检测你的 AI 工具并替你写好配置。 + +配好之后，就可以让你的智能体做这样的事： + +```text +抓取这些竞品的定价页面，并总结它们之间的差异。 +``` + +```text +爬取这个文档站点，为 RAG 索引准备干净的上下文。 +``` + +```text +从这家公司的官网提取品牌色、字体和 logo。 +``` + +--- + +## 作为智能体 Skill 使用 + +一条命令，就能把 webclaw 添加到 Claude Code、Cursor、Windsurf 及其他 MCP 智能体： + +```bash +npx skills add 0xMassi/webclaw-skill +``` + +你的智能体会把 scrape、crawl、map、extract、summarize、diff、brand、search 作为原生工具使用。多数站点无需 API Key 即可在本地提取；设置 `WEBCLAW_API_KEY` 后即可处理受保护以及需要 JavaScript 渲染的页面。 + +可在 [skills.sh](https://www.skills.sh/0xMassi/webclaw-skill/webclaw) 上找到它。 + +--- + +## 工具 + +| 工具 | 作用 | 本地 | +| --- | --- | :-: | +| `scrape` | 将单个 URL 提取为 markdown、text、JSON、LLM 格式或 HTML | 是 | +| `crawl` | 跟随同源链接并提取发现的页面 | 是 | +| `map` | 发现 URL，但不逐页提取 | 是 | +| `batch` | 并行抓取多个 URL | 是 | +| `extract` | 将页面内容转换为结构化数据 | 是（本地或已配置的 LLM） | +| `summarize` | 总结一个页面 | 是（本地或已配置的 LLM） | +| `diff` | 对比页面内容快照 | 是 | +| `brand` | 提取颜色、字体、logo 与元数据 | 是 | +| `search` | 搜索网络并抓取结果 | 托管 API | +| `research` | 多来源深度研究工作流 | 托管 API | + +--- + +## SDK + +```bash +npm install @webclaw/sdk +pip install webclaw +go get github.com/0xMassi/webclaw-go +``` + +
+TypeScript + +```ts +import { Webclaw } from "@webclaw/sdk"; + +const client = new Webclaw({ apiKey: process.env.WEBCLAW_API_KEY! }); + +const page = await client.scrape({ + url: "https://example.com", + formats: ["markdown"], + only_main_content: true, +}); + +console.log(page.markdown); +``` + +
+ +
+Python + +```python +from webclaw import Webclaw + +client = Webclaw(api_key="wc_your_key") + +page = client.scrape( + "https://example.com", + formats=["markdown"], + only_main_content=True, +) + +print(page.markdown) +``` + +
+ +
+cURL + +```bash +curl -X POST https://api.webclaw.io/v1/scrape \ + -H "Authorization: Bearer $WEBCLAW_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "url": "https://example.com", + "formats": ["markdown"], + "only_main_content": true + }' +``` + +
+ +--- + +## 输出格式 + +| 格式 | 适用场景 | +| --- | --- | +| `markdown` | 保留结构的干净页面内容 | +| `llm` | 面向智能体与 RAG 流水线的紧凑 LLM 上下文 | +| `text` | 最少格式的纯文本 | +| `json` | 结构化元数据、链接、图片与已提取字段 | +| `html` | 清洗后的 HTML，便于自定义处理 | + +--- + +## 本地优先，按需上云 + +核心提取路径下，CLI 与 MCP 服务器无需账号即可在本地运行。 + +当你需要以下能力时，使用托管 API [webclaw.io](https://webclaw.io)： + +- 无需自行运维基础设施即可访问受保护的站点 +- JavaScript 渲染 +- 异步的爬取与研究任务 +- 网络搜索 +- 页面监控（watches）与生产用量统计 +- 供应用代码调用的 SDK + +```bash +export WEBCLAW_API_KEY=wc_your_key + +webclaw https://example.com --cloud +``` + +--- + +## 你可以用它构建什么 + +| 场景 | 示例 | +| --- | --- | +| AI 智能体的网页访问 | 为 Claude、Cursor 或其他 MCP 客户端提供干净的页面上下文 | +| RAG 数据入库 | 爬取文档、帮助中心、博客与知识库 | +| 竞品监控 | 追踪定价页、更新日志、文档与产品页 | +| 结构化提取 | 把杂乱的页面变成带类型的 JSON，用于自动化 | +| 研究工作流 | 搜索、抓取、总结并引用多个来源 | +| 品牌情报 | 提取 logo、颜色、字体与社交元数据 | + +## 架构 + +```text +webclaw/ + crates/ + webclaw-core HTML 转 markdown、text、JSON 与 LLM 就绪输出 + webclaw-fetch 抓取、爬取、批处理与站点映射 + webclaw-llm 本地与托管 LLM 提供方支持 + webclaw-pdf PDF 文本提取 + webclaw-mcp 面向 AI 智能体的 MCP 服务器 + webclaw-cli 命令行界面 +``` + +`webclaw-core` 是纯粹的提取逻辑：没有网络 I/O，接口面小，可脱离抓取层独立使用。 + +--- + +## 配置 + +| 变量 | 说明 | +| --- | --- | +| `WEBCLAW_API_KEY` | 托管 API 密钥 | +| `OLLAMA_HOST` | 本地 LLM 功能所用的 Ollama 地址 | +| `OPENAI_API_KEY` | 兼容 OpenAI 的 LLM 提供方密钥 | +| `OPENAI_BASE_URL` | 兼容 OpenAI 的接口地址 | +| `ANTHROPIC_API_KEY` | 兼容 Anthropic 的 LLM 提供方密钥 | +| `ANTHROPIC_BASE_URL` | 兼容 Anthropic 的接口地址 | +| `WEBCLAW_PROXY` | 单个代理 URL | +| `WEBCLAW_PROXY_FILE` | 代理池文件 | + +--- + +## 参与贡献 + +眼下最有价值的贡献都很务实、也很小： + +- 为真实的智能体与 RAG 工作流补充示例 +- 改进 SDK 代码片段 +- 上报提取效果不佳的页面 +- 为杂乱的 HTML 添加会失败的测试样例（fixtures） +- 完善面向 MCP 客户端与本地配置的文档 +- 在更多 Linux / macOS 环境上测试 CLI + +不错的入手点： + +- [Good first issues](https://github.com/0xMassi/webclaw/issues?q=label%3A%22good+first+issue%22) +- [提交一个 bug 报告](https://github.com/0xMassi/webclaw/issues/new) +- [发起一场讨论](https://github.com/0xMassi/webclaw/discussions) + +如果某个页面提取效果不好，请附上： + +```text +URL: +命令或 API 请求: +期望输出: +实际输出: +使用的格式: markdown / llm / text / json / html +使用方式: CLI / MCP / SDK / API: +``` + +发帖前，请从日志中移除密钥、Cookie、私有令牌和客户数据。 + +--- + +## 基础设施合作伙伴 + +
- + NodeMaven
- - Proxy-Seller - - - Proxy-Seller maintains a global network of residential and datacenter proxies optimized for web extraction at scale. - The service supports high-volume concurrent scraping, geographic rotation, and integration with web extraction tools. - Use code WBC15 for 15% off IPv4, IPv6, ISP, and Residential proxies, and 10% off Mobile at - proxy-seller.com. -
- - RapidProxy - - - RapidProxy delivers fast, reliable proxy infrastructure for large-scale data collection. - With 90M+ residential IPs, smart rotation, high concurrency, AI-powered CAPTCHA bypass, and non-expiring traffic, it helps keep scraping workflows stable at scale. - Use code webclaw for 10% off, or - Try it free. + NodeMaven.
+ + + + + + +
+ + ColdProxy + +
+ ColdProxy 作为基础设施合作伙伴支持 webclaw，提供覆盖 195+ 个国家/地区的住宅 IPv4、 + 住宅 IPv6 与数据中心 IPv6 代理基础设施，适用于公开数据采集、区域测试、监控与网页抓取工作流。 + 在官网了解 ColdProxy 的最新套餐与优惠。 + 使用优惠码 webclaw8Off，首单可享 8% 折扣。 + 详见代理支持爬取指南，了解如何把 ColdProxy 接入 webclaw。 +
+ +--- + +## 工作室合作伙伴 + + + + + + + + + + +
+ + NodeMaven + + + NodeMaven 提供市场上最可靠、IP 质量最高的代理服务。适用于自动化、网页抓取、SEO 研究 + 与社媒管理：99.9% 在线率、最长 7 天的粘性会话、IP 过滤（所有代理欺诈评分低于 97%）、无需 KYC， + 并提供最高 10% 的流量返现。在 NodeMaven 使用 + WEBCLAW35 享移动与住宅代理 35% 折扣，或用 WEBCLAW40 享 ISP（静态）代理 40% 折扣。 +
+ + MangoProxy + + + MangoProxy 提供覆盖 200+ 地区的住宅、ISP、数据中心与移动代理，背靠 9000 万+ IP 池， + 支持 HTTP 与 SOCKS5，为大规模网页抓取与数据采集提供高稳定性。在 + mangoproxy.com + 使用优惠码 0XMASSI 享 ISP（静态）代理 8% 折扣。 +
+ +--- + +## 社区插件 + +将 webclaw 集成到 AI 智能体平台的第三方插件： + +| 插件 | 平台 | 作用 | +|---|---|---| +| [openclaw-webclaw](https://github.com/jal-co/openclaw-webclaw) | [OpenClaw](https://openclaw.ai) | 原生 webclaw v1 API 插件，含 9 个工具：scrape、search、crawl、extract、summarize、diff、map、batch、brand | +| [hermes-webclaw](https://github.com/jal-co/hermes-webclaw) | [Hermes Agent](https://github.com/NousResearch/hermes-agent) | 面向完整 v1 API 的网络搜索提供方与 9 个专用工具。通过 `hermes plugins install jal-co/hermes-webclaw` 安装 | + +做了 webclaw 集成？欢迎 [提交 PR](https://github.com/0xMassi/webclaw/pulls) 把它加到这里。 + +--- + +## 贡献者 + +感谢每一位通过 issue、示例、文档、bug 报告与 PR 改进 webclaw 的人。 + + + webclaw contributors + + +--- + +## Star 历史 + + + + + + Star History Chart + + + +--- + +## 许可证 + +[AGPL-3.0](LICENSE) diff --git a/SKILL.md b/SKILL.md deleted file mode 100644 index 39fc144..0000000 --- a/SKILL.md +++ /dev/null @@ -1,634 +0,0 @@ ---- -name: webclaw -description: Web extraction engine with antibot bypass. Scrape, crawl, extract, summarize, search, map, diff, monitor, research, and analyze any URL — including Cloudflare-protected sites. Use when you need reliable web content, the built-in web_fetch fails, or you need structured data extraction from web pages. -homepage: https://webclaw.io -user-invocable: true -metadata: {"openclaw":{"emoji":"🦀","requires":{"env":["WEBCLAW_API_KEY"]},"primaryEnv":"WEBCLAW_API_KEY","homepage":"https://webclaw.io","install":[{"id":"npx","kind":"node","bins":["webclaw-mcp"],"label":"npx create-webclaw"}]}} ---- - -# webclaw - -High-quality web extraction with automatic antibot bypass. Beats Firecrawl on extraction quality and handles Cloudflare, DataDome, and JS-rendered pages automatically. - -## When to use this skill - -- **Always** when you need to fetch web content and want reliable results -- When `web_fetch` returns empty/blocked content (403, Cloudflare challenges) -- When you need structured data extraction (pricing tables, product info) -- When you need to crawl an entire site or discover all URLs -- When you need LLM-optimized content (cleaner than raw markdown) -- When you need to summarize a page without reading the full content -- When you need to detect content changes between visits -- When you need brand identity analysis (colors, fonts, logos) -- When you need web search results with optional page scraping -- When you need deep multi-source research on a topic -- When you need AI-guided scraping to accomplish a goal on a page -- When you need to monitor a URL for changes over time - -## API base - -All requests go to `https://api.webclaw.io/v1/`. - -Authentication: `Authorization: Bearer $WEBCLAW_API_KEY` - -## Endpoints - -### 1. Scrape — extract content from a single URL - -```bash -curl -X POST https://api.webclaw.io/v1/scrape \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" \ - -H "Content-Type: application/json" \ - -d '{ - "url": "https://example.com", - "formats": ["markdown"], - "only_main_content": true - }' -``` - -**Request fields:** - -| Field | Type | Default | Description | -|-------|------|---------|-------------| -| `url` | string | required | URL to scrape | -| `formats` | string[] | `["markdown"]` | Output formats: `markdown`, `text`, `llm`, `json` | -| `include_selectors` | string[] | `[]` | CSS selectors to keep (e.g. `["article", ".content"]`) | -| `exclude_selectors` | string[] | `[]` | CSS selectors to remove (e.g. `["nav", "footer", ".ads"]`) | -| `only_main_content` | bool | `false` | Extract only the main article/content area | -| `no_cache` | bool | `false` | Skip cache, fetch fresh | -| `max_cache_age` | int | server default | Max acceptable cache age in seconds | - -**Response:** - -```json -{ - "url": "https://example.com", - "metadata": { - "title": "Example", - "description": "...", - "language": "en", - "word_count": 1234 - }, - "markdown": "# Page Title\n\nContent here...", - "cache": { "status": "miss" } -} -``` - -**Format options:** -- `markdown` — clean markdown, best for general use -- `text` — plain text without formatting -- `llm` — optimized for LLM consumption: includes page title, URL, and cleaned content with link references. Best for feeding to AI models. -- `json` — full extraction result with all metadata - -**When antibot bypass activates** (automatic, no extra config): -```json -{ - "antibot": { - "bypass": true, - "elapsed_ms": 3200 - } -} -``` - -### 2. Crawl — scrape an entire website - -Starts an async job. Poll for results. - -**Start crawl:** -```bash -curl -X POST https://api.webclaw.io/v1/crawl \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" \ - -H "Content-Type: application/json" \ - -d '{ - "url": "https://docs.example.com", - "max_depth": 3, - "max_pages": 50, - "use_sitemap": true - }' -``` - -Response: `{ "job_id": "abc-123", "status": "running" }` - -**Poll status:** -```bash -curl https://api.webclaw.io/v1/crawl/abc-123 \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" -``` - -Response when complete: -```json -{ - "job_id": "abc-123", - "status": "completed", - "total": 47, - "completed": 45, - "errors": 2, - "pages": [ - { - "url": "https://docs.example.com/intro", - "markdown": "# Introduction\n...", - "metadata": { "title": "Intro", "word_count": 500 } - } - ] -} -``` - -**Request fields:** - -| Field | Type | Default | Description | -|-------|------|---------|-------------| -| `url` | string | required | Starting URL | -| `max_depth` | int | `3` | How many links deep to follow | -| `max_pages` | int | `100` | Maximum pages to crawl | -| `use_sitemap` | bool | `false` | Seed URLs from sitemap.xml | -| `formats` | string[] | `["markdown"]` | Output formats per page | -| `include_selectors` | string[] | `[]` | CSS selectors to keep | -| `exclude_selectors` | string[] | `[]` | CSS selectors to remove | -| `only_main_content` | bool | `false` | Main content only | - -### 3. Map — discover all URLs on a site - -Fast URL discovery without full content extraction. - -```bash -curl -X POST https://api.webclaw.io/v1/map \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" \ - -H "Content-Type: application/json" \ - -d '{"url": "https://example.com"}' -``` - -Response: -```json -{ - "url": "https://example.com", - "count": 142, - "urls": [ - "https://example.com/about", - "https://example.com/pricing", - "https://example.com/docs/intro" - ] -} -``` - -### 4. Batch — scrape multiple URLs in parallel - -```bash -curl -X POST https://api.webclaw.io/v1/batch \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" \ - -H "Content-Type: application/json" \ - -d '{ - "urls": [ - "https://a.com", - "https://b.com", - "https://c.com" - ], - "formats": ["markdown"], - "concurrency": 5 - }' -``` - -Response: -```json -{ - "total": 3, - "completed": 3, - "errors": 0, - "results": [ - { "url": "https://a.com", "markdown": "...", "metadata": {} }, - { "url": "https://b.com", "markdown": "...", "metadata": {} }, - { "url": "https://c.com", "error": "timeout" } - ] -} -``` - -### 5. Extract — LLM-powered structured extraction - -Pull structured data from any page using a JSON schema or plain-text prompt. - -**With JSON schema:** -```bash -curl -X POST https://api.webclaw.io/v1/extract \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" \ - -H "Content-Type: application/json" \ - -d '{ - "url": "https://example.com/pricing", - "schema": { - "type": "object", - "properties": { - "plans": { - "type": "array", - "items": { - "type": "object", - "properties": { - "name": { "type": "string" }, - "price": { "type": "string" }, - "features": { "type": "array", "items": { "type": "string" } } - } - } - } - } - } - }' -``` - -**With prompt:** -```bash -curl -X POST https://api.webclaw.io/v1/extract \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" \ - -H "Content-Type: application/json" \ - -d '{ - "url": "https://example.com/pricing", - "prompt": "Extract all pricing tiers with names, monthly prices, and key features" - }' -``` - -Response: -```json -{ - "url": "https://example.com/pricing", - "data": { - "plans": [ - { "name": "Starter", "price": "$49/mo", "features": ["10k pages", "Email support"] }, - { "name": "Pro", "price": "$99/mo", "features": ["100k pages", "Priority support", "API access"] } - ] - } -} -``` - -### 6. Summarize — get a quick summary of any page - -```bash -curl -X POST https://api.webclaw.io/v1/summarize \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" \ - -H "Content-Type: application/json" \ - -d '{ - "url": "https://example.com/long-article", - "max_sentences": 3 - }' -``` - -Response: -```json -{ - "url": "https://example.com/long-article", - "summary": "The article discusses... Key findings include... The author concludes that..." -} -``` - -### 7. Diff — detect content changes - -Compare current page content against a previous snapshot. - -```bash -curl -X POST https://api.webclaw.io/v1/diff \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" \ - -H "Content-Type: application/json" \ - -d '{ - "url": "https://example.com", - "previous": { - "markdown": "# Old content...", - "metadata": { "title": "Old Title" } - } - }' -``` - -Response: -```json -{ - "url": "https://example.com", - "status": "changed", - "diff": "--- previous\n+++ current\n@@ -1 +1 @@\n-# Old content\n+# New content", - "metadata_changes": [ - { "field": "title", "old": "Old Title", "new": "New Title" } - ] -} -``` - -### 8. Brand — extract brand identity - -Analyze a website's visual identity: colors, fonts, logo. - -```bash -curl -X POST https://api.webclaw.io/v1/brand \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" \ - -H "Content-Type: application/json" \ - -d '{"url": "https://example.com"}' -``` - -Response: -```json -{ - "url": "https://example.com", - "brand": { - "colors": [ - { "hex": "#FF6B35", "usage": "primary" }, - { "hex": "#1A1A2E", "usage": "background" } - ], - "fonts": ["Inter", "JetBrains Mono"], - "logo_url": "https://example.com/logo.svg", - "favicon_url": "https://example.com/favicon.ico" - } -} -``` - -### 9. Search — web search with optional scraping - -Search the web and optionally scrape each result page. - -```bash -curl -X POST https://api.webclaw.io/v1/search \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" \ - -H "Content-Type: application/json" \ - -d '{ - "query": "best rust web frameworks 2026", - "num_results": 5, - "scrape": true, - "formats": ["markdown"] - }' -``` - -**Request fields:** - -| Field | Type | Default | Description | -|-------|------|---------|-------------| -| `query` | string | required | Search query | -| `num_results` | int | `10` | Number of search results to return | -| `scrape` | bool | `false` | Also scrape each result page for full content | -| `formats` | string[] | `["markdown"]` | Output formats when `scrape` is true | -| `country` | string | none | Country code for localized results (e.g. `"us"`, `"de"`) | -| `lang` | string | none | Language code for results (e.g. `"en"`, `"fr"`) | - -**Response:** - -```json -{ - "query": "best rust web frameworks 2026", - "results": [ - { - "title": "Top Rust Web Frameworks in 2026", - "url": "https://blog.example.com/rust-frameworks", - "snippet": "A comprehensive comparison of Axum, Actix, and Rocket...", - "position": 1, - "markdown": "# Top Rust Web Frameworks\n\n..." - }, - { - "title": "Choosing a Rust Backend Framework", - "url": "https://dev.to/rust-backends", - "snippet": "When starting a new Rust web project...", - "position": 2, - "markdown": "# Choosing a Rust Backend\n\n..." - } - ] -} -``` - -The `markdown` field on each result is only present when `scrape: true`. Without it, you get titles, URLs, snippets, and positions only. - -### 10. Research — deep multi-source research - -Starts an async research job that searches, scrapes, and synthesizes information across multiple sources. Poll for results. - -**Start research:** -```bash -curl -X POST https://api.webclaw.io/v1/research \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" \ - -H "Content-Type: application/json" \ - -d '{ - "query": "How does Cloudflare Turnstile work and what are its known bypass methods?", - "max_iterations": 5, - "max_sources": 10, - "topic": "security", - "deep": true - }' -``` - -**Request fields:** - -| Field | Type | Default | Description | -|-------|------|---------|-------------| -| `query` | string | required | Research question or topic | -| `max_iterations` | int | server default | Maximum research iterations (search-read-analyze cycles) | -| `max_sources` | int | server default | Maximum number of sources to consult | -| `topic` | string | none | Topic hint to guide search strategy (e.g. `"security"`, `"finance"`, `"engineering"`) | -| `deep` | bool | `false` | Enable deep research mode for more thorough analysis (costs 10 credits instead of 1) | - -Response: `{ "id": "res-abc-123", "status": "running" }` - -**Poll results:** -```bash -curl https://api.webclaw.io/v1/research/res-abc-123 \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" -``` - -Response when complete: -```json -{ - "id": "res-abc-123", - "status": "completed", - "query": "How does Cloudflare Turnstile work and what are its known bypass methods?", - "report": "# Cloudflare Turnstile Analysis\n\n## Overview\nCloudflare Turnstile is a CAPTCHA replacement that...\n\n## How It Works\n...\n\n## Known Bypass Methods\n...", - "sources": [ - { "url": "https://developers.cloudflare.com/turnstile/", "title": "Turnstile Documentation" }, - { "url": "https://blog.cloudflare.com/turnstile-ga/", "title": "Turnstile GA Announcement" } - ], - "findings": [ - "Turnstile uses browser environment signals and proof-of-work challenges", - "Managed mode auto-selects challenge difficulty based on visitor risk score", - "Known bypass approaches include instrumented browser automation" - ], - "iterations": 5, - "elapsed_ms": 34200 -} -``` - -**Status values:** `running`, `completed`, `failed` - -### 11. Agent Scrape — AI-guided scraping - -Use an AI agent to navigate and interact with a page to accomplish a specific goal. The agent can click, scroll, fill forms, and extract data across multiple steps. - -```bash -curl -X POST https://api.webclaw.io/v1/agent-scrape \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" \ - -H "Content-Type: application/json" \ - -d '{ - "url": "https://example.com/products", - "goal": "Find the cheapest laptop with at least 16GB RAM and extract its full specs", - "max_steps": 10 - }' -``` - -**Request fields:** - -| Field | Type | Default | Description | -|-------|------|---------|-------------| -| `url` | string | required | Starting URL | -| `goal` | string | required | What the agent should accomplish | -| `max_steps` | int | server default | Maximum number of actions the agent can take | - -**Response:** - -```json -{ - "url": "https://example.com/products", - "result": "The cheapest laptop with 16GB+ RAM is the ThinkPad E14 Gen 6 at $649. Specs: AMD Ryzen 5 7535U, 16GB DDR4, 512GB SSD, 14\" FHD IPS display, 57Wh battery.", - "steps": [ - { "action": "navigate", "detail": "Loaded products page" }, - { "action": "click", "detail": "Clicked 'Laptops' category filter" }, - { "action": "click", "detail": "Applied '16GB+' RAM filter" }, - { "action": "click", "detail": "Sorted by price: low to high" }, - { "action": "extract", "detail": "Extracted specs from first matching product" } - ] -} -``` - -### 12. Watch — monitor a URL for changes - -Create persistent monitors that check a URL on a schedule and notify via webhook when content changes. - -**Create a monitor:** -```bash -curl -X POST https://api.webclaw.io/v1/watch \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" \ - -H "Content-Type: application/json" \ - -d '{ - "url": "https://example.com/pricing", - "interval": "0 */6 * * *", - "webhook_url": "https://hooks.example.com/pricing-changed", - "formats": ["markdown"] - }' -``` - -**Request fields:** - -| Field | Type | Default | Description | -|-------|------|---------|-------------| -| `url` | string | required | URL to monitor | -| `interval` | string | required | Check frequency as cron expression or seconds (e.g. `"0 */6 * * *"` or `"3600"`) | -| `webhook_url` | string | none | URL to POST when changes are detected | -| `formats` | string[] | `["markdown"]` | Output formats for snapshots | - -Response: -```json -{ - "id": "watch-abc-123", - "url": "https://example.com/pricing", - "interval": "0 */6 * * *", - "webhook_url": "https://hooks.example.com/pricing-changed", - "formats": ["markdown"], - "created_at": "2026-03-20T10:00:00Z", - "last_check": null, - "status": "active" -} -``` - -**List all monitors:** -```bash -curl https://api.webclaw.io/v1/watch \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" -``` - -Response: -```json -{ - "monitors": [ - { - "id": "watch-abc-123", - "url": "https://example.com/pricing", - "interval": "0 */6 * * *", - "status": "active", - "last_check": "2026-03-20T16:00:00Z", - "checks": 4 - } - ] -} -``` - -**Get a monitor with snapshots:** -```bash -curl https://api.webclaw.io/v1/watch/watch-abc-123 \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" -``` - -Response: -```json -{ - "id": "watch-abc-123", - "url": "https://example.com/pricing", - "interval": "0 */6 * * *", - "status": "active", - "snapshots": [ - { - "checked_at": "2026-03-20T16:00:00Z", - "status": "changed", - "diff": "--- previous\n+++ current\n@@ -5 +5 @@\n-Pro: $99/mo\n+Pro: $119/mo" - }, - { - "checked_at": "2026-03-20T10:00:00Z", - "status": "baseline" - } - ] -} -``` - -**Trigger an immediate check:** -```bash -curl -X POST https://api.webclaw.io/v1/watch/watch-abc-123/check \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" -``` - -**Delete a monitor:** -```bash -curl -X DELETE https://api.webclaw.io/v1/watch/watch-abc-123 \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" -``` - -## Choosing the right format - -| Goal | Format | Why | -|------|--------|-----| -| Read and understand a page | `markdown` | Clean structure, headings, links preserved | -| Feed content to an AI model | `llm` | Optimized: includes title + URL header, clean link refs | -| Search or index content | `text` | Plain text, no formatting noise | -| Programmatic analysis | `json` | Full metadata, structured data, DOM statistics | - -## Tips - -- **Use `llm` format** when passing content to yourself or another AI — it's specifically optimized for LLM consumption with better context framing. -- **Use `only_main_content: true`** to skip navigation, sidebars, and footers. Reduces noise significantly. -- **Use `include_selectors`/`exclude_selectors`** for fine-grained control when `only_main_content` isn't enough. -- **Batch over individual scrapes** when fetching multiple URLs — it's faster and more efficient. -- **Use `map` before `crawl`** to discover the site structure first, then crawl specific sections. -- **Use `extract` with a JSON schema** for reliable structured output (e.g., pricing tables, product specs, contact info). -- **Antibot bypass is automatic** — no extra configuration needed. Works on Cloudflare, DataDome, AWS WAF, and JS-rendered SPAs. -- **Use `search` with `scrape: true`** to get full page content for each search result in one call instead of searching then scraping separately. -- **Use `research` for complex questions** that need multiple sources — it handles the search-read-synthesize loop automatically. Enable `deep: true` for thorough analysis. -- **Use `agent-scrape` for interactive pages** where data is behind filters, pagination, or form submissions that a simple scrape cannot reach. -- **Use `watch` for ongoing monitoring** — set up a cron schedule and a webhook to get notified when a page changes without polling manually. - -## Smart Fetch Architecture - -The webclaw MCP server uses a **local-first** approach: - -1. **Local fetch** — fast, free, no API credits used (~80% of sites) -2. **Cloud API fallback** — automatic when bot protection or JS rendering is detected - -This means: -- Most scrapes cost zero credits (local extraction) -- Cloudflare, DataDome, AWS WAF sites automatically fall back to the cloud API -- JS-rendered SPAs (React, Next.js, Vue) also fall back automatically -- Set `WEBCLAW_API_KEY` to enable cloud fallback - -## vs web_fetch - -| | webclaw | web_fetch | -|---|---------|-----------| -| Cloudflare bypass | Automatic (cloud fallback) | Fails (403) | -| JS-rendered pages | Automatic fallback | Readability only | -| Output quality | 20-step optimization pipeline | Basic HTML parsing | -| Structured extraction | LLM-powered, schema-based | None | -| Crawling | Full site crawl with sitemap | Single page only | -| Caching | Built-in, configurable TTL | Per-session | -| Rate limiting | Managed server-side | Client responsibility | - -Use `web_fetch` for simple, fast lookups. Use webclaw when you need reliability, quality, or advanced features. diff --git a/assets/sponsors/nodemaven-banner.png b/assets/sponsors/nodemaven-banner.png index de47edb..9a30fd9 100644 Binary files a/assets/sponsors/nodemaven-banner.png and b/assets/sponsors/nodemaven-banner.png differ diff --git a/assets/sponsors/proxy-seller-banner.png b/assets/sponsors/proxy-seller-banner.png deleted file mode 100644 index 6f9f15d..0000000 Binary files a/assets/sponsors/proxy-seller-banner.png and /dev/null differ diff --git a/assets/sponsors/rapidproxy-banner.png b/assets/sponsors/rapidproxy-banner.png deleted file mode 100644 index 1a3369b..0000000 Binary files a/assets/sponsors/rapidproxy-banner.png and /dev/null differ diff --git a/crates/webclaw-cli/src/main.rs b/crates/webclaw-cli/src/main.rs index 98e61ef..2624336 100644 --- a/crates/webclaw-cli/src/main.rs +++ b/crates/webclaw-cli/src/main.rs @@ -183,11 +183,11 @@ struct Cli { browser: Browser, /// Proxy URL (http://user:pass@host:port or socks5://host:port) - #[arg(short, long, env = "WEBCLAW_PROXY")] + #[arg(short, long, env = "WEBCLAW_PROXY", hide_env_values = true)] proxy: Option, /// File with proxies (host:port:user:pass, one per line). Rotates per request. - #[arg(long, env = "WEBCLAW_PROXY_FILE")] + #[arg(long, env = "WEBCLAW_PROXY_FILE", hide_env_values = true)] proxy_file: Option, /// Request timeout in seconds @@ -256,7 +256,7 @@ struct Cli { /// Webhook URL: POST a JSON payload when an operation completes. /// Works with crawl, batch, watch (on change), and single URL modes. - #[arg(long, env = "WEBCLAW_WEBHOOK_URL")] + #[arg(long, env = "WEBCLAW_WEBHOOK_URL", hide_env_values = true)] webhook: Option, /// Extract brand identity (colors, fonts, logo) @@ -338,7 +338,7 @@ struct Cli { #[arg(long, num_args = 0..=1, default_missing_value = "3")] summarize: Option, - /// Force a specific LLM provider (ollama, openai, anthropic) + /// Force a specific LLM provider (ollama, openai, atlascloud, anthropic) #[arg(long, env = "WEBCLAW_LLM_PROVIDER")] llm_provider: Option, @@ -347,12 +347,12 @@ struct Cli { llm_model: Option, /// Override the LLM base URL (Ollama, OpenAI-compatible, or Anthropic-compatible) - #[arg(long, env = "WEBCLAW_LLM_BASE_URL")] + #[arg(long, env = "WEBCLAW_LLM_BASE_URL", hide_env_values = true)] llm_base_url: Option, // -- Cloud API options -- /// Webclaw Cloud API key for automatic fallback on bot-protected or JS-rendered sites - #[arg(long, env = "WEBCLAW_API_KEY")] + #[arg(long, env = "WEBCLAW_API_KEY", hide_env_values = true)] api_key: Option, /// Force all requests through the cloud API (skip local extraction) @@ -436,7 +436,7 @@ enum Commands { query: String, /// Serper.dev API key. Falls back to the `SERPER_API_KEY` env var. - #[arg(long, env = "SERPER_API_KEY")] + #[arg(long, env = "SERPER_API_KEY", hide_env_values = true)] serper_key: Option, /// Number of results to return (1-10). @@ -864,6 +864,18 @@ fn collect_urls(cli: &Cli) -> Result)>, String> { Ok(entries) } +/// When `--urls-file` supplies exactly one URL and no positional URL was given, +/// treat it like a single positional URL so a one-line file behaves identically +/// to `webclaw ` (stdout, `--output-dir`, cloud fallback, `--raw-html`, +/// `--diff`, LLM paths). Without this the lone file URL gives `entries.len() == 1`, +/// which the batch gates skip, and the single path then finds an empty `cli.urls` +/// and errors "no input provided". See issue #86. +fn backfill_single_file_url(urls: &mut Vec, entries: &[(String, Option)]) { + if urls.is_empty() && entries.len() == 1 { + urls.push(entries[0].0.clone()); + } +} + /// Result that can be either a local extraction or a cloud API JSON response. enum FetchOutput { Local(Box), @@ -2239,6 +2251,15 @@ async fn build_llm_provider(cli: &Cli) -> Result, String> { .ok_or("OPENAI_API_KEY not set")?; Ok(Box::new(provider)) } + "atlascloud" => { + let provider = webclaw_llm::providers::atlascloud::AtlasCloudProvider::new( + None, + cli.llm_base_url.clone(), + cli.llm_model.clone(), + ) + .ok_or("ATLASCLOUD_API_KEY not set")?; + Ok(Box::new(provider)) + } "anthropic" => { let provider = webclaw_llm::providers::anthropic::AnthropicProvider::with_base_url( None, @@ -2249,7 +2270,7 @@ async fn build_llm_provider(cli: &Cli) -> Result, String> { Ok(Box::new(provider)) } other => Err(format!( - "unknown LLM provider: {other} (use ollama, openai, or anthropic)" + "unknown LLM provider: {other} (use ollama, openai, atlascloud, or anthropic)" )), } } else { @@ -2624,7 +2645,7 @@ async fn run_research(cli: &Cli, query: &str) -> Result<(), String> { async fn main() { dotenvy::dotenv().ok(); - let cli = Cli::parse(); + let mut cli = Cli::parse(); init_logging(cli.verbose); // Subcommand path. Handled before the flag dispatch so a subcommand @@ -2767,6 +2788,20 @@ async fn main() { return; } + // Collect URLs from args + --urls-file up front, and backfill a lone + // --urls-file URL into cli.urls so a one-line file behaves like a positional + // URL across ALL modes below (crawl / watch / diff / brand / batch / single). + // See issue #86 — the mode gates and single path read cli.urls, which is + // empty when the URL came only from --urls-file. + let entries = match collect_urls(&cli) { + Ok(u) => u, + Err(e) => { + eprintln!("error: {e}"); + process::exit(1); + } + }; + backfill_single_file_url(&mut cli.urls, &entries); + // --crawl: recursive crawl mode if cli.crawl { if let Err(e) = run_crawl(&cli).await { @@ -2778,13 +2813,7 @@ async fn main() { // --watch: poll URL(s) for changes if cli.watch { - let watch_urls: Vec = match collect_urls(&cli) { - Ok(entries) => entries.into_iter().map(|(url, _)| url).collect(), - Err(e) => { - eprintln!("error: {e}"); - process::exit(1); - } - }; + let watch_urls: Vec = entries.iter().map(|(url, _)| url.clone()).collect(); if let Err(e) = run_watch(&cli, &watch_urls).await { eprintln!("error: {e}"); process::exit(1); @@ -2819,15 +2848,6 @@ async fn main() { return; } - // Collect all URLs from args + --urls-file - let entries = match collect_urls(&cli) { - Ok(u) => u, - Err(e) => { - eprintln!("error: {e}"); - process::exit(1); - } - }; - // LLM modes: --extract-json, --extract-prompt, --summarize // When multiple URLs are provided, run batch LLM extraction over all of them. if has_llm_flags(&cli) { @@ -2899,6 +2919,36 @@ mod tests { use super::*; use webclaw_core::Content; + // issue #86: a single URL sourced only from --urls-file must be promoted to + // a positional URL so it takes the single-scrape path (batch gates need >1). + #[test] + fn single_file_url_backfilled_into_positional() { + let mut urls: Vec = Vec::new(); + let entries = vec![("https://example.com".to_string(), None)]; + backfill_single_file_url(&mut urls, &entries); + assert_eq!(urls, vec!["https://example.com".to_string()]); + } + + #[test] + fn positional_urls_left_untouched() { + let mut urls = vec!["https://a.com".to_string()]; + let entries = vec![("https://a.com".to_string(), None)]; + backfill_single_file_url(&mut urls, &entries); + assert_eq!(urls, vec!["https://a.com".to_string()]); + } + + #[test] + fn multiple_file_urls_not_backfilled() { + // 2+ URLs already route to run_batch, so cli.urls stays empty here. + let mut urls: Vec = Vec::new(); + let entries = vec![ + ("https://a.com".to_string(), None), + ("https://b.com".to_string(), None), + ]; + backfill_single_file_url(&mut urls, &entries); + assert!(urls.is_empty()); + } + fn empty_result(title: Option<&str>, url: Option<&str>, markdown: &str) -> ExtractionResult { ExtractionResult { metadata: Metadata { diff --git a/crates/webclaw-llm/src/chain.rs b/crates/webclaw-llm/src/chain.rs index e2c6b8b..f564172 100644 --- a/crates/webclaw-llm/src/chain.rs +++ b/crates/webclaw-llm/src/chain.rs @@ -1,5 +1,5 @@ /// Provider chain — tries providers in order until one succeeds. -/// Default order: Ollama (local, free) -> OpenAI -> Gemini -> Anthropic. +/// Default order: Ollama (local, free) -> OpenAI -> Gemini -> Anthropic -> Atlas Cloud (opt-in). /// Only includes providers that are actually configured/available. use async_trait::async_trait; use tracing::{debug, warn}; @@ -7,8 +7,8 @@ use tracing::{debug, warn}; use crate::error::LlmError; use crate::provider::{CompletionRequest, LlmProvider}; use crate::providers::{ - anthropic::AnthropicProvider, gemini::GeminiProvider, ollama::OllamaProvider, - openai::OpenAiProvider, + anthropic::AnthropicProvider, atlascloud::AtlasCloudProvider, gemini::GeminiProvider, + ollama::OllamaProvider, openai::OpenAiProvider, }; pub struct ProviderChain { @@ -16,11 +16,13 @@ pub struct ProviderChain { } impl ProviderChain { - /// Build the default chain: Ollama -> OpenAI -> Gemini -> Anthropic. + /// Build the default chain: Ollama -> OpenAI -> Gemini -> Anthropic -> Atlas Cloud. /// Ollama is always added (availability checked at call time). /// Cloud providers are only added if their API keys are configured. /// Gemini sits ahead of Anthropic so Google Cloud credits are preferred, - /// with Anthropic as the last-resort fallback. + /// with Anthropic as the last-resort fallback. Atlas Cloud is opt-in and + /// added last (only when `ATLASCLOUD_API_KEY` is set), so it never preempts + /// an already-configured provider. pub async fn default() -> Self { let mut providers: Vec> = Vec::new(); @@ -47,6 +49,11 @@ impl ProviderChain { providers.push(Box::new(anthropic)); } + if let Some(atlas) = AtlasCloudProvider::new(None, None, None) { + debug!("atlascloud configured, adding to chain"); + providers.push(Box::new(atlas)); + } + Self { providers } } diff --git a/crates/webclaw-llm/src/providers/atlascloud.rs b/crates/webclaw-llm/src/providers/atlascloud.rs new file mode 100644 index 0000000..8855d87 --- /dev/null +++ b/crates/webclaw-llm/src/providers/atlascloud.rs @@ -0,0 +1,78 @@ +/// Atlas Cloud provider — OpenAI-compatible chat completions with Atlas defaults. +use async_trait::async_trait; + +use crate::error::LlmError; +use crate::provider::{CompletionRequest, LlmProvider}; + +use super::openai::OpenAiProvider; + +pub struct AtlasCloudProvider { + inner: OpenAiProvider, +} + +impl AtlasCloudProvider { + /// Returns `None` if no Atlas Cloud API key is available (param or env). + pub fn new( + key_override: Option, + base_url: Option, + model: Option, + ) -> Option { + let key = super::load_api_key(key_override, "ATLASCLOUD_API_KEY")?; + let base_url = base_url + .or_else(|| std::env::var("ATLASCLOUD_BASE_URL").ok()) + .unwrap_or_else(|| "https://api.atlascloud.ai/v1".into()); + let model = model + .or_else(|| std::env::var("ATLASCLOUD_MODEL").ok()) + .unwrap_or_else(|| "qwen/qwen3.5-flash".into()); + let inner = OpenAiProvider::new(Some(key), Some(base_url), Some(model))?; + Some(Self { inner }) + } + + pub fn default_model(&self) -> &str { + self.inner.default_model() + } +} + +#[async_trait] +impl LlmProvider for AtlasCloudProvider { + async fn complete(&self, request: &CompletionRequest) -> Result { + self.inner.complete(request).await + } + + async fn is_available(&self) -> bool { + self.inner.is_available().await + } + + fn name(&self) -> &str { + "atlascloud" + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn empty_key_returns_none() { + assert!(AtlasCloudProvider::new(Some(String::new()), None, None).is_none()); + } + + #[test] + fn explicit_key_constructs_with_atlas_defaults() { + let provider = + AtlasCloudProvider::new(Some("test-key".into()), None, None).expect("should construct"); + assert_eq!(provider.name(), "atlascloud"); + assert_eq!(provider.default_model(), "qwen/qwen3.5-flash"); + } + + #[test] + fn explicit_model_override() { + let provider = AtlasCloudProvider::new( + Some("test-key".into()), + Some("https://proxy.example.com/v1".into()), + Some("deepseek-ai/deepseek-v4-pro".into()), + ) + .expect("should construct"); + assert_eq!(provider.default_model(), "deepseek-ai/deepseek-v4-pro"); + } +} diff --git a/crates/webclaw-llm/src/providers/mod.rs b/crates/webclaw-llm/src/providers/mod.rs index d6ae34a..7a5dd65 100644 --- a/crates/webclaw-llm/src/providers/mod.rs +++ b/crates/webclaw-llm/src/providers/mod.rs @@ -1,4 +1,5 @@ pub mod anthropic; +pub mod atlascloud; pub mod gemini; pub mod ollama; pub mod openai; diff --git a/crates/webclaw-mcp/src/server.rs b/crates/webclaw-mcp/src/server.rs index 2554512..4059198 100644 --- a/crates/webclaw-mcp/src/server.rs +++ b/crates/webclaw-mcp/src/server.rs @@ -65,6 +65,10 @@ const LOCAL_FETCH_TIMEOUT: Duration = Duration::from_secs(30); /// Maximum poll iterations for research jobs (~10 minutes at 3s intervals). const RESEARCH_MAX_POLLS: u32 = 200; +/// Maximum poll iterations for lead batch jobs (~10 minutes at 3s intervals); +/// enough for a full 25-URL batch at ~20-40s per lead, 4 in parallel. +const LEAD_BATCH_MAX_POLLS: u32 = 200; + #[tool_router] impl WebclawMcp { pub async fn new() -> Self { @@ -668,6 +672,95 @@ impl WebclawMcp { )) } + /// Enrich a company into an outreach-ready lead: the founders and leadership + /// with their LinkedIn and X (recovered from open-web search), plus company + /// summary, socials, tech stack, pricing, and public emails. Requires + /// WEBCLAW_API_KEY. Flat 100 credits per lead. + #[tool] + async fn lead(&self, Parameters(params): Parameters) -> Result { + let cloud = self + .cloud + .as_ref() + .ok_or("Lead enrichment requires WEBCLAW_API_KEY. Get a key at https://webclaw.io")?; + + let mut body = json!({ "url": params.url }); + if let Some(no_cache) = params.no_cache { + body["no_cache"] = json!(no_cache); + } + + let resp = cloud.post("lead", body).await?; + Ok(serde_json::to_string_pretty(&resp).unwrap_or_default()) + } + + /// Enrich many companies at once (up to 25 URLs) into outreach-ready leads. + /// Async: starts a batch job, polls until it finishes, and returns the + /// per-URL results. Requires WEBCLAW_API_KEY. Billed 100 credits per + /// successfully enriched lead (unreachable/empty URLs are not charged). + #[tool] + async fn lead_batch( + &self, + Parameters(params): Parameters, + ) -> Result { + let cloud = self + .cloud + .as_ref() + .ok_or("Lead enrichment requires WEBCLAW_API_KEY. Get a key at https://webclaw.io")?; + + let url_count = params.urls.len(); + if url_count == 0 { + return Err("lead_batch requires at least one URL".to_string()); + } + + let mut body = json!({ "urls": params.urls }); + if let Some(no_cache) = params.no_cache { + body["no_cache"] = json!(no_cache); + } + + // Start the async batch job. + let start_resp = cloud.post("lead/batch", body).await?; + let job_id = start_resp + .get("id") + .and_then(|v| v.as_str()) + .ok_or("Lead batch API did not return a job ID")? + .to_string(); + + info!(job_id = %job_id, urls = url_count, "lead batch started, polling for completion"); + + // Poll until completed or failed. + for poll in 0..LEAD_BATCH_MAX_POLLS { + tokio::time::sleep(Duration::from_secs(3)).await; + + let status_resp = cloud.get(&format!("lead/batch/{job_id}")).await?; + let status = status_resp + .get("status") + .and_then(|v| v.as_str()) + .unwrap_or("unknown"); + + match status { + "completed" => { + return Ok(serde_json::to_string_pretty(&status_resp).unwrap_or_default()); + } + "failed" => { + let error = status_resp + .get("error") + .and_then(|v| v.as_str()) + .unwrap_or("unknown error"); + return Err(format!("Lead batch job failed: {error}")); + } + _ => { + if poll % 20 == 19 { + info!(job_id = %job_id, poll, "lead batch still in progress..."); + } + } + } + } + + Err(format!( + "Lead batch job {job_id} timed out after ~10 minutes of polling. \ + Check status manually via the webclaw API: GET /v1/lead/batch/{job_id}" + )) + } + /// Search the web for a query and return structured results. /// /// Resolves the backend in priority order: @@ -810,7 +903,7 @@ impl ServerHandler for WebclawMcp { .with_instructions(String::from( "Webclaw MCP server -- web content extraction for AI agents. \ Tools: scrape, crawl, map, batch, extract, summarize, diff, brand, research, search, \ - list_extractors, vertical_scrape.", + list_extractors, vertical_scrape, lead, lead_batch.", )) } } diff --git a/crates/webclaw-mcp/src/tools.rs b/crates/webclaw-mcp/src/tools.rs index c20a9e8..2efd9da 100644 --- a/crates/webclaw-mcp/src/tools.rs +++ b/crates/webclaw-mcp/src/tools.rs @@ -187,6 +187,24 @@ pub struct ResearchParams { pub topic: Option, } +#[derive(Debug, Deserialize, JsonSchema)] +pub struct LeadParams { + /// Company website URL to enrich into an outreach-ready lead + pub url: String, + /// Skip the cache and force a fresh enrichment (default: false) + #[serde(default, deserialize_with = "deser_opt_bool_or_str")] + pub no_cache: Option, +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct LeadBatchParams { + /// Company website URLs to enrich (up to 25 per batch) + pub urls: Vec, + /// Skip the cache and force a fresh enrichment (default: false) + #[serde(default, deserialize_with = "deser_opt_bool_or_str")] + pub no_cache: Option, +} + #[derive(Debug, Deserialize, JsonSchema)] pub struct SearchParams { /// Search query diff --git a/crates/webclaw-server/src/main.rs b/crates/webclaw-server/src/main.rs index 869cf56..b44039e 100644 --- a/crates/webclaw-server/src/main.rs +++ b/crates/webclaw-server/src/main.rs @@ -57,7 +57,7 @@ struct Args { /// `/v1/*` request must present `Authorization: Bearer `. /// When unset, the server runs in open mode (no auth) — only /// safe on a local-bound interface or behind another auth layer. - #[arg(long, env = "WEBCLAW_API_KEY")] + #[arg(long, env = "WEBCLAW_API_KEY", hide_env_values = true)] api_key: Option, /// Tracing filter. Env: RUST_LOG. diff --git a/examples/README.md b/examples/README.md index 0967d74..e5acf09 100644 --- a/examples/README.md +++ b/examples/README.md @@ -273,14 +273,15 @@ webclaw https://example.com # Automatically detects and uses proxies.txt ## MCP Server (AI Agent Integration) ```bash -# Start the MCP server (stdio transport) -webclaw-mcp +# Run the MCP server (stdio transport) — no install needed +npx -y @webclaw/mcp # Configure in Claude Desktop (~/.config/claude/claude_desktop_config.json): # { # "mcpServers": { # "webclaw": { -# "command": "/path/to/webclaw-mcp", +# "command": "npx", +# "args": ["-y", "@webclaw/mcp"], # "env": { # "WEBCLAW_API_KEY": "wc_your_key" // optional, enables cloud fallback # } @@ -288,7 +289,7 @@ webclaw-mcp # } # } -# Available tools: scrape, crawl, map, batch, extract, summarize, diff, brand, research, search +# Available tools: scrape, search, crawl, map, batch, extract, summarize, diff, brand, research, lead, lead_batch, list_extractors, vertical_scrape ``` ## Real-World Recipes diff --git a/examples/mcp-web-scraping/README.md b/examples/mcp-web-scraping/README.md index 0663670..c5c7c43 100644 --- a/examples/mcp-web-scraping/README.md +++ b/examples/mcp-web-scraping/README.md @@ -16,7 +16,8 @@ The installer detects supported MCP clients and can write the config for you. { "mcpServers": { "webclaw": { - "command": "~/.webclaw/webclaw-mcp", + "command": "npx", + "args": ["-y", "@webclaw/mcp"], "env": { "WEBCLAW_API_KEY": "wc_your_key" } diff --git a/examples/proxy-backed-crawling/README.md b/examples/proxy-backed-crawling/README.md index 3bf3829..e628769 100644 --- a/examples/proxy-backed-crawling/README.md +++ b/examples/proxy-backed-crawling/README.md @@ -4,11 +4,11 @@ Use proxy rotation when you need to distribute a crawl across a proxy pool. webc ## Using ColdProxy -[ColdProxy](https://coldproxy.com/) is webclaw's infrastructure partner, providing residential IPv4, residential IPv6, and datacenter IPv6 proxies across 195+ countries. Use a ColdProxy endpoint as a full URL with `--proxy` / `WEBCLAW_PROXY`, or list several in a `--proxy-file` pool. +[ColdProxy](https://coldproxy.com/?utm_source=github&utm_medium=sponsorship&utm_campaign=webclaw-sponsor) is webclaw's infrastructure partner, providing residential IPv4, residential IPv6, and datacenter IPv6 proxies across 195+ countries. Use a ColdProxy endpoint as a full URL with `--proxy` / `WEBCLAW_PROXY`, or list several in a `--proxy-file` pool. ### 1. Get your endpoint -Sign in to your [ColdProxy dashboard](https://coldproxy.com/) and copy your proxy host, port, and credentials. Assemble them into a standard proxy URL: +Sign in to your [ColdProxy dashboard](https://coldproxy.com/?utm_source=github&utm_medium=sponsorship&utm_campaign=webclaw-sponsor) and copy your proxy host, port, and credentials. Assemble them into a standard proxy URL: ```text http://USERNAME:PASSWORD@HOST:PORT diff --git a/packages/create-webclaw/README.md b/packages/create-webclaw/README.md index 52d4760..ebeacbc 100644 --- a/packages/create-webclaw/README.md +++ b/packages/create-webclaw/README.md @@ -26,7 +26,7 @@ npx create-webclaw ``` -That's it. Auto-detects your AI tools, downloads the MCP server, configures everything. +That's it. Auto-detects your AI tools and writes the `npx @webclaw/mcp` config into each — nothing to install. Works with **Claude Desktop**, **Claude Code**, **Cursor**, **Windsurf**, **VS Code**, **OpenCode**, **Codex CLI**, and **Antigravity**. @@ -40,7 +40,7 @@ When it does work, you get 100KB+ of raw HTML — navigation, ads, cookie banner ## The Fix -webclaw impersonates Chrome 146 at the TLS protocol level. Perfect JA4 fingerprint. Perfect HTTP/2 Akamai hash. 99% bypass rate on 102 tested sites. +webclaw impersonates the latest Chrome at the TLS protocol level. Perfect JA4 fingerprint. Perfect HTTP/2 Akamai hash. 99% bypass rate on 102 tested sites. Then it extracts just the content — clean markdown, 67% fewer tokens. @@ -67,13 +67,29 @@ npx create-webclaw ``` 1. Detects installed AI tools (Claude, Cursor, Windsurf, VS Code, OpenCode, Codex, Antigravity) -2. Downloads the `webclaw-mcp` binary for your platform (macOS arm64/x86, Linux x86/arm64) -3. Asks for your API key (optional — **works locally without one**) -4. Writes the MCP config for each detected tool +2. Asks for your API key (optional — **works locally without one**) +3. Writes the `npx @webclaw/mcp` config into each detected tool -## 10 MCP Tools +The server itself runs via [`@webclaw/mcp`](https://www.npmjs.com/package/@webclaw/mcp) — `npx` fetches it on first launch and caches it. `create-webclaw` is just the convenience that writes that config for you. -After setup, your AI agent has access to: +### Prefer to configure by hand? + +Add this one block to your client's `mcpServers` config — it's identical to what `create-webclaw` writes: + +```json +{ + "mcpServers": { + "webclaw": { + "command": "npx", + "args": ["-y", "@webclaw/mcp"] + } + } +} +``` + +## MCP Tools + +After setup, your AI agent has access to these 14 tools: | Tool | What it does | API key needed? | |------|-------------|-----------------| @@ -87,8 +103,12 @@ After setup, your AI agent has access to: | **diff** | Track content changes | No | | **brand** | Extract brand identity | No | | **research** | Deep multi-page research | Yes | +| **list_extractors** | List the built-in vertical extractors | No | +| **vertical_scrape** | Scrape with a built-in vertical extractor | No | +| **lead** | Enrich a company URL into an outreach-ready lead (leadership + socials, tech, pricing, public emails) | Yes (webclaw) | +| **lead_batch** | Enrich up to 25 company URLs at once | Yes (webclaw) | -**8 of 10 tools work fully offline.** No API key, no cloud, no tracking. +**8 of the 14 tools work fully offline** — no API key, no cloud, no tracking. ## Supported Tools @@ -99,9 +119,9 @@ After setup, your AI agent has access to: | Cursor | `.cursor/mcp.json` | | Windsurf | `~/.codeium/windsurf/mcp_config.json` | | VS Code (Continue) | `~/.continue/config.json` | -| OpenCode | `~/.opencode/config.json` | -| Codex CLI | `~/.codex/config.json` | -| Antigravity | `~/.antigravity/mcp.json` | +| OpenCode | `~/.config/opencode/opencode.json` (or `./opencode.json`) | +| Codex CLI | `~/.codex/config.toml` | +| Antigravity | `~/.config/antigravity/mcp.json` (or `~/.antigravity/mcp.json`) | ## Sites That Work @@ -140,7 +160,6 @@ Download from [GitHub Releases](https://github.com/0xMassi/webclaw/releases) for - [Website](https://webclaw.io) - [Documentation](https://webclaw.io/docs) - [GitHub](https://github.com/0xMassi/webclaw) -- [TLS Library](https://github.com/0xMassi/webclaw-tls) - [Discord](https://discord.gg/KDfd48EpnW) - [Status](https://status.webclaw.io) diff --git a/packages/create-webclaw/index.mjs b/packages/create-webclaw/index.mjs index 2bb04a4..8dea1ee 100644 --- a/packages/create-webclaw/index.mjs +++ b/packages/create-webclaw/index.mjs @@ -1,29 +1,22 @@ #!/usr/bin/env node -import { - existsSync, - mkdirSync, - readFileSync, - writeFileSync, - copyFileSync, - rmSync, -} from "fs"; +// create-webclaw — optional convenience installer for the webclaw MCP server. +// +// It auto-detects your AI tools (Claude Desktop, Claude Code, Cursor, Windsurf, +// OpenCode, Codex, Antigravity, ...) and writes the canonical `npx @webclaw/mcp` +// config into each. It does NOT download a binary: the runtime is always the +// `@webclaw/mcp` launcher, so a scaffolded config is byte-identical to what +// webclaw.io/docs and the MCP registries document. If you'd rather not run this, +// just add the one config block below by hand. + +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"; import { createInterface } from "readline"; -import { homedir, platform, arch } from "os"; +import { homedir, platform } from "os"; import { join, dirname } from "path"; -import { execSync } from "child_process"; -import { createWriteStream } from "fs"; -import { chmod } from "fs/promises"; -import https from "https"; -import http from "http"; // ── Constants ── -const REPO = "0xMassi/webclaw"; -const IS_WINDOWS = platform() === "win32"; -const BINARY_NAME = IS_WINDOWS ? "webclaw-mcp.exe" : "webclaw-mcp"; -const INSTALL_DIR = join(homedir(), ".webclaw"); -const BINARY_PATH = join(INSTALL_DIR, BINARY_NAME); +const MCP_PACKAGE = "@webclaw/mcp"; const COLORS = { reset: "\x1b[0m", @@ -82,7 +75,6 @@ const AI_TOOLS = [ id: "cursor", name: "Cursor", detect: () => { - // Check for .cursor directory in home or current project return ( existsSync(join(homedir(), ".cursor")) || existsSync(join(process.cwd(), ".cursor")) @@ -173,77 +165,6 @@ function ask(question) { }); } -function download(url, extraHeaders = {}) { - return new Promise((resolve, reject) => { - const client = url.startsWith("https") ? https : http; - const headers = { "User-Agent": "create-webclaw", ...extraHeaders }; - client - .get(url, { headers }, (res) => { - // Follow redirects, dropping extra headers so an Authorization token - // never leaks to the release CDN (its signed URLs reject it anyway). - if ( - res.statusCode >= 300 && - res.statusCode < 400 && - res.headers.location - ) { - return download(res.headers.location).then(resolve).catch(reject); - } - if (res.statusCode !== 200) { - return reject(new Error(`HTTP ${res.statusCode}`)); - } - const chunks = []; - res.on("data", (chunk) => chunks.push(chunk)); - res.on("end", () => resolve(Buffer.concat(chunks))); - res.on("error", reject); - }) - .on("error", reject); - }); -} - -async function downloadFile(url, dest) { - return new Promise((resolve, reject) => { - const client = url.startsWith("https") ? https : http; - client - .get(url, { headers: { "User-Agent": "create-webclaw" } }, (res) => { - if ( - res.statusCode >= 300 && - res.statusCode < 400 && - res.headers.location - ) { - return downloadFile(res.headers.location, dest) - .then(resolve) - .catch(reject); - } - if (res.statusCode !== 200) { - return reject(new Error(`HTTP ${res.statusCode}`)); - } - const file = createWriteStream(dest); - res.pipe(file); - file.on("finish", () => { - file.close(); - resolve(); - }); - file.on("error", reject); - }) - .on("error", reject); - }); -} - -// Map the current platform to its Rust release target triple. Release assets -// are named `webclaw--.` (e.g. -// webclaw-v0.6.13-x86_64-unknown-linux-gnu.tar.gz), so the asset name is built -// from the release's tag_name at fetch time — it can't be hardcoded here. -function getTarget() { - const targets = { - "darwin-arm64": "aarch64-apple-darwin", - "darwin-x64": "x86_64-apple-darwin", - "linux-x64": "x86_64-unknown-linux-gnu", - "linux-arm64": "aarch64-unknown-linux-gnu", - "win32-x64": "x86_64-pc-windows-msvc", - }; - return targets[`${platform()}-${arch()}`] || null; -} - function readJsonFile(path) { try { return JSON.parse(readFileSync(path, "utf-8")); @@ -258,43 +179,20 @@ function writeJsonFile(path, data) { writeFileSync(path, JSON.stringify(data, null, 2) + "\n"); } +// The single canonical way to run the webclaw MCP server: the npx launcher. +// `@webclaw/mcp` fetches the prebuilt binary on first run and speaks MCP over +// stdio, so this config matches webclaw.io/docs and the MCP registries exactly. function buildMcpEntry(apiKey) { - const entry = { - command: BINARY_PATH, - }; - if (apiKey) { - entry.env = { WEBCLAW_API_KEY: apiKey }; - } + const entry = { command: "npx", args: ["-y", MCP_PACKAGE] }; + if (apiKey) entry.env = { WEBCLAW_API_KEY: apiKey }; return entry; } +const MANUAL_CONFIG = `{ "mcpServers": { "webclaw": { "command": "npx", "args": ["-y", "${MCP_PACKAGE}"] } } }`; + // ── MCP Config Writers ── -function addToClaudeDesktop(configPath, apiKey) { - const config = readJsonFile(configPath); - if (!config.mcpServers) config.mcpServers = {}; - config.mcpServers.webclaw = buildMcpEntry(apiKey); - writeJsonFile(configPath, config); -} - -function addToClaudeCode(configPath, apiKey) { - const config = readJsonFile(configPath); - if (!config.mcpServers) config.mcpServers = {}; - config.mcpServers.webclaw = buildMcpEntry(apiKey); - writeJsonFile(configPath, config); -} - -function addToCursor(configPath, apiKey) { - const config = readJsonFile(configPath); - if (!config.mcpServers) config.mcpServers = {}; - config.mcpServers.webclaw = { - command: BINARY_PATH, - ...(apiKey ? { env: { WEBCLAW_API_KEY: apiKey } } : {}), - }; - writeJsonFile(configPath, config); -} - -function addToWindsurf(configPath, apiKey) { +function addToMcpServers(configPath, apiKey) { const config = readJsonFile(configPath); if (!config.mcpServers) config.mcpServers = {}; config.mcpServers.webclaw = buildMcpEntry(apiKey); @@ -304,13 +202,9 @@ function addToWindsurf(configPath, apiKey) { function addToVSCodeContinue(configPath, apiKey) { const config = readJsonFile(configPath); if (!config.mcpServers) config.mcpServers = []; - // Continue uses array format + // Continue uses array format. const existing = config.mcpServers.findIndex?.((s) => s.name === "webclaw"); - const entry = { - name: "webclaw", - command: BINARY_PATH, - ...(apiKey ? { env: { WEBCLAW_API_KEY: apiKey } } : {}), - }; + const entry = { name: "webclaw", ...buildMcpEntry(apiKey) }; if (existing >= 0) { config.mcpServers[existing] = entry; } else if (Array.isArray(config.mcpServers)) { @@ -324,7 +218,7 @@ function addToOpenCode(configPath, apiKey) { if (!config.mcp) config.mcp = {}; config.mcp.webclaw = { type: "local", - command: [BINARY_PATH], + command: ["npx", "-y", MCP_PACKAGE], enabled: true, }; if (apiKey) { @@ -333,15 +227,8 @@ function addToOpenCode(configPath, apiKey) { writeJsonFile(configPath, config); } -function addToAntigravity(configPath, apiKey) { - const config = readJsonFile(configPath); - if (!config.mcpServers) config.mcpServers = {}; - config.mcpServers.webclaw = buildMcpEntry(apiKey); - writeJsonFile(configPath, config); -} - function addToCodex(configPath, apiKey) { - // Codex uses TOML format, not JSON. Append MCP server config section. + // Codex uses TOML, not JSON. Replace any existing webclaw section. const dir = dirname(configPath); if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); @@ -349,31 +236,28 @@ function addToCodex(configPath, apiKey) { try { existing = readFileSync(configPath, "utf-8"); } catch { - // File doesn't exist yet + // File doesn't exist yet. } - - // Remove any existing webclaw MCP section existing = existing.replace( /\n?\[mcp_servers\.webclaw\][^\[]*(?=\[|$)/gs, "", ); - let section = `\n[mcp_servers.webclaw]\ncommand = "${BINARY_PATH}"\nargs = []\nenabled = true\n`; + let section = `\n[mcp_servers.webclaw]\ncommand = "npx"\nargs = ["-y", "${MCP_PACKAGE}"]\nenabled = true\n`; if (apiKey) { section += `env = { WEBCLAW_API_KEY = "${apiKey}" }\n`; } - writeFileSync(configPath, existing.trimEnd() + "\n" + section); } const CONFIG_WRITERS = { - "claude-desktop": addToClaudeDesktop, - "claude-code": addToClaudeCode, - cursor: addToCursor, - windsurf: addToWindsurf, + "claude-desktop": addToMcpServers, + "claude-code": addToMcpServers, + cursor: addToMcpServers, + windsurf: addToMcpServers, "vscode-continue": addToVSCodeContinue, opencode: addToOpenCode, - antigravity: addToAntigravity, + antigravity: addToMcpServers, codex: addToCodex, }; @@ -391,7 +275,7 @@ async function main() { console.log(c("bold", " └─────────────────────────────────────┘")); console.log(); - // 1. Detect installed AI tools + // 1. Detect installed AI tools. console.log(c("bold", " Detecting AI tools...")); console.log(); @@ -406,20 +290,10 @@ async function main() { if (detected.length === 0) { console.log(c("yellow", " No supported AI tools detected.")); console.log(); - console.log(c("dim", " Supported tools:")); - for (const tool of AI_TOOLS) { - console.log(c("dim", ` • ${tool.name}`)); - } + console.log(c("dim", " Add this to your MCP client config by hand:")); + console.log(c("cyan", ` ${MANUAL_CONFIG}`)); console.log(); - console.log( - c("dim", " Install one of these tools and run this command again."), - ); - console.log(c("dim", " Or use --manual to configure manually.")); - console.log(); - - if (process.argv.includes("--manual")) { - // Continue anyway for manual setup - } else { + if (!process.argv.includes("--manual")) { process.exit(0); } } @@ -429,10 +303,12 @@ async function main() { } console.log(); - // 2. Ask for API key - console.log(c("dim", " An API key enables cloud features.")); + // 2. Ask for an optional API key. console.log( - c("dim", " Without one, webclaw runs locally (free, no account needed)."), + c("dim", " An API key unlocks the cloud tools (bot bypass, JS rendering,"), + ); + console.log( + c("dim", " search, research, leads). Without one, webclaw runs locally."), ); console.log(); @@ -442,153 +318,15 @@ async function main() { ); console.log(); - // 3. Download binary - console.log(c("bold", " Downloading webclaw-mcp...")); - - const target = getTarget(); - if (!target) { - console.log(c("red", ` Unsupported platform: ${platform()}-${arch()}`)); - console.log( - c( - "dim", - " Build from source: cargo install --git https://github.com/0xMassi/webclaw webclaw-mcp", - ), - ); - process.exit(1); - } - - if (!existsSync(INSTALL_DIR)) { - mkdirSync(INSTALL_DIR, { recursive: true }); - } - - let downloaded = false; - let prebuiltError = null; - - try { - // Resolve the latest release. Its tag_name drives the asset name. An - // unauthenticated GitHub API call is rate-limited to 60/hour per IP, so - // honour GITHUB_TOKEN when set — but only on this api.github.com request, - // never on the asset download (which redirects to a CDN). - const apiHeaders = process.env.GITHUB_TOKEN - ? { Authorization: `Bearer ${process.env.GITHUB_TOKEN}` } - : {}; - const release = JSON.parse( - ( - await download( - `https://api.github.com/repos/${REPO}/releases/latest`, - apiHeaders, - ) - ).toString(), - ); - - const version = release.tag_name; // e.g. "v0.6.13" - const ext = IS_WINDOWS ? "zip" : "tar.gz"; - const assetName = `webclaw-${version}-${target}.${ext}`; - const asset = release.assets?.find((a) => a.name === assetName); - if (!asset) { - throw new Error(`asset ${assetName} not found in release ${version}`); - } - - const archivePath = join(INSTALL_DIR, assetName); - await downloadFile(asset.browser_download_url, archivePath); - - // Each archive holds a top-level `webclaw--/` directory - // containing webclaw, webclaw-mcp, webclaw-server, and docs. - if (ext === "tar.gz") { - execSync(`tar xzf "${archivePath}" -C "${INSTALL_DIR}"`, { - stdio: "ignore", - }); - } else if (IS_WINDOWS) { - // Windows ships no `unzip`; Expand-Archive comes with PowerShell 5+. - execSync( - `powershell -NoProfile -Command "Expand-Archive -Path '${archivePath}' -DestinationPath '${INSTALL_DIR}' -Force"`, - { stdio: "ignore" }, - ); - } else { - execSync(`unzip -o "${archivePath}" -d "${INSTALL_DIR}"`, { - stdio: "ignore", - }); - } - - // Lift webclaw-mcp out of the extracted directory to BINARY_PATH, then - // drop the rest (the other two binaries + docs). - const extractedDir = join(INSTALL_DIR, `webclaw-${version}-${target}`); - const extractedBin = join(extractedDir, BINARY_NAME); - if (!existsSync(extractedBin)) { - throw new Error(`binary missing after extract: ${extractedBin}`); - } - copyFileSync(extractedBin, BINARY_PATH); - if (!IS_WINDOWS) await chmod(BINARY_PATH, 0o755); - - try { - rmSync(extractedDir, { recursive: true, force: true }); - rmSync(archivePath, { force: true }); - } catch {} - - console.log(c("green", ` ✓ Installed to ${BINARY_PATH}`)); - downloaded = true; - } catch (e) { - prebuiltError = e; - } - - if (!downloaded) { - // Surface why the prebuilt path failed instead of hiding it — a 403 here - // is almost always a GitHub API rate limit, which Rust can't fix. - if (prebuiltError) { - const m = prebuiltError.message || String(prebuiltError); - if (m.includes("403") || /rate limit/i.test(m)) { - console.log( - c( - "yellow", - " GitHub API rate limit hit. Retry in a few minutes, or set GITHUB_TOKEN.", - ), - ); - } else { - console.log(c("yellow", ` Prebuilt binary unavailable (${m}).`)); - } - } - - // Fall back to building from source. - console.log(c("yellow", " Trying cargo install...")); - try { - execSync( - `cargo install --git https://github.com/${REPO} webclaw-mcp --root "${INSTALL_DIR}"`, - { stdio: "inherit" }, - ); - // cargo install puts the binary in INSTALL_DIR/bin/ - const cargoPath = join(INSTALL_DIR, "bin", BINARY_NAME); - if (existsSync(cargoPath)) { - copyFileSync(cargoPath, BINARY_PATH); - console.log(c("green", ` ✓ Built and installed to ${BINARY_PATH}`)); - downloaded = true; - } - } catch { - console.log( - c("red", " Failed to install. Make sure Rust is installed:"), - ); - console.log( - c( - "dim", - " curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh", - ), - ); - process.exit(1); - } - } - - console.log(); - - // 4. Configure each detected tool - console.log(c("bold", " Configuring MCP servers...")); + // 3. Write the npx config into each detected tool. + console.log(c("bold", " Writing MCP config...")); console.log(); for (const tool of detected) { const configPath = tool.configPath(); if (!configPath) continue; - const writer = CONFIG_WRITERS[tool.id]; if (!writer) continue; - try { writer(configPath, apiKey || null); console.log( @@ -598,31 +336,27 @@ async function main() { console.log(c("red", ` ✗ ${tool.name}: ${e.message}`)); } } - console.log(); - // 5. Verify - if (downloaded) { - try { - const version = execSync(`"${BINARY_PATH}" --version`, { - encoding: "utf-8", - }).trim(); - console.log(c("green", ` ✓ ${version}`)); - } catch { - console.log(c("green", ` ✓ webclaw-mcp installed`)); - } - } - - // 6. Summary + // 4. Summary. + console.log(c("bold", " Done! webclaw is configured.")); console.log(); - console.log(c("bold", " Done! webclaw is ready.")); + console.log( + c("dim", " Your client runs the server on demand via npx — nothing to"), + ); + console.log( + c("dim", " install. First launch fetches @webclaw/mcp, then it's cached."), + ); console.log(); - console.log(c("dim", " Your AI agent now has these tools:")); - console.log(c("dim", " • scrape — extract content from any URL")); - console.log(c("dim", " • crawl — recursively crawl a website")); - console.log(c("dim", " • search — web search + parallel scrape")); - console.log(c("dim", " • map — discover URLs from sitemaps")); - console.log(c("dim", " • batch — extract multiple URLs in parallel")); + console.log( + c("dim", " Tools: scrape, search, crawl, map, batch, extract, summarize,"), + ); + console.log( + c( + "dim", + " diff, brand, research, lead, lead_batch, + 30 site extractors.", + ), + ); console.log(); if (!apiKey) { @@ -630,7 +364,7 @@ async function main() { console.log( c( "dim", - " Get an API key at https://webclaw.io/dashboard for cloud features.", + " Get a key at https://webclaw.io/dashboard for cloud features.", ), ); console.log(); diff --git a/packages/create-webclaw/package.json b/packages/create-webclaw/package.json index d410447..9ee61cf 100644 --- a/packages/create-webclaw/package.json +++ b/packages/create-webclaw/package.json @@ -1,12 +1,15 @@ { "name": "create-webclaw", - "version": "0.1.5", - "mcpName": "io.github.0xMassi/webclaw", + "version": "0.1.7", "description": "Set up webclaw MCP server for AI agents (Claude, Cursor, Windsurf, OpenCode, Codex, Antigravity)", "bin": { "create-webclaw": "./index.mjs" }, "type": "module", + "files": [ + "index.mjs", + "README.md" + ], "keywords": [ "webclaw", "mcp", diff --git a/packages/create-webclaw/server.json b/packages/create-webclaw/server.json index 0cfc140..d57a083 100644 --- a/packages/create-webclaw/server.json +++ b/packages/create-webclaw/server.json @@ -2,13 +2,13 @@ "$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json", "name": "io.github.0xMassi/webclaw", "title": "webclaw", - "description": "Web extraction MCP server. Scrape, crawl, extract, summarize any URL to clean markdown.", - "version": "0.1.4", + "description": "Turn any URL into clean markdown/JSON for AI agents. Self-hostable web content extraction.", + "version": "0.6.17", "packages": [ { "registryType": "npm", - "identifier": "create-webclaw", - "version": "0.1.4", + "identifier": "@webclaw/mcp", + "version": "0.6.16", "transport": { "type": "stdio" } diff --git a/packages/webclaw-mcp/README.md b/packages/webclaw-mcp/README.md new file mode 100644 index 0000000..7a501fe --- /dev/null +++ b/packages/webclaw-mcp/README.md @@ -0,0 +1,71 @@ +# @webclaw/mcp + +**Clean web access for AI agents, over MCP.** Turn any URL into markdown, JSON, or LLM-ready context — including pages that block bots or need JavaScript — straight from Claude, Cursor, and any MCP client. + +Zero-install launcher for the [webclaw](https://webclaw.io) MCP server. `npx @webclaw/mcp` downloads a prebuilt `webclaw-mcp` binary once (verified against the release `SHA256SUMS`), caches it, and runs it as an MCP stdio server. No Rust build, no global install. It runs on your machine; the hosted webclaw cloud is used only for the tools that need it, and only when you set `WEBCLAW_API_KEY`. + +## Add to your MCP client + +Point any stdio MCP client at the npx command — Claude Desktop, Cursor, Windsurf, Antigravity (`mcpServers` JSON): + +```json +{ + "mcpServers": { + "webclaw": { + "command": "npx", + "args": ["-y", "@webclaw/mcp"] + } + } +} +``` + +Add a key to unlock the cloud-backed tools (bot-protection bypass, JS rendering, web search, research, lead enrichment): + +```json +{ + "mcpServers": { + "webclaw": { + "command": "npx", + "args": ["-y", "@webclaw/mcp"], + "env": { "WEBCLAW_API_KEY": "wc_your_key" } + } + } +} +``` + +Claude Code: + +```bash +claude mcp add webclaw -- npx -y @webclaw/mcp +``` + +Or run `npx create-webclaw` to auto-detect your AI tools and write their configs for you. + +## Tools (14) + +scrape, search, crawl, map, batch, extract, summarize, diff, brand, research, lead, lead_batch, plus `list_extractors` / `vertical_scrape` for 30+ site-specific extractors (Amazon, GitHub, Reddit, YouTube, npm, PyPI, and more). + +- **No key needed:** scrape, crawl, map, batch, diff, brand, list_extractors, vertical_scrape. +- **Needs an LLM** (local Ollama or a provider key): extract, summarize. +- **Needs `WEBCLAW_API_KEY`:** search, research, lead, lead_batch — plus automatic bot-protection bypass and JS rendering for the fetch tools. + +Get a key at [webclaw.io](https://webclaw.io). + +## Environment + +| Variable | Purpose | +|---|---| +| `WEBCLAW_API_KEY` | Enables cloud-backed tools (optional). | +| `WEBCLAW_MCP_BIN` | Absolute path to a `webclaw-mcp` binary; skips the download. | +| `WEBCLAW_MCP_VERSION` | Release tag to install (default: the pinned release). | +| `WEBCLAW_MCP_CACHE` | Cache directory (default: `~/.cache/webclaw`). | + +## Links + +- **Docs:** [webclaw.io/docs/mcp](https://webclaw.io/docs/mcp) +- **Source** (CLI, REST API, SDKs, extraction engine): [github.com/0xMassi/webclaw](https://github.com/0xMassi/webclaw) +- **Hosted API & keys:** [webclaw.io](https://webclaw.io) + +## License + +AGPL-3.0-only diff --git a/packages/webclaw-mcp/package.json b/packages/webclaw-mcp/package.json new file mode 100644 index 0000000..76212ba --- /dev/null +++ b/packages/webclaw-mcp/package.json @@ -0,0 +1,39 @@ +{ + "name": "@webclaw/mcp", + "version": "0.6.16", + "description": "Zero-install launcher for the webclaw MCP server: web extraction and anti-bot access for AI agents. Runs `npx @webclaw/mcp`.", + "type": "module", + "bin": { + "webclaw-mcp-launcher": "./webclaw-mcp.mjs" + }, + "mcpName": "io.github.0xMassi/webclaw", + "files": [ + "webclaw-mcp.mjs", + "README.md" + ], + "engines": { + "node": ">=18" + }, + "license": "AGPL-3.0-only", + "author": "0xMassi", + "homepage": "https://webclaw.io", + "repository": { + "type": "git", + "url": "git+https://github.com/0xMassi/webclaw.git", + "directory": "packages/webclaw-mcp" + }, + "bugs": { + "url": "https://github.com/0xMassi/webclaw/issues" + }, + "keywords": [ + "mcp", + "model-context-protocol", + "webclaw", + "web-scraping", + "web-extraction", + "ai-agents", + "cloudflare", + "anti-bot", + "llm" + ] +} diff --git a/packages/webclaw-mcp/webclaw-mcp.mjs b/packages/webclaw-mcp/webclaw-mcp.mjs new file mode 100755 index 0000000..bedf9a6 --- /dev/null +++ b/packages/webclaw-mcp/webclaw-mcp.mjs @@ -0,0 +1,266 @@ +#!/usr/bin/env node +/** + * @webclaw/mcp — zero-install launcher for the webclaw MCP server. + * + * `npx -y @webclaw/mcp` resolves a prebuilt `webclaw-mcp` binary (downloaded + * once from the pinned GitHub release, then cached) and execs it as an MCP + * stdio server. This is what makes webclaw introspectable/installable in MCP + * clients and registries (Claude Desktop, Cursor, Glama, Smithery, ...) without + * a Rust build. + * + * HARD RULE: stdout carries the MCP JSON-RPC stream. Every diagnostic this + * launcher emits MUST go to stderr, or it corrupts the protocol. Never + * console.log here — only logErr(). + * + * Overrides (env): + * WEBCLAW_MCP_BIN absolute path to a webclaw-mcp binary; skip download + * WEBCLAW_MCP_VERSION release tag to install (default: pinned RELEASE_TAG) + * WEBCLAW_MCP_CACHE cache root (default: ~/.cache/webclaw) + * GITHUB_TOKEN only used to fetch SHA256SUMS if rate-limited + */ + +import { + existsSync, + mkdirSync, + createWriteStream, + readFileSync, + renameSync, + copyFileSync, + rmSync, + chmodSync, +} from "node:fs"; +import { homedir, platform, arch } from "node:os"; +import { join } from "node:path"; +import { spawn, execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import https from "node:https"; + +const REPO = "0xMassi/webclaw"; +// Release the wrapper installs. Bump this (and the package version) on each +// core release, or override at runtime with WEBCLAW_MCP_VERSION. +const RELEASE_TAG = process.env.WEBCLAW_MCP_VERSION || "v0.6.15"; + +const IS_WINDOWS = platform() === "win32"; +const BIN_NAME = IS_WINDOWS ? "webclaw-mcp.exe" : "webclaw-mcp"; +const CACHE_ROOT = + process.env.WEBCLAW_MCP_CACHE || join(homedir(), ".cache", "webclaw"); +const CACHE_DIR = join(CACHE_ROOT, RELEASE_TAG); +const CACHED_BIN = join(CACHE_DIR, BIN_NAME); + +function logErr(msg) { + process.stderr.write(`[@webclaw/mcp] ${msg}\n`); +} + +function target() { + const map = { + "darwin-arm64": "aarch64-apple-darwin", + "darwin-x64": "x86_64-apple-darwin", + "linux-x64": "x86_64-unknown-linux-gnu", + "linux-arm64": "aarch64-unknown-linux-gnu", + "win32-x64": "x86_64-pc-windows-msvc", + }; + return map[`${platform()}-${arch()}`] || null; +} + +// GET a URL to a Buffer, following redirects. Auth headers are dropped on +// redirect so a token never reaches the release CDN (its signed URLs reject it). +function getBuffer(url, headers = {}) { + return new Promise((resolve, reject) => { + https + .get( + url, + { headers: { "User-Agent": "@webclaw/mcp", ...headers } }, + (res) => { + if ( + res.statusCode >= 300 && + res.statusCode < 400 && + res.headers.location + ) { + res.resume(); + return getBuffer(res.headers.location).then(resolve, reject); + } + if (res.statusCode !== 200) { + res.resume(); + return reject(new Error(`HTTP ${res.statusCode} for ${url}`)); + } + const chunks = []; + res.on("data", (c) => chunks.push(c)); + res.on("end", () => resolve(Buffer.concat(chunks))); + res.on("error", reject); + }, + ) + .on("error", reject); + }); +} + +// Stream a URL to a file, following redirects. +function getFile(url, dest) { + return new Promise((resolve, reject) => { + https + .get(url, { headers: { "User-Agent": "@webclaw/mcp" } }, (res) => { + if ( + res.statusCode >= 300 && + res.statusCode < 400 && + res.headers.location + ) { + res.resume(); + return getFile(res.headers.location, dest).then(resolve, reject); + } + if (res.statusCode !== 200) { + res.resume(); + return reject(new Error(`HTTP ${res.statusCode} for ${url}`)); + } + const out = createWriteStream(dest); + res.pipe(out); + out.on("finish", () => out.close(() => resolve())); + out.on("error", reject); + }) + .on("error", reject); + }); +} + +function sha256(path) { + return createHash("sha256").update(readFileSync(path)).digest("hex"); +} + +function extract(archivePath, destDir) { + if (IS_WINDOWS) { + execFileSync( + "powershell", + [ + "-NoProfile", + "-Command", + `Expand-Archive -Path '${archivePath}' -DestinationPath '${destDir}' -Force`, + ], + { stdio: "ignore" }, + ); + } else { + execFileSync("tar", ["xzf", archivePath, "-C", destDir], { + stdio: "ignore", + }); + } +} + +async function ensureBinary() { + // 0. Explicit local binary (dev / offline / CI). + const override = process.env.WEBCLAW_MCP_BIN; + if (override) { + if (!existsSync(override)) { + throw new Error(`WEBCLAW_MCP_BIN points at a missing file: ${override}`); + } + return override; + } + + // 1. Cache hit for this release. + if (existsSync(CACHED_BIN)) return CACHED_BIN; + + // 2. Download, verify, extract, cache. + const tgt = target(); + if (!tgt) { + throw new Error( + `unsupported platform ${platform()}-${arch()} — install the webclaw-mcp binary manually ` + + `(https://github.com/${REPO}/releases) and set WEBCLAW_MCP_BIN`, + ); + } + const ext = IS_WINDOWS ? "zip" : "tar.gz"; + const assetName = `webclaw-${RELEASE_TAG}-${tgt}.${ext}`; + const base = `https://github.com/${REPO}/releases/download/${RELEASE_TAG}`; + + mkdirSync(CACHE_DIR, { recursive: true }); + const archivePath = join(CACHE_DIR, assetName); + const tmpPath = `${archivePath}.download`; + + logErr(`fetching ${assetName} (first run for ${RELEASE_TAG}) ...`); + await getFile(`${base}/${assetName}`, tmpPath); + + // Verify against the release SHA256SUMS. If the sums file can't be fetched we + // proceed with a warning rather than hard-failing an otherwise-good install. + try { + const apiHeaders = process.env.GITHUB_TOKEN + ? { Authorization: `Bearer ${process.env.GITHUB_TOKEN}` } + : {}; + const sums = (await getBuffer(`${base}/SHA256SUMS`, apiHeaders)).toString( + "utf8", + ); + const line = sums.split("\n").find((l) => l.trim().endsWith(assetName)); + const expected = line ? line.trim().split(/\s+/)[0] : null; + if (expected) { + const actual = sha256(tmpPath); + if (actual !== expected) { + rmSync(tmpPath, { force: true }); + throw new Error( + `checksum mismatch for ${assetName}: expected ${expected}, got ${actual}`, + ); + } + } else { + logErr( + `warning: ${assetName} not found in SHA256SUMS — skipping verification`, + ); + } + } catch (e) { + if (/checksum mismatch/.test(e.message)) throw e; + logErr(`warning: could not verify checksum (${e.message}) — proceeding`); + } + + renameSync(tmpPath, archivePath); + extract(archivePath, CACHE_DIR); + + // Archives hold a top-level `webclaw--/` dir with all binaries. + const extractedDir = join(CACHE_DIR, `webclaw-${RELEASE_TAG}-${tgt}`); + const extractedBin = join(extractedDir, BIN_NAME); + if (!existsSync(extractedBin)) { + throw new Error(`binary missing after extract: ${extractedBin}`); + } + copyFileSync(extractedBin, CACHED_BIN); + if (!IS_WINDOWS) chmodSync(CACHED_BIN, 0o755); + + // Drop the archive and the extra binaries; keep only the cached webclaw-mcp. + try { + rmSync(extractedDir, { recursive: true, force: true }); + rmSync(archivePath, { force: true }); + } catch { + /* non-fatal */ + } + + logErr(`installed ${BIN_NAME} for ${RELEASE_TAG} → ${CACHED_BIN}`); + return CACHED_BIN; +} + +async function main() { + let bin; + try { + bin = await ensureBinary(); + } catch (e) { + logErr(`error: ${e.message}`); + process.exit(1); + } + + // Hand off to the real server. stdio:inherit wires the MCP JSON-RPC stream + // straight through; env passes WEBCLAW_API_KEY and friends unchanged. + const child = spawn(bin, process.argv.slice(2), { + stdio: "inherit", + env: process.env, + }); + + child.on("error", (e) => { + logErr(`failed to start webclaw-mcp: ${e.message}`); + process.exit(1); + }); + child.on("exit", (code, signal) => { + if (signal) { + // Re-raise the signal so the parent's exit reflects it. + process.kill(process.pid, signal); + } else { + process.exit(code ?? 0); + } + }); + + // Forward termination signals to the child so clients can stop the server. + for (const sig of ["SIGINT", "SIGTERM", "SIGHUP"]) { + process.on(sig, () => { + if (!child.killed) child.kill(sig); + }); + } +} + +main(); diff --git a/skill/SKILL.md b/skill/SKILL.md deleted file mode 100644 index 39fc144..0000000 --- a/skill/SKILL.md +++ /dev/null @@ -1,634 +0,0 @@ ---- -name: webclaw -description: Web extraction engine with antibot bypass. Scrape, crawl, extract, summarize, search, map, diff, monitor, research, and analyze any URL — including Cloudflare-protected sites. Use when you need reliable web content, the built-in web_fetch fails, or you need structured data extraction from web pages. -homepage: https://webclaw.io -user-invocable: true -metadata: {"openclaw":{"emoji":"🦀","requires":{"env":["WEBCLAW_API_KEY"]},"primaryEnv":"WEBCLAW_API_KEY","homepage":"https://webclaw.io","install":[{"id":"npx","kind":"node","bins":["webclaw-mcp"],"label":"npx create-webclaw"}]}} ---- - -# webclaw - -High-quality web extraction with automatic antibot bypass. Beats Firecrawl on extraction quality and handles Cloudflare, DataDome, and JS-rendered pages automatically. - -## When to use this skill - -- **Always** when you need to fetch web content and want reliable results -- When `web_fetch` returns empty/blocked content (403, Cloudflare challenges) -- When you need structured data extraction (pricing tables, product info) -- When you need to crawl an entire site or discover all URLs -- When you need LLM-optimized content (cleaner than raw markdown) -- When you need to summarize a page without reading the full content -- When you need to detect content changes between visits -- When you need brand identity analysis (colors, fonts, logos) -- When you need web search results with optional page scraping -- When you need deep multi-source research on a topic -- When you need AI-guided scraping to accomplish a goal on a page -- When you need to monitor a URL for changes over time - -## API base - -All requests go to `https://api.webclaw.io/v1/`. - -Authentication: `Authorization: Bearer $WEBCLAW_API_KEY` - -## Endpoints - -### 1. Scrape — extract content from a single URL - -```bash -curl -X POST https://api.webclaw.io/v1/scrape \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" \ - -H "Content-Type: application/json" \ - -d '{ - "url": "https://example.com", - "formats": ["markdown"], - "only_main_content": true - }' -``` - -**Request fields:** - -| Field | Type | Default | Description | -|-------|------|---------|-------------| -| `url` | string | required | URL to scrape | -| `formats` | string[] | `["markdown"]` | Output formats: `markdown`, `text`, `llm`, `json` | -| `include_selectors` | string[] | `[]` | CSS selectors to keep (e.g. `["article", ".content"]`) | -| `exclude_selectors` | string[] | `[]` | CSS selectors to remove (e.g. `["nav", "footer", ".ads"]`) | -| `only_main_content` | bool | `false` | Extract only the main article/content area | -| `no_cache` | bool | `false` | Skip cache, fetch fresh | -| `max_cache_age` | int | server default | Max acceptable cache age in seconds | - -**Response:** - -```json -{ - "url": "https://example.com", - "metadata": { - "title": "Example", - "description": "...", - "language": "en", - "word_count": 1234 - }, - "markdown": "# Page Title\n\nContent here...", - "cache": { "status": "miss" } -} -``` - -**Format options:** -- `markdown` — clean markdown, best for general use -- `text` — plain text without formatting -- `llm` — optimized for LLM consumption: includes page title, URL, and cleaned content with link references. Best for feeding to AI models. -- `json` — full extraction result with all metadata - -**When antibot bypass activates** (automatic, no extra config): -```json -{ - "antibot": { - "bypass": true, - "elapsed_ms": 3200 - } -} -``` - -### 2. Crawl — scrape an entire website - -Starts an async job. Poll for results. - -**Start crawl:** -```bash -curl -X POST https://api.webclaw.io/v1/crawl \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" \ - -H "Content-Type: application/json" \ - -d '{ - "url": "https://docs.example.com", - "max_depth": 3, - "max_pages": 50, - "use_sitemap": true - }' -``` - -Response: `{ "job_id": "abc-123", "status": "running" }` - -**Poll status:** -```bash -curl https://api.webclaw.io/v1/crawl/abc-123 \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" -``` - -Response when complete: -```json -{ - "job_id": "abc-123", - "status": "completed", - "total": 47, - "completed": 45, - "errors": 2, - "pages": [ - { - "url": "https://docs.example.com/intro", - "markdown": "# Introduction\n...", - "metadata": { "title": "Intro", "word_count": 500 } - } - ] -} -``` - -**Request fields:** - -| Field | Type | Default | Description | -|-------|------|---------|-------------| -| `url` | string | required | Starting URL | -| `max_depth` | int | `3` | How many links deep to follow | -| `max_pages` | int | `100` | Maximum pages to crawl | -| `use_sitemap` | bool | `false` | Seed URLs from sitemap.xml | -| `formats` | string[] | `["markdown"]` | Output formats per page | -| `include_selectors` | string[] | `[]` | CSS selectors to keep | -| `exclude_selectors` | string[] | `[]` | CSS selectors to remove | -| `only_main_content` | bool | `false` | Main content only | - -### 3. Map — discover all URLs on a site - -Fast URL discovery without full content extraction. - -```bash -curl -X POST https://api.webclaw.io/v1/map \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" \ - -H "Content-Type: application/json" \ - -d '{"url": "https://example.com"}' -``` - -Response: -```json -{ - "url": "https://example.com", - "count": 142, - "urls": [ - "https://example.com/about", - "https://example.com/pricing", - "https://example.com/docs/intro" - ] -} -``` - -### 4. Batch — scrape multiple URLs in parallel - -```bash -curl -X POST https://api.webclaw.io/v1/batch \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" \ - -H "Content-Type: application/json" \ - -d '{ - "urls": [ - "https://a.com", - "https://b.com", - "https://c.com" - ], - "formats": ["markdown"], - "concurrency": 5 - }' -``` - -Response: -```json -{ - "total": 3, - "completed": 3, - "errors": 0, - "results": [ - { "url": "https://a.com", "markdown": "...", "metadata": {} }, - { "url": "https://b.com", "markdown": "...", "metadata": {} }, - { "url": "https://c.com", "error": "timeout" } - ] -} -``` - -### 5. Extract — LLM-powered structured extraction - -Pull structured data from any page using a JSON schema or plain-text prompt. - -**With JSON schema:** -```bash -curl -X POST https://api.webclaw.io/v1/extract \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" \ - -H "Content-Type: application/json" \ - -d '{ - "url": "https://example.com/pricing", - "schema": { - "type": "object", - "properties": { - "plans": { - "type": "array", - "items": { - "type": "object", - "properties": { - "name": { "type": "string" }, - "price": { "type": "string" }, - "features": { "type": "array", "items": { "type": "string" } } - } - } - } - } - } - }' -``` - -**With prompt:** -```bash -curl -X POST https://api.webclaw.io/v1/extract \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" \ - -H "Content-Type: application/json" \ - -d '{ - "url": "https://example.com/pricing", - "prompt": "Extract all pricing tiers with names, monthly prices, and key features" - }' -``` - -Response: -```json -{ - "url": "https://example.com/pricing", - "data": { - "plans": [ - { "name": "Starter", "price": "$49/mo", "features": ["10k pages", "Email support"] }, - { "name": "Pro", "price": "$99/mo", "features": ["100k pages", "Priority support", "API access"] } - ] - } -} -``` - -### 6. Summarize — get a quick summary of any page - -```bash -curl -X POST https://api.webclaw.io/v1/summarize \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" \ - -H "Content-Type: application/json" \ - -d '{ - "url": "https://example.com/long-article", - "max_sentences": 3 - }' -``` - -Response: -```json -{ - "url": "https://example.com/long-article", - "summary": "The article discusses... Key findings include... The author concludes that..." -} -``` - -### 7. Diff — detect content changes - -Compare current page content against a previous snapshot. - -```bash -curl -X POST https://api.webclaw.io/v1/diff \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" \ - -H "Content-Type: application/json" \ - -d '{ - "url": "https://example.com", - "previous": { - "markdown": "# Old content...", - "metadata": { "title": "Old Title" } - } - }' -``` - -Response: -```json -{ - "url": "https://example.com", - "status": "changed", - "diff": "--- previous\n+++ current\n@@ -1 +1 @@\n-# Old content\n+# New content", - "metadata_changes": [ - { "field": "title", "old": "Old Title", "new": "New Title" } - ] -} -``` - -### 8. Brand — extract brand identity - -Analyze a website's visual identity: colors, fonts, logo. - -```bash -curl -X POST https://api.webclaw.io/v1/brand \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" \ - -H "Content-Type: application/json" \ - -d '{"url": "https://example.com"}' -``` - -Response: -```json -{ - "url": "https://example.com", - "brand": { - "colors": [ - { "hex": "#FF6B35", "usage": "primary" }, - { "hex": "#1A1A2E", "usage": "background" } - ], - "fonts": ["Inter", "JetBrains Mono"], - "logo_url": "https://example.com/logo.svg", - "favicon_url": "https://example.com/favicon.ico" - } -} -``` - -### 9. Search — web search with optional scraping - -Search the web and optionally scrape each result page. - -```bash -curl -X POST https://api.webclaw.io/v1/search \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" \ - -H "Content-Type: application/json" \ - -d '{ - "query": "best rust web frameworks 2026", - "num_results": 5, - "scrape": true, - "formats": ["markdown"] - }' -``` - -**Request fields:** - -| Field | Type | Default | Description | -|-------|------|---------|-------------| -| `query` | string | required | Search query | -| `num_results` | int | `10` | Number of search results to return | -| `scrape` | bool | `false` | Also scrape each result page for full content | -| `formats` | string[] | `["markdown"]` | Output formats when `scrape` is true | -| `country` | string | none | Country code for localized results (e.g. `"us"`, `"de"`) | -| `lang` | string | none | Language code for results (e.g. `"en"`, `"fr"`) | - -**Response:** - -```json -{ - "query": "best rust web frameworks 2026", - "results": [ - { - "title": "Top Rust Web Frameworks in 2026", - "url": "https://blog.example.com/rust-frameworks", - "snippet": "A comprehensive comparison of Axum, Actix, and Rocket...", - "position": 1, - "markdown": "# Top Rust Web Frameworks\n\n..." - }, - { - "title": "Choosing a Rust Backend Framework", - "url": "https://dev.to/rust-backends", - "snippet": "When starting a new Rust web project...", - "position": 2, - "markdown": "# Choosing a Rust Backend\n\n..." - } - ] -} -``` - -The `markdown` field on each result is only present when `scrape: true`. Without it, you get titles, URLs, snippets, and positions only. - -### 10. Research — deep multi-source research - -Starts an async research job that searches, scrapes, and synthesizes information across multiple sources. Poll for results. - -**Start research:** -```bash -curl -X POST https://api.webclaw.io/v1/research \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" \ - -H "Content-Type: application/json" \ - -d '{ - "query": "How does Cloudflare Turnstile work and what are its known bypass methods?", - "max_iterations": 5, - "max_sources": 10, - "topic": "security", - "deep": true - }' -``` - -**Request fields:** - -| Field | Type | Default | Description | -|-------|------|---------|-------------| -| `query` | string | required | Research question or topic | -| `max_iterations` | int | server default | Maximum research iterations (search-read-analyze cycles) | -| `max_sources` | int | server default | Maximum number of sources to consult | -| `topic` | string | none | Topic hint to guide search strategy (e.g. `"security"`, `"finance"`, `"engineering"`) | -| `deep` | bool | `false` | Enable deep research mode for more thorough analysis (costs 10 credits instead of 1) | - -Response: `{ "id": "res-abc-123", "status": "running" }` - -**Poll results:** -```bash -curl https://api.webclaw.io/v1/research/res-abc-123 \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" -``` - -Response when complete: -```json -{ - "id": "res-abc-123", - "status": "completed", - "query": "How does Cloudflare Turnstile work and what are its known bypass methods?", - "report": "# Cloudflare Turnstile Analysis\n\n## Overview\nCloudflare Turnstile is a CAPTCHA replacement that...\n\n## How It Works\n...\n\n## Known Bypass Methods\n...", - "sources": [ - { "url": "https://developers.cloudflare.com/turnstile/", "title": "Turnstile Documentation" }, - { "url": "https://blog.cloudflare.com/turnstile-ga/", "title": "Turnstile GA Announcement" } - ], - "findings": [ - "Turnstile uses browser environment signals and proof-of-work challenges", - "Managed mode auto-selects challenge difficulty based on visitor risk score", - "Known bypass approaches include instrumented browser automation" - ], - "iterations": 5, - "elapsed_ms": 34200 -} -``` - -**Status values:** `running`, `completed`, `failed` - -### 11. Agent Scrape — AI-guided scraping - -Use an AI agent to navigate and interact with a page to accomplish a specific goal. The agent can click, scroll, fill forms, and extract data across multiple steps. - -```bash -curl -X POST https://api.webclaw.io/v1/agent-scrape \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" \ - -H "Content-Type: application/json" \ - -d '{ - "url": "https://example.com/products", - "goal": "Find the cheapest laptop with at least 16GB RAM and extract its full specs", - "max_steps": 10 - }' -``` - -**Request fields:** - -| Field | Type | Default | Description | -|-------|------|---------|-------------| -| `url` | string | required | Starting URL | -| `goal` | string | required | What the agent should accomplish | -| `max_steps` | int | server default | Maximum number of actions the agent can take | - -**Response:** - -```json -{ - "url": "https://example.com/products", - "result": "The cheapest laptop with 16GB+ RAM is the ThinkPad E14 Gen 6 at $649. Specs: AMD Ryzen 5 7535U, 16GB DDR4, 512GB SSD, 14\" FHD IPS display, 57Wh battery.", - "steps": [ - { "action": "navigate", "detail": "Loaded products page" }, - { "action": "click", "detail": "Clicked 'Laptops' category filter" }, - { "action": "click", "detail": "Applied '16GB+' RAM filter" }, - { "action": "click", "detail": "Sorted by price: low to high" }, - { "action": "extract", "detail": "Extracted specs from first matching product" } - ] -} -``` - -### 12. Watch — monitor a URL for changes - -Create persistent monitors that check a URL on a schedule and notify via webhook when content changes. - -**Create a monitor:** -```bash -curl -X POST https://api.webclaw.io/v1/watch \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" \ - -H "Content-Type: application/json" \ - -d '{ - "url": "https://example.com/pricing", - "interval": "0 */6 * * *", - "webhook_url": "https://hooks.example.com/pricing-changed", - "formats": ["markdown"] - }' -``` - -**Request fields:** - -| Field | Type | Default | Description | -|-------|------|---------|-------------| -| `url` | string | required | URL to monitor | -| `interval` | string | required | Check frequency as cron expression or seconds (e.g. `"0 */6 * * *"` or `"3600"`) | -| `webhook_url` | string | none | URL to POST when changes are detected | -| `formats` | string[] | `["markdown"]` | Output formats for snapshots | - -Response: -```json -{ - "id": "watch-abc-123", - "url": "https://example.com/pricing", - "interval": "0 */6 * * *", - "webhook_url": "https://hooks.example.com/pricing-changed", - "formats": ["markdown"], - "created_at": "2026-03-20T10:00:00Z", - "last_check": null, - "status": "active" -} -``` - -**List all monitors:** -```bash -curl https://api.webclaw.io/v1/watch \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" -``` - -Response: -```json -{ - "monitors": [ - { - "id": "watch-abc-123", - "url": "https://example.com/pricing", - "interval": "0 */6 * * *", - "status": "active", - "last_check": "2026-03-20T16:00:00Z", - "checks": 4 - } - ] -} -``` - -**Get a monitor with snapshots:** -```bash -curl https://api.webclaw.io/v1/watch/watch-abc-123 \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" -``` - -Response: -```json -{ - "id": "watch-abc-123", - "url": "https://example.com/pricing", - "interval": "0 */6 * * *", - "status": "active", - "snapshots": [ - { - "checked_at": "2026-03-20T16:00:00Z", - "status": "changed", - "diff": "--- previous\n+++ current\n@@ -5 +5 @@\n-Pro: $99/mo\n+Pro: $119/mo" - }, - { - "checked_at": "2026-03-20T10:00:00Z", - "status": "baseline" - } - ] -} -``` - -**Trigger an immediate check:** -```bash -curl -X POST https://api.webclaw.io/v1/watch/watch-abc-123/check \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" -``` - -**Delete a monitor:** -```bash -curl -X DELETE https://api.webclaw.io/v1/watch/watch-abc-123 \ - -H "Authorization: Bearer $WEBCLAW_API_KEY" -``` - -## Choosing the right format - -| Goal | Format | Why | -|------|--------|-----| -| Read and understand a page | `markdown` | Clean structure, headings, links preserved | -| Feed content to an AI model | `llm` | Optimized: includes title + URL header, clean link refs | -| Search or index content | `text` | Plain text, no formatting noise | -| Programmatic analysis | `json` | Full metadata, structured data, DOM statistics | - -## Tips - -- **Use `llm` format** when passing content to yourself or another AI — it's specifically optimized for LLM consumption with better context framing. -- **Use `only_main_content: true`** to skip navigation, sidebars, and footers. Reduces noise significantly. -- **Use `include_selectors`/`exclude_selectors`** for fine-grained control when `only_main_content` isn't enough. -- **Batch over individual scrapes** when fetching multiple URLs — it's faster and more efficient. -- **Use `map` before `crawl`** to discover the site structure first, then crawl specific sections. -- **Use `extract` with a JSON schema** for reliable structured output (e.g., pricing tables, product specs, contact info). -- **Antibot bypass is automatic** — no extra configuration needed. Works on Cloudflare, DataDome, AWS WAF, and JS-rendered SPAs. -- **Use `search` with `scrape: true`** to get full page content for each search result in one call instead of searching then scraping separately. -- **Use `research` for complex questions** that need multiple sources — it handles the search-read-synthesize loop automatically. Enable `deep: true` for thorough analysis. -- **Use `agent-scrape` for interactive pages** where data is behind filters, pagination, or form submissions that a simple scrape cannot reach. -- **Use `watch` for ongoing monitoring** — set up a cron schedule and a webhook to get notified when a page changes without polling manually. - -## Smart Fetch Architecture - -The webclaw MCP server uses a **local-first** approach: - -1. **Local fetch** — fast, free, no API credits used (~80% of sites) -2. **Cloud API fallback** — automatic when bot protection or JS rendering is detected - -This means: -- Most scrapes cost zero credits (local extraction) -- Cloudflare, DataDome, AWS WAF sites automatically fall back to the cloud API -- JS-rendered SPAs (React, Next.js, Vue) also fall back automatically -- Set `WEBCLAW_API_KEY` to enable cloud fallback - -## vs web_fetch - -| | webclaw | web_fetch | -|---|---------|-----------| -| Cloudflare bypass | Automatic (cloud fallback) | Fails (403) | -| JS-rendered pages | Automatic fallback | Readability only | -| Output quality | 20-step optimization pipeline | Basic HTML parsing | -| Structured extraction | LLM-powered, schema-based | None | -| Crawling | Full site crawl with sitemap | Single page only | -| Caching | Built-in, configurable TTL | Per-session | -| Rate limiting | Managed server-side | Client responsibility | - -Use `web_fetch` for simple, fast lookups. Use webclaw when you need reliability, quality, or advanced features. diff --git a/smithery.yaml b/smithery.yaml index 00a69aa..0ed5a34 100644 --- a/smithery.yaml +++ b/smithery.yaml @@ -16,8 +16,8 @@ startCommand: secret: true commandFunction: | (config) => ({ - command: 'webclaw-mcp', - args: [], + command: 'npx', + args: ['-y', '@webclaw/mcp'], env: config.apiKey ? { WEBCLAW_API_KEY: config.apiKey } : {} }) exampleConfig: