diff --git a/CHANGELOG.md b/CHANGELOG.md index bc91010..b1b56f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ Format follows [Keep a Changelog](https://keepachangelog.com/). ## [Unreleased] +### Added +- **`@webclaw/mcp` — zero-install MCP launcher.** `npx -y @webclaw/mcp` downloads the prebuilt `webclaw-mcp` binary once, verifies it against the release `SHA256SUMS`, caches it, and runs it as an MCP stdio server. This is now the canonical way to add webclaw to an MCP client (`{"command": "npx", "args": ["-y", "@webclaw/mcp"]}`) — no Rust build — and it makes the server introspectable in MCP registries (Glama, Smithery, the MCP registry). `create-webclaw` stays as the auto-detect installer. + ## [0.6.15] - 2026-07-18 ### Added diff --git a/CLAUDE.md b/CLAUDE.md index 35d5f74..49e6659 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -190,7 +190,8 @@ Add to Claude Desktop config (`~/Library/Application Support/Claude/claude_deskt { "mcpServers": { "webclaw": { - "command": "/path/to/webclaw-mcp" + "command": "npx", + "args": ["-y", "@webclaw/mcp"] } } } diff --git a/README.md b/README.md index 10c8bce..d2b7f25 100644 --- a/README.md +++ b/README.md @@ -164,22 +164,21 @@ webclaw https://example.com/pricing --diff-with pricing-old.json webclaw ships with an MCP server for AI agents. -```bash -npx create-webclaw -``` - -Manual config: +Zero-install — point any MCP client at the npx launcher: ```json { "mcpServers": { "webclaw": { - "command": "~/.webclaw/webclaw-mcp" + "command": "npx", + "args": ["-y", "@webclaw/mcp"] } } } ``` +Or run `npx create-webclaw` to auto-detect your AI tools and write their configs for you. + Then ask your agent things like: ```text diff --git a/examples/README.md b/examples/README.md index 0967d74..e5acf09 100644 --- a/examples/README.md +++ b/examples/README.md @@ -273,14 +273,15 @@ webclaw https://example.com # Automatically detects and uses proxies.txt ## MCP Server (AI Agent Integration) ```bash -# Start the MCP server (stdio transport) -webclaw-mcp +# Run the MCP server (stdio transport) — no install needed +npx -y @webclaw/mcp # Configure in Claude Desktop (~/.config/claude/claude_desktop_config.json): # { # "mcpServers": { # "webclaw": { -# "command": "/path/to/webclaw-mcp", +# "command": "npx", +# "args": ["-y", "@webclaw/mcp"], # "env": { # "WEBCLAW_API_KEY": "wc_your_key" // optional, enables cloud fallback # } @@ -288,7 +289,7 @@ webclaw-mcp # } # } -# Available tools: scrape, crawl, map, batch, extract, summarize, diff, brand, research, search +# Available tools: scrape, search, crawl, map, batch, extract, summarize, diff, brand, research, lead, lead_batch, list_extractors, vertical_scrape ``` ## Real-World Recipes diff --git a/examples/mcp-web-scraping/README.md b/examples/mcp-web-scraping/README.md index 0663670..c5c7c43 100644 --- a/examples/mcp-web-scraping/README.md +++ b/examples/mcp-web-scraping/README.md @@ -16,7 +16,8 @@ The installer detects supported MCP clients and can write the config for you. { "mcpServers": { "webclaw": { - "command": "~/.webclaw/webclaw-mcp", + "command": "npx", + "args": ["-y", "@webclaw/mcp"], "env": { "WEBCLAW_API_KEY": "wc_your_key" } diff --git a/packages/create-webclaw/package.json b/packages/create-webclaw/package.json index 1c70511..4315971 100644 --- a/packages/create-webclaw/package.json +++ b/packages/create-webclaw/package.json @@ -1,7 +1,6 @@ { "name": "create-webclaw", "version": "0.1.6", - "mcpName": "io.github.0xMassi/webclaw", "description": "Set up webclaw MCP server for AI agents (Claude, Cursor, Windsurf, OpenCode, Codex, Antigravity)", "bin": { "create-webclaw": "./index.mjs" diff --git a/packages/create-webclaw/server.json b/packages/create-webclaw/server.json index 3216592..75d2bfe 100644 --- a/packages/create-webclaw/server.json +++ b/packages/create-webclaw/server.json @@ -3,12 +3,12 @@ "name": "io.github.0xMassi/webclaw", "title": "webclaw", "description": "Turn any URL into clean markdown/JSON for AI agents. Self-hostable Firecrawl alternative.", - "version": "0.1.6", + "version": "0.1.0", "packages": [ { "registryType": "npm", - "identifier": "create-webclaw", - "version": "0.1.6", + "identifier": "@webclaw/mcp", + "version": "0.1.0", "transport": { "type": "stdio" } diff --git a/packages/webclaw-mcp/README.md b/packages/webclaw-mcp/README.md new file mode 100644 index 0000000..f80d596 --- /dev/null +++ b/packages/webclaw-mcp/README.md @@ -0,0 +1,64 @@ +# @webclaw/mcp + +Zero-install launcher for the [webclaw](https://webclaw.io) MCP server — web extraction and anti-bot web access for AI agents, over the Model Context Protocol. + +`npx @webclaw/mcp` downloads a prebuilt `webclaw-mcp` binary once (from the pinned GitHub release), caches it, and runs it as an MCP stdio server. No Rust build, no global install. It runs on your machine; the hosted webclaw cloud is only used for tools that need it, and only when you set `WEBCLAW_API_KEY`. + +## Use in an MCP client + +Point any stdio MCP client at the npx command. Claude Desktop / Cursor / Windsurf / Antigravity (`mcpServers` JSON): + +```json +{ + "mcpServers": { + "webclaw": { + "command": "npx", + "args": ["-y", "@webclaw/mcp"] + } + } +} +``` + +Add a key for cloud-backed tools (anti-bot bypass, JS rendering, search, research): + +```json +{ + "mcpServers": { + "webclaw": { + "command": "npx", + "args": ["-y", "@webclaw/mcp"], + "env": { "WEBCLAW_API_KEY": "wc_your_key" } + } + } +} +``` + +Claude Code: + +```bash +claude mcp add webclaw -- npx -y @webclaw/mcp +``` + +## Tools + +scrape, search, crawl, map, batch, extract, summarize, diff, brand, research, lead, lead_batch, plus `list_extractors` / `vertical_scrape` for 30+ site-specific extractors. Local tools run with no API key; cloud tools require `WEBCLAW_API_KEY`. + +## Environment + +| Variable | Purpose | +|---|---| +| `WEBCLAW_API_KEY` | Enables cloud-backed tools (optional). | +| `WEBCLAW_MCP_BIN` | Absolute path to a `webclaw-mcp` binary; skips download. | +| `WEBCLAW_MCP_VERSION` | Release tag to install (default: the pinned release). | +| `WEBCLAW_MCP_CACHE` | Cache directory (default: `~/.cache/webclaw`). | + +The downloaded archive is verified against the release `SHA256SUMS` before use. + +## Alternatives + +- `npx create-webclaw` — interactive installer that auto-detects your AI tools and writes their configs for you. +- Prebuilt binaries, Homebrew, Docker, and `cargo install` — see the [main README](https://github.com/0xMassi/webclaw). + +## License + +AGPL-3.0-only diff --git a/packages/webclaw-mcp/package.json b/packages/webclaw-mcp/package.json new file mode 100644 index 0000000..6222845 --- /dev/null +++ b/packages/webclaw-mcp/package.json @@ -0,0 +1,39 @@ +{ + "name": "@webclaw/mcp", + "version": "0.1.0", + "description": "Zero-install launcher for the webclaw MCP server: web extraction and anti-bot access for AI agents. Runs `npx @webclaw/mcp`.", + "type": "module", + "bin": { + "webclaw-mcp": "./webclaw-mcp.mjs" + }, + "mcpName": "io.github.0xMassi/webclaw", + "files": [ + "webclaw-mcp.mjs", + "README.md" + ], + "engines": { + "node": ">=18" + }, + "license": "AGPL-3.0-only", + "author": "0xMassi", + "homepage": "https://webclaw.io", + "repository": { + "type": "git", + "url": "git+https://github.com/0xMassi/webclaw.git", + "directory": "packages/webclaw-mcp" + }, + "bugs": { + "url": "https://github.com/0xMassi/webclaw/issues" + }, + "keywords": [ + "mcp", + "model-context-protocol", + "webclaw", + "web-scraping", + "web-extraction", + "ai-agents", + "cloudflare", + "anti-bot", + "llm" + ] +} diff --git a/packages/webclaw-mcp/webclaw-mcp.mjs b/packages/webclaw-mcp/webclaw-mcp.mjs new file mode 100644 index 0000000..bedf9a6 --- /dev/null +++ b/packages/webclaw-mcp/webclaw-mcp.mjs @@ -0,0 +1,266 @@ +#!/usr/bin/env node +/** + * @webclaw/mcp — zero-install launcher for the webclaw MCP server. + * + * `npx -y @webclaw/mcp` resolves a prebuilt `webclaw-mcp` binary (downloaded + * once from the pinned GitHub release, then cached) and execs it as an MCP + * stdio server. This is what makes webclaw introspectable/installable in MCP + * clients and registries (Claude Desktop, Cursor, Glama, Smithery, ...) without + * a Rust build. + * + * HARD RULE: stdout carries the MCP JSON-RPC stream. Every diagnostic this + * launcher emits MUST go to stderr, or it corrupts the protocol. Never + * console.log here — only logErr(). + * + * Overrides (env): + * WEBCLAW_MCP_BIN absolute path to a webclaw-mcp binary; skip download + * WEBCLAW_MCP_VERSION release tag to install (default: pinned RELEASE_TAG) + * WEBCLAW_MCP_CACHE cache root (default: ~/.cache/webclaw) + * GITHUB_TOKEN only used to fetch SHA256SUMS if rate-limited + */ + +import { + existsSync, + mkdirSync, + createWriteStream, + readFileSync, + renameSync, + copyFileSync, + rmSync, + chmodSync, +} from "node:fs"; +import { homedir, platform, arch } from "node:os"; +import { join } from "node:path"; +import { spawn, execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import https from "node:https"; + +const REPO = "0xMassi/webclaw"; +// Release the wrapper installs. Bump this (and the package version) on each +// core release, or override at runtime with WEBCLAW_MCP_VERSION. +const RELEASE_TAG = process.env.WEBCLAW_MCP_VERSION || "v0.6.15"; + +const IS_WINDOWS = platform() === "win32"; +const BIN_NAME = IS_WINDOWS ? "webclaw-mcp.exe" : "webclaw-mcp"; +const CACHE_ROOT = + process.env.WEBCLAW_MCP_CACHE || join(homedir(), ".cache", "webclaw"); +const CACHE_DIR = join(CACHE_ROOT, RELEASE_TAG); +const CACHED_BIN = join(CACHE_DIR, BIN_NAME); + +function logErr(msg) { + process.stderr.write(`[@webclaw/mcp] ${msg}\n`); +} + +function target() { + const map = { + "darwin-arm64": "aarch64-apple-darwin", + "darwin-x64": "x86_64-apple-darwin", + "linux-x64": "x86_64-unknown-linux-gnu", + "linux-arm64": "aarch64-unknown-linux-gnu", + "win32-x64": "x86_64-pc-windows-msvc", + }; + return map[`${platform()}-${arch()}`] || null; +} + +// GET a URL to a Buffer, following redirects. Auth headers are dropped on +// redirect so a token never reaches the release CDN (its signed URLs reject it). +function getBuffer(url, headers = {}) { + return new Promise((resolve, reject) => { + https + .get( + url, + { headers: { "User-Agent": "@webclaw/mcp", ...headers } }, + (res) => { + if ( + res.statusCode >= 300 && + res.statusCode < 400 && + res.headers.location + ) { + res.resume(); + return getBuffer(res.headers.location).then(resolve, reject); + } + if (res.statusCode !== 200) { + res.resume(); + return reject(new Error(`HTTP ${res.statusCode} for ${url}`)); + } + const chunks = []; + res.on("data", (c) => chunks.push(c)); + res.on("end", () => resolve(Buffer.concat(chunks))); + res.on("error", reject); + }, + ) + .on("error", reject); + }); +} + +// Stream a URL to a file, following redirects. +function getFile(url, dest) { + return new Promise((resolve, reject) => { + https + .get(url, { headers: { "User-Agent": "@webclaw/mcp" } }, (res) => { + if ( + res.statusCode >= 300 && + res.statusCode < 400 && + res.headers.location + ) { + res.resume(); + return getFile(res.headers.location, dest).then(resolve, reject); + } + if (res.statusCode !== 200) { + res.resume(); + return reject(new Error(`HTTP ${res.statusCode} for ${url}`)); + } + const out = createWriteStream(dest); + res.pipe(out); + out.on("finish", () => out.close(() => resolve())); + out.on("error", reject); + }) + .on("error", reject); + }); +} + +function sha256(path) { + return createHash("sha256").update(readFileSync(path)).digest("hex"); +} + +function extract(archivePath, destDir) { + if (IS_WINDOWS) { + execFileSync( + "powershell", + [ + "-NoProfile", + "-Command", + `Expand-Archive -Path '${archivePath}' -DestinationPath '${destDir}' -Force`, + ], + { stdio: "ignore" }, + ); + } else { + execFileSync("tar", ["xzf", archivePath, "-C", destDir], { + stdio: "ignore", + }); + } +} + +async function ensureBinary() { + // 0. Explicit local binary (dev / offline / CI). + const override = process.env.WEBCLAW_MCP_BIN; + if (override) { + if (!existsSync(override)) { + throw new Error(`WEBCLAW_MCP_BIN points at a missing file: ${override}`); + } + return override; + } + + // 1. Cache hit for this release. + if (existsSync(CACHED_BIN)) return CACHED_BIN; + + // 2. Download, verify, extract, cache. + const tgt = target(); + if (!tgt) { + throw new Error( + `unsupported platform ${platform()}-${arch()} — install the webclaw-mcp binary manually ` + + `(https://github.com/${REPO}/releases) and set WEBCLAW_MCP_BIN`, + ); + } + const ext = IS_WINDOWS ? "zip" : "tar.gz"; + const assetName = `webclaw-${RELEASE_TAG}-${tgt}.${ext}`; + const base = `https://github.com/${REPO}/releases/download/${RELEASE_TAG}`; + + mkdirSync(CACHE_DIR, { recursive: true }); + const archivePath = join(CACHE_DIR, assetName); + const tmpPath = `${archivePath}.download`; + + logErr(`fetching ${assetName} (first run for ${RELEASE_TAG}) ...`); + await getFile(`${base}/${assetName}`, tmpPath); + + // Verify against the release SHA256SUMS. If the sums file can't be fetched we + // proceed with a warning rather than hard-failing an otherwise-good install. + try { + const apiHeaders = process.env.GITHUB_TOKEN + ? { Authorization: `Bearer ${process.env.GITHUB_TOKEN}` } + : {}; + const sums = (await getBuffer(`${base}/SHA256SUMS`, apiHeaders)).toString( + "utf8", + ); + const line = sums.split("\n").find((l) => l.trim().endsWith(assetName)); + const expected = line ? line.trim().split(/\s+/)[0] : null; + if (expected) { + const actual = sha256(tmpPath); + if (actual !== expected) { + rmSync(tmpPath, { force: true }); + throw new Error( + `checksum mismatch for ${assetName}: expected ${expected}, got ${actual}`, + ); + } + } else { + logErr( + `warning: ${assetName} not found in SHA256SUMS — skipping verification`, + ); + } + } catch (e) { + if (/checksum mismatch/.test(e.message)) throw e; + logErr(`warning: could not verify checksum (${e.message}) — proceeding`); + } + + renameSync(tmpPath, archivePath); + extract(archivePath, CACHE_DIR); + + // Archives hold a top-level `webclaw--/` dir with all binaries. + const extractedDir = join(CACHE_DIR, `webclaw-${RELEASE_TAG}-${tgt}`); + const extractedBin = join(extractedDir, BIN_NAME); + if (!existsSync(extractedBin)) { + throw new Error(`binary missing after extract: ${extractedBin}`); + } + copyFileSync(extractedBin, CACHED_BIN); + if (!IS_WINDOWS) chmodSync(CACHED_BIN, 0o755); + + // Drop the archive and the extra binaries; keep only the cached webclaw-mcp. + try { + rmSync(extractedDir, { recursive: true, force: true }); + rmSync(archivePath, { force: true }); + } catch { + /* non-fatal */ + } + + logErr(`installed ${BIN_NAME} for ${RELEASE_TAG} → ${CACHED_BIN}`); + return CACHED_BIN; +} + +async function main() { + let bin; + try { + bin = await ensureBinary(); + } catch (e) { + logErr(`error: ${e.message}`); + process.exit(1); + } + + // Hand off to the real server. stdio:inherit wires the MCP JSON-RPC stream + // straight through; env passes WEBCLAW_API_KEY and friends unchanged. + const child = spawn(bin, process.argv.slice(2), { + stdio: "inherit", + env: process.env, + }); + + child.on("error", (e) => { + logErr(`failed to start webclaw-mcp: ${e.message}`); + process.exit(1); + }); + child.on("exit", (code, signal) => { + if (signal) { + // Re-raise the signal so the parent's exit reflects it. + process.kill(process.pid, signal); + } else { + process.exit(code ?? 0); + } + }); + + // Forward termination signals to the child so clients can stop the server. + for (const sig of ["SIGINT", "SIGTERM", "SIGHUP"]) { + process.on(sig, () => { + if (!child.killed) child.kill(sig); + }); + } +} + +main(); diff --git a/smithery.yaml b/smithery.yaml index 00a69aa..0ed5a34 100644 --- a/smithery.yaml +++ b/smithery.yaml @@ -16,8 +16,8 @@ startCommand: secret: true commandFunction: | (config) => ({ - command: 'webclaw-mcp', - args: [], + command: 'npx', + args: ['-y', '@webclaw/mcp'], env: config.apiKey ? { WEBCLAW_API_KEY: config.apiKey } : {} }) exampleConfig: