diff --git a/packages/create-webclaw/README.md b/packages/create-webclaw/README.md index 9f75986..ebeacbc 100644 --- a/packages/create-webclaw/README.md +++ b/packages/create-webclaw/README.md @@ -26,7 +26,7 @@ npx create-webclaw ``` -That's it. Auto-detects your AI tools, downloads the MCP server, configures everything. +That's it. Auto-detects your AI tools and writes the `npx @webclaw/mcp` config into each — nothing to install. Works with **Claude Desktop**, **Claude Code**, **Cursor**, **Windsurf**, **VS Code**, **OpenCode**, **Codex CLI**, and **Antigravity**. @@ -67,9 +67,25 @@ npx create-webclaw ``` 1. Detects installed AI tools (Claude, Cursor, Windsurf, VS Code, OpenCode, Codex, Antigravity) -2. Downloads the `webclaw-mcp` binary for your platform (macOS arm64/x86, Linux x86/arm64) -3. Asks for your API key (optional — **works locally without one**) -4. Writes the MCP config for each detected tool +2. Asks for your API key (optional — **works locally without one**) +3. Writes the `npx @webclaw/mcp` config into each detected tool + +The server itself runs via [`@webclaw/mcp`](https://www.npmjs.com/package/@webclaw/mcp) — `npx` fetches it on first launch and caches it. `create-webclaw` is just the convenience that writes that config for you. + +### Prefer to configure by hand? + +Add this one block to your client's `mcpServers` config — it's identical to what `create-webclaw` writes: + +```json +{ + "mcpServers": { + "webclaw": { + "command": "npx", + "args": ["-y", "@webclaw/mcp"] + } + } +} +``` ## MCP Tools diff --git a/packages/create-webclaw/index.mjs b/packages/create-webclaw/index.mjs index 2bb04a4..8dea1ee 100644 --- a/packages/create-webclaw/index.mjs +++ b/packages/create-webclaw/index.mjs @@ -1,29 +1,22 @@ #!/usr/bin/env node -import { - existsSync, - mkdirSync, - readFileSync, - writeFileSync, - copyFileSync, - rmSync, -} from "fs"; +// create-webclaw — optional convenience installer for the webclaw MCP server. +// +// It auto-detects your AI tools (Claude Desktop, Claude Code, Cursor, Windsurf, +// OpenCode, Codex, Antigravity, ...) and writes the canonical `npx @webclaw/mcp` +// config into each. It does NOT download a binary: the runtime is always the +// `@webclaw/mcp` launcher, so a scaffolded config is byte-identical to what +// webclaw.io/docs and the MCP registries document. If you'd rather not run this, +// just add the one config block below by hand. + +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"; import { createInterface } from "readline"; -import { homedir, platform, arch } from "os"; +import { homedir, platform } from "os"; import { join, dirname } from "path"; -import { execSync } from "child_process"; -import { createWriteStream } from "fs"; -import { chmod } from "fs/promises"; -import https from "https"; -import http from "http"; // ── Constants ── -const REPO = "0xMassi/webclaw"; -const IS_WINDOWS = platform() === "win32"; -const BINARY_NAME = IS_WINDOWS ? "webclaw-mcp.exe" : "webclaw-mcp"; -const INSTALL_DIR = join(homedir(), ".webclaw"); -const BINARY_PATH = join(INSTALL_DIR, BINARY_NAME); +const MCP_PACKAGE = "@webclaw/mcp"; const COLORS = { reset: "\x1b[0m", @@ -82,7 +75,6 @@ const AI_TOOLS = [ id: "cursor", name: "Cursor", detect: () => { - // Check for .cursor directory in home or current project return ( existsSync(join(homedir(), ".cursor")) || existsSync(join(process.cwd(), ".cursor")) @@ -173,77 +165,6 @@ function ask(question) { }); } -function download(url, extraHeaders = {}) { - return new Promise((resolve, reject) => { - const client = url.startsWith("https") ? https : http; - const headers = { "User-Agent": "create-webclaw", ...extraHeaders }; - client - .get(url, { headers }, (res) => { - // Follow redirects, dropping extra headers so an Authorization token - // never leaks to the release CDN (its signed URLs reject it anyway). - if ( - res.statusCode >= 300 && - res.statusCode < 400 && - res.headers.location - ) { - return download(res.headers.location).then(resolve).catch(reject); - } - if (res.statusCode !== 200) { - return reject(new Error(`HTTP ${res.statusCode}`)); - } - const chunks = []; - res.on("data", (chunk) => chunks.push(chunk)); - res.on("end", () => resolve(Buffer.concat(chunks))); - res.on("error", reject); - }) - .on("error", reject); - }); -} - -async function downloadFile(url, dest) { - return new Promise((resolve, reject) => { - const client = url.startsWith("https") ? https : http; - client - .get(url, { headers: { "User-Agent": "create-webclaw" } }, (res) => { - if ( - res.statusCode >= 300 && - res.statusCode < 400 && - res.headers.location - ) { - return downloadFile(res.headers.location, dest) - .then(resolve) - .catch(reject); - } - if (res.statusCode !== 200) { - return reject(new Error(`HTTP ${res.statusCode}`)); - } - const file = createWriteStream(dest); - res.pipe(file); - file.on("finish", () => { - file.close(); - resolve(); - }); - file.on("error", reject); - }) - .on("error", reject); - }); -} - -// Map the current platform to its Rust release target triple. Release assets -// are named `webclaw--.` (e.g. -// webclaw-v0.6.13-x86_64-unknown-linux-gnu.tar.gz), so the asset name is built -// from the release's tag_name at fetch time — it can't be hardcoded here. -function getTarget() { - const targets = { - "darwin-arm64": "aarch64-apple-darwin", - "darwin-x64": "x86_64-apple-darwin", - "linux-x64": "x86_64-unknown-linux-gnu", - "linux-arm64": "aarch64-unknown-linux-gnu", - "win32-x64": "x86_64-pc-windows-msvc", - }; - return targets[`${platform()}-${arch()}`] || null; -} - function readJsonFile(path) { try { return JSON.parse(readFileSync(path, "utf-8")); @@ -258,43 +179,20 @@ function writeJsonFile(path, data) { writeFileSync(path, JSON.stringify(data, null, 2) + "\n"); } +// The single canonical way to run the webclaw MCP server: the npx launcher. +// `@webclaw/mcp` fetches the prebuilt binary on first run and speaks MCP over +// stdio, so this config matches webclaw.io/docs and the MCP registries exactly. function buildMcpEntry(apiKey) { - const entry = { - command: BINARY_PATH, - }; - if (apiKey) { - entry.env = { WEBCLAW_API_KEY: apiKey }; - } + const entry = { command: "npx", args: ["-y", MCP_PACKAGE] }; + if (apiKey) entry.env = { WEBCLAW_API_KEY: apiKey }; return entry; } +const MANUAL_CONFIG = `{ "mcpServers": { "webclaw": { "command": "npx", "args": ["-y", "${MCP_PACKAGE}"] } } }`; + // ── MCP Config Writers ── -function addToClaudeDesktop(configPath, apiKey) { - const config = readJsonFile(configPath); - if (!config.mcpServers) config.mcpServers = {}; - config.mcpServers.webclaw = buildMcpEntry(apiKey); - writeJsonFile(configPath, config); -} - -function addToClaudeCode(configPath, apiKey) { - const config = readJsonFile(configPath); - if (!config.mcpServers) config.mcpServers = {}; - config.mcpServers.webclaw = buildMcpEntry(apiKey); - writeJsonFile(configPath, config); -} - -function addToCursor(configPath, apiKey) { - const config = readJsonFile(configPath); - if (!config.mcpServers) config.mcpServers = {}; - config.mcpServers.webclaw = { - command: BINARY_PATH, - ...(apiKey ? { env: { WEBCLAW_API_KEY: apiKey } } : {}), - }; - writeJsonFile(configPath, config); -} - -function addToWindsurf(configPath, apiKey) { +function addToMcpServers(configPath, apiKey) { const config = readJsonFile(configPath); if (!config.mcpServers) config.mcpServers = {}; config.mcpServers.webclaw = buildMcpEntry(apiKey); @@ -304,13 +202,9 @@ function addToWindsurf(configPath, apiKey) { function addToVSCodeContinue(configPath, apiKey) { const config = readJsonFile(configPath); if (!config.mcpServers) config.mcpServers = []; - // Continue uses array format + // Continue uses array format. const existing = config.mcpServers.findIndex?.((s) => s.name === "webclaw"); - const entry = { - name: "webclaw", - command: BINARY_PATH, - ...(apiKey ? { env: { WEBCLAW_API_KEY: apiKey } } : {}), - }; + const entry = { name: "webclaw", ...buildMcpEntry(apiKey) }; if (existing >= 0) { config.mcpServers[existing] = entry; } else if (Array.isArray(config.mcpServers)) { @@ -324,7 +218,7 @@ function addToOpenCode(configPath, apiKey) { if (!config.mcp) config.mcp = {}; config.mcp.webclaw = { type: "local", - command: [BINARY_PATH], + command: ["npx", "-y", MCP_PACKAGE], enabled: true, }; if (apiKey) { @@ -333,15 +227,8 @@ function addToOpenCode(configPath, apiKey) { writeJsonFile(configPath, config); } -function addToAntigravity(configPath, apiKey) { - const config = readJsonFile(configPath); - if (!config.mcpServers) config.mcpServers = {}; - config.mcpServers.webclaw = buildMcpEntry(apiKey); - writeJsonFile(configPath, config); -} - function addToCodex(configPath, apiKey) { - // Codex uses TOML format, not JSON. Append MCP server config section. + // Codex uses TOML, not JSON. Replace any existing webclaw section. const dir = dirname(configPath); if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); @@ -349,31 +236,28 @@ function addToCodex(configPath, apiKey) { try { existing = readFileSync(configPath, "utf-8"); } catch { - // File doesn't exist yet + // File doesn't exist yet. } - - // Remove any existing webclaw MCP section existing = existing.replace( /\n?\[mcp_servers\.webclaw\][^\[]*(?=\[|$)/gs, "", ); - let section = `\n[mcp_servers.webclaw]\ncommand = "${BINARY_PATH}"\nargs = []\nenabled = true\n`; + let section = `\n[mcp_servers.webclaw]\ncommand = "npx"\nargs = ["-y", "${MCP_PACKAGE}"]\nenabled = true\n`; if (apiKey) { section += `env = { WEBCLAW_API_KEY = "${apiKey}" }\n`; } - writeFileSync(configPath, existing.trimEnd() + "\n" + section); } const CONFIG_WRITERS = { - "claude-desktop": addToClaudeDesktop, - "claude-code": addToClaudeCode, - cursor: addToCursor, - windsurf: addToWindsurf, + "claude-desktop": addToMcpServers, + "claude-code": addToMcpServers, + cursor: addToMcpServers, + windsurf: addToMcpServers, "vscode-continue": addToVSCodeContinue, opencode: addToOpenCode, - antigravity: addToAntigravity, + antigravity: addToMcpServers, codex: addToCodex, }; @@ -391,7 +275,7 @@ async function main() { console.log(c("bold", " └─────────────────────────────────────┘")); console.log(); - // 1. Detect installed AI tools + // 1. Detect installed AI tools. console.log(c("bold", " Detecting AI tools...")); console.log(); @@ -406,20 +290,10 @@ async function main() { if (detected.length === 0) { console.log(c("yellow", " No supported AI tools detected.")); console.log(); - console.log(c("dim", " Supported tools:")); - for (const tool of AI_TOOLS) { - console.log(c("dim", ` • ${tool.name}`)); - } + console.log(c("dim", " Add this to your MCP client config by hand:")); + console.log(c("cyan", ` ${MANUAL_CONFIG}`)); console.log(); - console.log( - c("dim", " Install one of these tools and run this command again."), - ); - console.log(c("dim", " Or use --manual to configure manually.")); - console.log(); - - if (process.argv.includes("--manual")) { - // Continue anyway for manual setup - } else { + if (!process.argv.includes("--manual")) { process.exit(0); } } @@ -429,10 +303,12 @@ async function main() { } console.log(); - // 2. Ask for API key - console.log(c("dim", " An API key enables cloud features.")); + // 2. Ask for an optional API key. console.log( - c("dim", " Without one, webclaw runs locally (free, no account needed)."), + c("dim", " An API key unlocks the cloud tools (bot bypass, JS rendering,"), + ); + console.log( + c("dim", " search, research, leads). Without one, webclaw runs locally."), ); console.log(); @@ -442,153 +318,15 @@ async function main() { ); console.log(); - // 3. Download binary - console.log(c("bold", " Downloading webclaw-mcp...")); - - const target = getTarget(); - if (!target) { - console.log(c("red", ` Unsupported platform: ${platform()}-${arch()}`)); - console.log( - c( - "dim", - " Build from source: cargo install --git https://github.com/0xMassi/webclaw webclaw-mcp", - ), - ); - process.exit(1); - } - - if (!existsSync(INSTALL_DIR)) { - mkdirSync(INSTALL_DIR, { recursive: true }); - } - - let downloaded = false; - let prebuiltError = null; - - try { - // Resolve the latest release. Its tag_name drives the asset name. An - // unauthenticated GitHub API call is rate-limited to 60/hour per IP, so - // honour GITHUB_TOKEN when set — but only on this api.github.com request, - // never on the asset download (which redirects to a CDN). - const apiHeaders = process.env.GITHUB_TOKEN - ? { Authorization: `Bearer ${process.env.GITHUB_TOKEN}` } - : {}; - const release = JSON.parse( - ( - await download( - `https://api.github.com/repos/${REPO}/releases/latest`, - apiHeaders, - ) - ).toString(), - ); - - const version = release.tag_name; // e.g. "v0.6.13" - const ext = IS_WINDOWS ? "zip" : "tar.gz"; - const assetName = `webclaw-${version}-${target}.${ext}`; - const asset = release.assets?.find((a) => a.name === assetName); - if (!asset) { - throw new Error(`asset ${assetName} not found in release ${version}`); - } - - const archivePath = join(INSTALL_DIR, assetName); - await downloadFile(asset.browser_download_url, archivePath); - - // Each archive holds a top-level `webclaw--/` directory - // containing webclaw, webclaw-mcp, webclaw-server, and docs. - if (ext === "tar.gz") { - execSync(`tar xzf "${archivePath}" -C "${INSTALL_DIR}"`, { - stdio: "ignore", - }); - } else if (IS_WINDOWS) { - // Windows ships no `unzip`; Expand-Archive comes with PowerShell 5+. - execSync( - `powershell -NoProfile -Command "Expand-Archive -Path '${archivePath}' -DestinationPath '${INSTALL_DIR}' -Force"`, - { stdio: "ignore" }, - ); - } else { - execSync(`unzip -o "${archivePath}" -d "${INSTALL_DIR}"`, { - stdio: "ignore", - }); - } - - // Lift webclaw-mcp out of the extracted directory to BINARY_PATH, then - // drop the rest (the other two binaries + docs). - const extractedDir = join(INSTALL_DIR, `webclaw-${version}-${target}`); - const extractedBin = join(extractedDir, BINARY_NAME); - if (!existsSync(extractedBin)) { - throw new Error(`binary missing after extract: ${extractedBin}`); - } - copyFileSync(extractedBin, BINARY_PATH); - if (!IS_WINDOWS) await chmod(BINARY_PATH, 0o755); - - try { - rmSync(extractedDir, { recursive: true, force: true }); - rmSync(archivePath, { force: true }); - } catch {} - - console.log(c("green", ` ✓ Installed to ${BINARY_PATH}`)); - downloaded = true; - } catch (e) { - prebuiltError = e; - } - - if (!downloaded) { - // Surface why the prebuilt path failed instead of hiding it — a 403 here - // is almost always a GitHub API rate limit, which Rust can't fix. - if (prebuiltError) { - const m = prebuiltError.message || String(prebuiltError); - if (m.includes("403") || /rate limit/i.test(m)) { - console.log( - c( - "yellow", - " GitHub API rate limit hit. Retry in a few minutes, or set GITHUB_TOKEN.", - ), - ); - } else { - console.log(c("yellow", ` Prebuilt binary unavailable (${m}).`)); - } - } - - // Fall back to building from source. - console.log(c("yellow", " Trying cargo install...")); - try { - execSync( - `cargo install --git https://github.com/${REPO} webclaw-mcp --root "${INSTALL_DIR}"`, - { stdio: "inherit" }, - ); - // cargo install puts the binary in INSTALL_DIR/bin/ - const cargoPath = join(INSTALL_DIR, "bin", BINARY_NAME); - if (existsSync(cargoPath)) { - copyFileSync(cargoPath, BINARY_PATH); - console.log(c("green", ` ✓ Built and installed to ${BINARY_PATH}`)); - downloaded = true; - } - } catch { - console.log( - c("red", " Failed to install. Make sure Rust is installed:"), - ); - console.log( - c( - "dim", - " curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh", - ), - ); - process.exit(1); - } - } - - console.log(); - - // 4. Configure each detected tool - console.log(c("bold", " Configuring MCP servers...")); + // 3. Write the npx config into each detected tool. + console.log(c("bold", " Writing MCP config...")); console.log(); for (const tool of detected) { const configPath = tool.configPath(); if (!configPath) continue; - const writer = CONFIG_WRITERS[tool.id]; if (!writer) continue; - try { writer(configPath, apiKey || null); console.log( @@ -598,31 +336,27 @@ async function main() { console.log(c("red", ` ✗ ${tool.name}: ${e.message}`)); } } - console.log(); - // 5. Verify - if (downloaded) { - try { - const version = execSync(`"${BINARY_PATH}" --version`, { - encoding: "utf-8", - }).trim(); - console.log(c("green", ` ✓ ${version}`)); - } catch { - console.log(c("green", ` ✓ webclaw-mcp installed`)); - } - } - - // 6. Summary + // 4. Summary. + console.log(c("bold", " Done! webclaw is configured.")); console.log(); - console.log(c("bold", " Done! webclaw is ready.")); + console.log( + c("dim", " Your client runs the server on demand via npx — nothing to"), + ); + console.log( + c("dim", " install. First launch fetches @webclaw/mcp, then it's cached."), + ); console.log(); - console.log(c("dim", " Your AI agent now has these tools:")); - console.log(c("dim", " • scrape — extract content from any URL")); - console.log(c("dim", " • crawl — recursively crawl a website")); - console.log(c("dim", " • search — web search + parallel scrape")); - console.log(c("dim", " • map — discover URLs from sitemaps")); - console.log(c("dim", " • batch — extract multiple URLs in parallel")); + console.log( + c("dim", " Tools: scrape, search, crawl, map, batch, extract, summarize,"), + ); + console.log( + c( + "dim", + " diff, brand, research, lead, lead_batch, + 30 site extractors.", + ), + ); console.log(); if (!apiKey) { @@ -630,7 +364,7 @@ async function main() { console.log( c( "dim", - " Get an API key at https://webclaw.io/dashboard for cloud features.", + " Get a key at https://webclaw.io/dashboard for cloud features.", ), ); console.log(); diff --git a/packages/create-webclaw/package.json b/packages/create-webclaw/package.json index 4315971..79cc010 100644 --- a/packages/create-webclaw/package.json +++ b/packages/create-webclaw/package.json @@ -1,6 +1,6 @@ { "name": "create-webclaw", - "version": "0.1.6", + "version": "0.1.7", "description": "Set up webclaw MCP server for AI agents (Claude, Cursor, Windsurf, OpenCode, Codex, Antigravity)", "bin": { "create-webclaw": "./index.mjs"