apunkt/webclaw
1
0
Fork 0
mirror of https://github.com/0xMassi/webclaw.git synced 2026-07-25 07:41:01 +02:00
Code Issues Projects Releases Packages Wiki Activity

refactor(create-webclaw): write the npx @webclaw/mcp config, drop binary download

Browse source 
Aligns with the industry standard: Firecrawl, the official MCP reference servers,
Exa, Brave, Apify, Bright Data, etc. all install their MCP via a single
`npx -y <pkg>` config line — none use a create-* scaffolder for the MCP itself
(create-* is a project-bootstrap convention). create-webclaw now auto-detects AI
tools and writes the canonical `npx @webclaw/mcp` config into each, byte-identical
to webclaw.io/docs and the MCP registries, so a scaffolded config can never
diverge from a hand-written one.

- buildMcpEntry + all writers emit {command:"npx", args:["-y","@webclaw/mcp"]}
  across every format (JSON, Continue array, OpenCode array, Codex TOML)
- remove the entire binary-download machinery (getTarget/download/extract/cargo
  fallback, ~250 lines) — the @webclaw/mcp launcher handles that now
- README: reflect the new behavior + show the one-block manual config
- version 0.1.6 -> 0.1.7

Verified: correct npx config (with/without API key) across all formats;
idempotent re-runs; zero binary-download code remains.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Valerio 2026-07-22 11:34:01 +02:00
parent 248da0abc4
commit d387669c13
3 changed files with 81 additions and 331 deletions
Download patch file Download diff file Expand all files Collapse all files

24
packages/create-webclaw/README.md
View file

@ -26,7 +26,7 @@
npx create-webclaw
```
That's it. Auto-detects your AI tools, downloads the MCP server, configures everything.
That's it. Auto-detects your AI tools and writes the `npx @webclaw/mcp` config into each — nothing to install.
Works with **Claude Desktop**, **Claude Code**, **Cursor**, **Windsurf**, **VS Code**, **OpenCode**, **Codex CLI**, and **Antigravity**.
@ -67,9 +67,25 @@ npx create-webclaw
```
1. Detects installed AI tools (Claude, Cursor, Windsurf, VS Code, OpenCode, Codex, Antigravity)
2. Downloads the `webclaw-mcp` binary for your platform (macOS arm64/x86, Linux x86/arm64)
3. Asks for your API key (optional — **works locally without one**)
4. Writes the MCP config for each detected tool
2. Asks for your API key (optional — **works locally without one**)
3. Writes the `npx @webclaw/mcp` config into each detected tool
The server itself runs via [`@webclaw/mcp`](https://www.npmjs.com/package/@webclaw/mcp) — `npx` fetches it on first launch and caches it. `create-webclaw` is just the convenience that writes that config for you.
### Prefer to configure by hand?
Add this one block to your client's `mcpServers` config — it's identical to what `create-webclaw` writes:
```json
{
"mcpServers": {
"webclaw": {
"command": "npx",
"args": ["-y", "@webclaw/mcp"]
}
}
}
```
## MCP Tools

386
packages/create-webclaw/index.mjs
View file

@ -1,29 +1,22 @@
#!/usr/bin/env node
import {
existsSync,
mkdirSync,
readFileSync,
writeFileSync,
copyFileSync,
rmSync,
} from "fs";
// create-webclaw — optional convenience installer for the webclaw MCP server.
//
// It auto-detects your AI tools (Claude Desktop, Claude Code, Cursor, Windsurf,
// OpenCode, Codex, Antigravity, ...) and writes the canonical `npx @webclaw/mcp`
// config into each. It does NOT download a binary: the runtime is always the
// `@webclaw/mcp` launcher, so a scaffolded config is byte-identical to what
// webclaw.io/docs and the MCP registries document. If you'd rather not run this,
// just add the one config block below by hand.
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
import { createInterface } from "readline";
import { homedir, platform, arch } from "os";
import { homedir, platform } from "os";
import { join, dirname } from "path";
import { execSync } from "child_process";
import { createWriteStream } from "fs";
import { chmod } from "fs/promises";
import https from "https";
import http from "http";
// ── Constants ──
const REPO = "0xMassi/webclaw";
const IS_WINDOWS = platform() === "win32";
const BINARY_NAME = IS_WINDOWS ? "webclaw-mcp.exe" : "webclaw-mcp";
const INSTALL_DIR = join(homedir(), ".webclaw");
const BINARY_PATH = join(INSTALL_DIR, BINARY_NAME);
const MCP_PACKAGE = "@webclaw/mcp";
const COLORS = {
reset: "\x1b[0m",
@ -82,7 +75,6 @@ const AI_TOOLS = [
id: "cursor",
name: "Cursor",
detect: () => {
// Check for .cursor directory in home or current project
return (
existsSync(join(homedir(), ".cursor")) ||
existsSync(join(process.cwd(), ".cursor"))
@ -173,77 +165,6 @@ function ask(question) {
});
}
function download(url, extraHeaders = {}) {
return new Promise((resolve, reject) => {
const client = url.startsWith("https") ? https : http;
const headers = { "User-Agent": "create-webclaw", ...extraHeaders };
client
.get(url, { headers }, (res) => {
// Follow redirects, dropping extra headers so an Authorization token
// never leaks to the release CDN (its signed URLs reject it anyway).
if (
res.statusCode >= 300 &&
res.statusCode < 400 &&
res.headers.location
) {
return download(res.headers.location).then(resolve).catch(reject);
}
if (res.statusCode !== 200) {
return reject(new Error(`HTTP ${res.statusCode}`));
}
const chunks = [];
res.on("data", (chunk) => chunks.push(chunk));
res.on("end", () => resolve(Buffer.concat(chunks)));
res.on("error", reject);
})
.on("error", reject);
});
}
async function downloadFile(url, dest) {
return new Promise((resolve, reject) => {
const client = url.startsWith("https") ? https : http;
client
.get(url, { headers: { "User-Agent": "create-webclaw" } }, (res) => {
if (
res.statusCode >= 300 &&
res.statusCode < 400 &&
res.headers.location
) {
return downloadFile(res.headers.location, dest)
.then(resolve)
.catch(reject);
}
if (res.statusCode !== 200) {
return reject(new Error(`HTTP ${res.statusCode}`));
}
const file = createWriteStream(dest);
res.pipe(file);
file.on("finish", () => {
file.close();
resolve();
});
file.on("error", reject);
})
.on("error", reject);
});
}
// Map the current platform to its Rust release target triple. Release assets
// are named `webclaw-<tag>-<target>.<ext>` (e.g.
// webclaw-v0.6.13-x86_64-unknown-linux-gnu.tar.gz), so the asset name is built
// from the release's tag_name at fetch time — it can't be hardcoded here.
function getTarget() {
const targets = {
"darwin-arm64": "aarch64-apple-darwin",
"darwin-x64": "x86_64-apple-darwin",
"linux-x64": "x86_64-unknown-linux-gnu",
"linux-arm64": "aarch64-unknown-linux-gnu",
"win32-x64": "x86_64-pc-windows-msvc",
};
return targets[`${platform()}-${arch()}`] || null;
}
function readJsonFile(path) {
try {
return JSON.parse(readFileSync(path, "utf-8"));
@ -258,43 +179,20 @@ function writeJsonFile(path, data) {
writeFileSync(path, JSON.stringify(data, null, 2) + "\n");
}
// The single canonical way to run the webclaw MCP server: the npx launcher.
// `@webclaw/mcp` fetches the prebuilt binary on first run and speaks MCP over
// stdio, so this config matches webclaw.io/docs and the MCP registries exactly.
function buildMcpEntry(apiKey) {
const entry = {
command: BINARY_PATH,
};
if (apiKey) {
entry.env = { WEBCLAW_API_KEY: apiKey };
}
const entry = { command: "npx", args: ["-y", MCP_PACKAGE] };
if (apiKey) entry.env = { WEBCLAW_API_KEY: apiKey };
return entry;
}
const MANUAL_CONFIG = `{ "mcpServers": { "webclaw": { "command": "npx", "args": ["-y", "${MCP_PACKAGE}"] } } }`;
// ── MCP Config Writers ──
function addToClaudeDesktop(configPath, apiKey) {
const config = readJsonFile(configPath);
if (!config.mcpServers) config.mcpServers = {};
config.mcpServers.webclaw = buildMcpEntry(apiKey);
writeJsonFile(configPath, config);
}
function addToClaudeCode(configPath, apiKey) {
const config = readJsonFile(configPath);
if (!config.mcpServers) config.mcpServers = {};
config.mcpServers.webclaw = buildMcpEntry(apiKey);
writeJsonFile(configPath, config);
}
function addToCursor(configPath, apiKey) {
const config = readJsonFile(configPath);
if (!config.mcpServers) config.mcpServers = {};
config.mcpServers.webclaw = {
command: BINARY_PATH,
...(apiKey ? { env: { WEBCLAW_API_KEY: apiKey } } : {}),
};
writeJsonFile(configPath, config);
}
function addToWindsurf(configPath, apiKey) {
function addToMcpServers(configPath, apiKey) {
const config = readJsonFile(configPath);
if (!config.mcpServers) config.mcpServers = {};
config.mcpServers.webclaw = buildMcpEntry(apiKey);
@ -304,13 +202,9 @@ function addToWindsurf(configPath, apiKey) {
function addToVSCodeContinue(configPath, apiKey) {
const config = readJsonFile(configPath);
if (!config.mcpServers) config.mcpServers = [];
// Continue uses array format
// Continue uses array format.
const existing = config.mcpServers.findIndex?.((s) => s.name === "webclaw");
const entry = {
name: "webclaw",
command: BINARY_PATH,
...(apiKey ? { env: { WEBCLAW_API_KEY: apiKey } } : {}),
};
const entry = { name: "webclaw", ...buildMcpEntry(apiKey) };
if (existing >= 0) {
config.mcpServers[existing] = entry;
} else if (Array.isArray(config.mcpServers)) {
@ -324,7 +218,7 @@ function addToOpenCode(configPath, apiKey) {
if (!config.mcp) config.mcp = {};
config.mcp.webclaw = {
type: "local",
command: [BINARY_PATH],
command: ["npx", "-y", MCP_PACKAGE],
enabled: true,
};
if (apiKey) {
@ -333,15 +227,8 @@ function addToOpenCode(configPath, apiKey) {
writeJsonFile(configPath, config);
}
function addToAntigravity(configPath, apiKey) {
const config = readJsonFile(configPath);
if (!config.mcpServers) config.mcpServers = {};
config.mcpServers.webclaw = buildMcpEntry(apiKey);
writeJsonFile(configPath, config);
}
function addToCodex(configPath, apiKey) {
// Codex uses TOML format, not JSON. Append MCP server config section.
// Codex uses TOML, not JSON. Replace any existing webclaw section.
const dir = dirname(configPath);
if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
@ -349,31 +236,28 @@ function addToCodex(configPath, apiKey) {
try {
existing = readFileSync(configPath, "utf-8");
} catch {
// File doesn't exist yet
// File doesn't exist yet.
}
// Remove any existing webclaw MCP section
existing = existing.replace(
/\n?\[mcp_servers\.webclaw\][^\[]*(?=\[|$)/gs,
"",
);
let section = `\n[mcp_servers.webclaw]\ncommand = "${BINARY_PATH}"\nargs = []\nenabled = true\n`;
let section = `\n[mcp_servers.webclaw]\ncommand = "npx"\nargs = ["-y", "${MCP_PACKAGE}"]\nenabled = true\n`;
if (apiKey) {
section += `env = { WEBCLAW_API_KEY = "${apiKey}" }\n`;
}
writeFileSync(configPath, existing.trimEnd() + "\n" + section);
}
const CONFIG_WRITERS = {
"claude-desktop": addToClaudeDesktop,
"claude-code": addToClaudeCode,
cursor: addToCursor,
windsurf: addToWindsurf,
"claude-desktop": addToMcpServers,
"claude-code": addToMcpServers,
cursor: addToMcpServers,
windsurf: addToMcpServers,
"vscode-continue": addToVSCodeContinue,
opencode: addToOpenCode,
antigravity: addToAntigravity,
antigravity: addToMcpServers,
codex: addToCodex,
};
@ -391,7 +275,7 @@ async function main() {
console.log(c("bold", " └─────────────────────────────────────┘"));
console.log();
// 1. Detect installed AI tools
// 1. Detect installed AI tools.
console.log(c("bold", " Detecting AI tools..."));
console.log();
@ -406,20 +290,10 @@ async function main() {
if (detected.length === 0) {
console.log(c("yellow", " No supported AI tools detected."));
console.log();
console.log(c("dim", " Supported tools:"));
for (const tool of AI_TOOLS) {
console.log(c("dim", ` • ${tool.name}`));
}
console.log(c("dim", " Add this to your MCP client config by hand:"));
console.log(c("cyan", ` ${MANUAL_CONFIG}`));
console.log();
console.log(
c("dim", " Install one of these tools and run this command again."),
);
console.log(c("dim", " Or use --manual to configure manually."));
console.log();
if (process.argv.includes("--manual")) {
// Continue anyway for manual setup
} else {
if (!process.argv.includes("--manual")) {
process.exit(0);
}
}
@ -429,10 +303,12 @@ async function main() {
}
console.log();
// 2. Ask for API key
console.log(c("dim", " An API key enables cloud features."));
// 2. Ask for an optional API key.
console.log(
c("dim", " Without one, webclaw runs locally (free, no account needed)."),
c("dim", " An API key unlocks the cloud tools (bot bypass, JS rendering,"),
);
console.log(
c("dim", " search, research, leads). Without one, webclaw runs locally."),
);
console.log();
@ -442,153 +318,15 @@ async function main() {
);
console.log();
// 3. Download binary
console.log(c("bold", " Downloading webclaw-mcp..."));
const target = getTarget();
if (!target) {
console.log(c("red", ` Unsupported platform: ${platform()}-${arch()}`));
console.log(
c(
"dim",
" Build from source: cargo install --git https://github.com/0xMassi/webclaw webclaw-mcp",
),
);
process.exit(1);
}
if (!existsSync(INSTALL_DIR)) {
mkdirSync(INSTALL_DIR, { recursive: true });
}
let downloaded = false;
let prebuiltError = null;
try {
// Resolve the latest release. Its tag_name drives the asset name. An
// unauthenticated GitHub API call is rate-limited to 60/hour per IP, so
// honour GITHUB_TOKEN when set — but only on this api.github.com request,
// never on the asset download (which redirects to a CDN).
const apiHeaders = process.env.GITHUB_TOKEN
? { Authorization: `Bearer ${process.env.GITHUB_TOKEN}` }
: {};
const release = JSON.parse(
(
await download(
`https://api.github.com/repos/${REPO}/releases/latest`,
apiHeaders,
)
).toString(),
);
const version = release.tag_name; // e.g. "v0.6.13"
const ext = IS_WINDOWS ? "zip" : "tar.gz";
const assetName = `webclaw-${version}-${target}.${ext}`;
const asset = release.assets?.find((a) => a.name === assetName);
if (!asset) {
throw new Error(`asset ${assetName} not found in release ${version}`);
}
const archivePath = join(INSTALL_DIR, assetName);
await downloadFile(asset.browser_download_url, archivePath);
// Each archive holds a top-level `webclaw-<version>-<target>/` directory
// containing webclaw, webclaw-mcp, webclaw-server, and docs.
if (ext === "tar.gz") {
execSync(`tar xzf "${archivePath}" -C "${INSTALL_DIR}"`, {
stdio: "ignore",
});
} else if (IS_WINDOWS) {
// Windows ships no `unzip`; Expand-Archive comes with PowerShell 5+.
execSync(
`powershell -NoProfile -Command "Expand-Archive -Path '${archivePath}' -DestinationPath '${INSTALL_DIR}' -Force"`,
{ stdio: "ignore" },
);
} else {
execSync(`unzip -o "${archivePath}" -d "${INSTALL_DIR}"`, {
stdio: "ignore",
});
}
// Lift webclaw-mcp out of the extracted directory to BINARY_PATH, then
// drop the rest (the other two binaries + docs).
const extractedDir = join(INSTALL_DIR, `webclaw-${version}-${target}`);
const extractedBin = join(extractedDir, BINARY_NAME);
if (!existsSync(extractedBin)) {
throw new Error(`binary missing after extract: ${extractedBin}`);
}
copyFileSync(extractedBin, BINARY_PATH);
if (!IS_WINDOWS) await chmod(BINARY_PATH, 0o755);
try {
rmSync(extractedDir, { recursive: true, force: true });
rmSync(archivePath, { force: true });
} catch {}
console.log(c("green", ` ✓ Installed to ${BINARY_PATH}`));
downloaded = true;
} catch (e) {
prebuiltError = e;
}
if (!downloaded) {
// Surface why the prebuilt path failed instead of hiding it — a 403 here
// is almost always a GitHub API rate limit, which Rust can't fix.
if (prebuiltError) {
const m = prebuiltError.message || String(prebuiltError);
if (m.includes("403") || /rate limit/i.test(m)) {
console.log(
c(
"yellow",
" GitHub API rate limit hit. Retry in a few minutes, or set GITHUB_TOKEN.",
),
);
} else {
console.log(c("yellow", ` Prebuilt binary unavailable (${m}).`));
}
}
// Fall back to building from source.
console.log(c("yellow", " Trying cargo install..."));
try {
execSync(
`cargo install --git https://github.com/${REPO} webclaw-mcp --root "${INSTALL_DIR}"`,
{ stdio: "inherit" },
);
// cargo install puts the binary in INSTALL_DIR/bin/
const cargoPath = join(INSTALL_DIR, "bin", BINARY_NAME);
if (existsSync(cargoPath)) {
copyFileSync(cargoPath, BINARY_PATH);
console.log(c("green", ` ✓ Built and installed to ${BINARY_PATH}`));
downloaded = true;
}
} catch {
console.log(
c("red", " Failed to install. Make sure Rust is installed:"),
);
console.log(
c(
"dim",
" curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh",
),
);
process.exit(1);
}
}
console.log();
// 4. Configure each detected tool
console.log(c("bold", " Configuring MCP servers..."));
// 3. Write the npx config into each detected tool.
console.log(c("bold", " Writing MCP config..."));
console.log();
for (const tool of detected) {
const configPath = tool.configPath();
if (!configPath) continue;
const writer = CONFIG_WRITERS[tool.id];
if (!writer) continue;
try {
writer(configPath, apiKey || null);
console.log(
@ -598,31 +336,27 @@ async function main() {
console.log(c("red", ` ✗ ${tool.name}: ${e.message}`));
}
}
console.log();
// 5. Verify
if (downloaded) {
try {
const version = execSync(`"${BINARY_PATH}" --version`, {
encoding: "utf-8",
}).trim();
console.log(c("green", ` ✓ ${version}`));
} catch {
console.log(c("green", ` ✓ webclaw-mcp installed`));
}
}
// 6. Summary
// 4. Summary.
console.log(c("bold", " Done! webclaw is configured."));
console.log();
console.log(c("bold", " Done! webclaw is ready."));
console.log(
c("dim", " Your client runs the server on demand via npx — nothing to"),
);
console.log(
c("dim", " install. First launch fetches @webclaw/mcp, then it's cached."),
);
console.log();
console.log(c("dim", " Your AI agent now has these tools:"));
console.log(c("dim", " • scrape — extract content from any URL"));
console.log(c("dim", " • crawl — recursively crawl a website"));
console.log(c("dim", " • search — web search + parallel scrape"));
console.log(c("dim", " • map — discover URLs from sitemaps"));
console.log(c("dim", " • batch — extract multiple URLs in parallel"));
console.log(
c("dim", " Tools: scrape, search, crawl, map, batch, extract, summarize,"),
);
console.log(
c(
"dim",
" diff, brand, research, lead, lead_batch, + 30 site extractors.",
),
);
console.log();
if (!apiKey) {
@ -630,7 +364,7 @@ async function main() {
console.log(
c(
"dim",
" Get an API key at https://webclaw.io/dashboard for cloud features.",
" Get a key at https://webclaw.io/dashboard for cloud features.",
),
);
console.log();

2
packages/create-webclaw/package.json
View file

@ -1,6 +1,6 @@
{
"name": "create-webclaw",
"version": "0.1.6",
"version": "0.1.7",
"description": "Set up webclaw MCP server for AI agents (Claude, Cursor, Windsurf, OpenCode, Codex, Antigravity)",
"bin": {
"create-webclaw": "./index.mjs"