diff --git a/CLAUDE.md b/CLAUDE.md index 959e49c..35d5f74 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -71,7 +71,7 @@ Three binaries: `webclaw` (CLI), `webclaw-mcp` (MCP server), `webclaw-server` (R ### MCP Server (`webclaw-mcp`) - Model Context Protocol server over stdio transport -- 12 tools: scrape, crawl, map, batch, extract, summarize, diff, brand, research, search, list_extractors, vertical_scrape. `search` is local-first via the caller's `SERPER_API_KEY` (falls back to the hosted API when unset); `research` uses the hosted deep-research API. The rest run locally. +- 14 tools: scrape, crawl, map, batch, extract, summarize, diff, brand, research, search, list_extractors, vertical_scrape, lead, lead_batch. `search` is local-first via the caller's `SERPER_API_KEY` (falls back to the hosted API when unset); `research`, `lead`, and `lead_batch` are hosted-only cloud proxies (require `WEBCLAW_API_KEY`) — `lead`/`lead_batch` call `/v1/lead` and the async `/v1/lead/batch`. The rest run locally. - Works with Claude Desktop, Claude Code, and any MCP client - Uses `rmcp` crate (official Rust MCP SDK) diff --git a/crates/webclaw-mcp/src/server.rs b/crates/webclaw-mcp/src/server.rs index 2554512..4059198 100644 --- a/crates/webclaw-mcp/src/server.rs +++ b/crates/webclaw-mcp/src/server.rs @@ -65,6 +65,10 @@ const LOCAL_FETCH_TIMEOUT: Duration = Duration::from_secs(30); /// Maximum poll iterations for research jobs (~10 minutes at 3s intervals). const RESEARCH_MAX_POLLS: u32 = 200; +/// Maximum poll iterations for lead batch jobs (~10 minutes at 3s intervals); +/// enough for a full 25-URL batch at ~20-40s per lead, 4 in parallel. +const LEAD_BATCH_MAX_POLLS: u32 = 200; + #[tool_router] impl WebclawMcp { pub async fn new() -> Self { @@ -668,6 +672,95 @@ impl WebclawMcp { )) } + /// Enrich a company into an outreach-ready lead: the founders and leadership + /// with their LinkedIn and X (recovered from open-web search), plus company + /// summary, socials, tech stack, pricing, and public emails. Requires + /// WEBCLAW_API_KEY. Flat 100 credits per lead. + #[tool] + async fn lead(&self, Parameters(params): Parameters) -> Result { + let cloud = self + .cloud + .as_ref() + .ok_or("Lead enrichment requires WEBCLAW_API_KEY. Get a key at https://webclaw.io")?; + + let mut body = json!({ "url": params.url }); + if let Some(no_cache) = params.no_cache { + body["no_cache"] = json!(no_cache); + } + + let resp = cloud.post("lead", body).await?; + Ok(serde_json::to_string_pretty(&resp).unwrap_or_default()) + } + + /// Enrich many companies at once (up to 25 URLs) into outreach-ready leads. + /// Async: starts a batch job, polls until it finishes, and returns the + /// per-URL results. Requires WEBCLAW_API_KEY. Billed 100 credits per + /// successfully enriched lead (unreachable/empty URLs are not charged). + #[tool] + async fn lead_batch( + &self, + Parameters(params): Parameters, + ) -> Result { + let cloud = self + .cloud + .as_ref() + .ok_or("Lead enrichment requires WEBCLAW_API_KEY. Get a key at https://webclaw.io")?; + + let url_count = params.urls.len(); + if url_count == 0 { + return Err("lead_batch requires at least one URL".to_string()); + } + + let mut body = json!({ "urls": params.urls }); + if let Some(no_cache) = params.no_cache { + body["no_cache"] = json!(no_cache); + } + + // Start the async batch job. + let start_resp = cloud.post("lead/batch", body).await?; + let job_id = start_resp + .get("id") + .and_then(|v| v.as_str()) + .ok_or("Lead batch API did not return a job ID")? + .to_string(); + + info!(job_id = %job_id, urls = url_count, "lead batch started, polling for completion"); + + // Poll until completed or failed. + for poll in 0..LEAD_BATCH_MAX_POLLS { + tokio::time::sleep(Duration::from_secs(3)).await; + + let status_resp = cloud.get(&format!("lead/batch/{job_id}")).await?; + let status = status_resp + .get("status") + .and_then(|v| v.as_str()) + .unwrap_or("unknown"); + + match status { + "completed" => { + return Ok(serde_json::to_string_pretty(&status_resp).unwrap_or_default()); + } + "failed" => { + let error = status_resp + .get("error") + .and_then(|v| v.as_str()) + .unwrap_or("unknown error"); + return Err(format!("Lead batch job failed: {error}")); + } + _ => { + if poll % 20 == 19 { + info!(job_id = %job_id, poll, "lead batch still in progress..."); + } + } + } + } + + Err(format!( + "Lead batch job {job_id} timed out after ~10 minutes of polling. \ + Check status manually via the webclaw API: GET /v1/lead/batch/{job_id}" + )) + } + /// Search the web for a query and return structured results. /// /// Resolves the backend in priority order: @@ -810,7 +903,7 @@ impl ServerHandler for WebclawMcp { .with_instructions(String::from( "Webclaw MCP server -- web content extraction for AI agents. \ Tools: scrape, crawl, map, batch, extract, summarize, diff, brand, research, search, \ - list_extractors, vertical_scrape.", + list_extractors, vertical_scrape, lead, lead_batch.", )) } } diff --git a/crates/webclaw-mcp/src/tools.rs b/crates/webclaw-mcp/src/tools.rs index c20a9e8..2efd9da 100644 --- a/crates/webclaw-mcp/src/tools.rs +++ b/crates/webclaw-mcp/src/tools.rs @@ -187,6 +187,24 @@ pub struct ResearchParams { pub topic: Option, } +#[derive(Debug, Deserialize, JsonSchema)] +pub struct LeadParams { + /// Company website URL to enrich into an outreach-ready lead + pub url: String, + /// Skip the cache and force a fresh enrichment (default: false) + #[serde(default, deserialize_with = "deser_opt_bool_or_str")] + pub no_cache: Option, +} + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct LeadBatchParams { + /// Company website URLs to enrich (up to 25 per batch) + pub urls: Vec, + /// Skip the cache and force a fresh enrichment (default: false) + #[serde(default, deserialize_with = "deser_opt_bool_or_str")] + pub no_cache: Option, +} + #[derive(Debug, Deserialize, JsonSchema)] pub struct SearchParams { /// Search query