mirror of
https://github.com/0xMassi/webclaw.git
synced 2026-07-02 04:08:08 +02:00
feat(server): add OSS webclaw-server REST API binary (closes #29)
Self-hosters hitting docs/self-hosting were promised three binaries
but the OSS Docker image only shipped two. webclaw-server lived in
the closed-source hosted-platform repo, which couldn't be opened. This
adds a minimal axum REST API in the OSS repo so self-hosting actually
works without pretending to ship the cloud platform.
Crate at crates/webclaw-server/. Stateless, no database, no job queue,
single binary. Endpoints: GET /health, POST /v1/{scrape, crawl, map,
batch, extract, summarize, diff, brand}. JSON shapes mirror
api.webclaw.io for the endpoints OSS can support, so swapping between
self-hosted and hosted is a base-URL change.
Auth: optional bearer token via WEBCLAW_API_KEY / --api-key. Comparison
is constant-time (subtle::ConstantTimeEq). Open mode (no key) is
allowed and binds 127.0.0.1 by default; the Docker image flips
WEBCLAW_HOST=0.0.0.0 so the container is reachable out of the box.
Hard caps to keep naive callers from OOMing the process: crawl capped
at 500 pages synchronously, batch capped at 100 URLs / 20 concurrent.
For unbounded crawls or anti-bot bypass the docs point users at the
hosted API.
Dockerfile + Dockerfile.ci updated to copy webclaw-server into
/usr/local/bin and EXPOSE 3000. Workspace version bumped to 0.4.0
(new public binary).
This commit is contained in:
parent
b4bfff120e
commit
2ba682adf3
20 changed files with 1116 additions and 11 deletions
81
crates/webclaw-server/src/routes/extract.rs
Normal file
81
crates/webclaw-server/src/routes/extract.rs
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
//! POST /v1/extract — LLM-powered structured extraction.
|
||||
//!
|
||||
//! Two modes:
|
||||
//! * `schema` — JSON Schema describing what to extract.
|
||||
//! * `prompt` — natural-language instructions.
|
||||
//!
|
||||
//! At least one must be provided. The provider chain is built per
|
||||
//! request from env (Ollama -> OpenAI -> Anthropic). Self-hosters
|
||||
//! get the same fallback behaviour as the CLI.
|
||||
|
||||
use axum::{Json, extract::State};
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Value, json};
|
||||
use webclaw_llm::{ProviderChain, extract::extract_json, extract::extract_with_prompt};
|
||||
|
||||
use crate::{error::ApiError, state::AppState};
|
||||
|
||||
#[derive(Debug, Deserialize, Default)]
|
||||
#[serde(default)]
|
||||
pub struct ExtractRequest {
|
||||
pub url: String,
|
||||
pub schema: Option<Value>,
|
||||
pub prompt: Option<String>,
|
||||
/// Optional override of the provider model name (e.g. `gpt-4o-mini`).
|
||||
pub model: Option<String>,
|
||||
}
|
||||
|
||||
pub async fn extract(
|
||||
State(state): State<AppState>,
|
||||
Json(req): Json<ExtractRequest>,
|
||||
) -> Result<Json<Value>, ApiError> {
|
||||
if req.url.trim().is_empty() {
|
||||
return Err(ApiError::bad_request("`url` is required"));
|
||||
}
|
||||
let has_schema = req.schema.is_some();
|
||||
let has_prompt = req
|
||||
.prompt
|
||||
.as_deref()
|
||||
.map(|p| !p.trim().is_empty())
|
||||
.unwrap_or(false);
|
||||
if !has_schema && !has_prompt {
|
||||
return Err(ApiError::bad_request(
|
||||
"either `schema` or `prompt` is required",
|
||||
));
|
||||
}
|
||||
|
||||
// Fetch + extract first so we feed the LLM clean markdown instead of
|
||||
// raw HTML. Cheaper tokens, better signal.
|
||||
let extraction = state.fetch().fetch_and_extract(&req.url).await?;
|
||||
let content = if extraction.content.markdown.trim().is_empty() {
|
||||
extraction.content.plain_text.clone()
|
||||
} else {
|
||||
extraction.content.markdown.clone()
|
||||
};
|
||||
if content.trim().is_empty() {
|
||||
return Err(ApiError::Extract(
|
||||
"no extractable content on page".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let chain = ProviderChain::default().await;
|
||||
if chain.is_empty() {
|
||||
return Err(ApiError::Llm(
|
||||
"no LLM providers configured (set OLLAMA_HOST, OPENAI_API_KEY, or ANTHROPIC_API_KEY)"
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let model = req.model.as_deref();
|
||||
let data = if let Some(schema) = req.schema.as_ref() {
|
||||
extract_json(&content, schema, &chain, model).await?
|
||||
} else {
|
||||
let prompt = req.prompt.as_deref().unwrap_or_default();
|
||||
extract_with_prompt(&content, prompt, &chain, model).await?
|
||||
};
|
||||
|
||||
Ok(Json(json!({
|
||||
"url": req.url,
|
||||
"data": data,
|
||||
})))
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue