From f76df1fdba1100e1a77d58a7a43de2e54ee9df8e Mon Sep 17 00:00:00 2001 From: Rodrigo Motta Date: Tue, 31 Mar 2026 21:52:17 -0300 Subject: [PATCH] feat: add webclaw-api rest server with axum --- Dockerfile | 7 +- crates/webclaw-api/Cargo.toml | 21 ++++++ crates/webclaw-api/src/main.rs | 127 +++++++++++++++++++++++++++++++++ 3 files changed, 153 insertions(+), 2 deletions(-) create mode 100644 crates/webclaw-api/Cargo.toml create mode 100644 crates/webclaw-api/src/main.rs diff --git a/Dockerfile b/Dockerfile index 2c66307..c0bdf33 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,6 +23,7 @@ COPY crates/webclaw-llm/Cargo.toml crates/webclaw-llm/Cargo.toml COPY crates/webclaw-pdf/Cargo.toml crates/webclaw-pdf/Cargo.toml COPY crates/webclaw-mcp/Cargo.toml crates/webclaw-mcp/Cargo.toml COPY crates/webclaw-cli/Cargo.toml crates/webclaw-cli/Cargo.toml +COPY crates/webclaw-api/Cargo.toml crates/webclaw-api/Cargo.toml # RUSTFLAGS (reqwest_unstable) — required by Impit's patched rustls COPY .cargo .cargo @@ -33,7 +34,8 @@ RUN mkdir -p crates/webclaw-core/src && echo "" > crates/webclaw-core/src/lib.rs && mkdir -p crates/webclaw-llm/src && echo "" > crates/webclaw-llm/src/lib.rs \ && mkdir -p crates/webclaw-pdf/src && echo "" > crates/webclaw-pdf/src/lib.rs \ && mkdir -p crates/webclaw-mcp/src && echo "fn main() {}" > crates/webclaw-mcp/src/main.rs \ - && mkdir -p crates/webclaw-cli/src && echo "fn main() {}" > crates/webclaw-cli/src/main.rs + && mkdir -p crates/webclaw-cli/src && echo "fn main() {}" > crates/webclaw-cli/src/main.rs \ + && mkdir -p crates/webclaw-api/src && echo "fn main() {}" > crates/webclaw-api/src/main.rs # Pre-build dependencies (this layer is cached until Cargo.toml/lock changes) RUN cargo build --release 2>/dev/null || true @@ -52,9 +54,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates \ && rm -rf /var/lib/apt/lists/* -# Copy both binaries +# Copy all binaries COPY --from=builder /build/target/release/webclaw /usr/local/bin/webclaw COPY --from=builder /build/target/release/webclaw-mcp /usr/local/bin/webclaw-mcp +COPY --from=builder /build/target/release/webclaw-api /usr/local/bin/webclaw-api # Default: run the CLI CMD ["webclaw"] diff --git a/crates/webclaw-api/Cargo.toml b/crates/webclaw-api/Cargo.toml new file mode 100644 index 0000000..bc383c1 --- /dev/null +++ b/crates/webclaw-api/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "webclaw-api" +description = "REST API server for webclaw web extraction toolkit" +version.workspace = true +edition.workspace = true +license.workspace = true + +[[bin]] +name = "webclaw-api" +path = "src/main.rs" + +[dependencies] +webclaw-core = { workspace = true } +webclaw-fetch = { workspace = true } +dotenvy = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +axum = "0.7" diff --git a/crates/webclaw-api/src/main.rs b/crates/webclaw-api/src/main.rs new file mode 100644 index 0000000..7b40fa6 --- /dev/null +++ b/crates/webclaw-api/src/main.rs @@ -0,0 +1,127 @@ +use axum::{ + extract::{Json, State}, + http::{HeaderMap, StatusCode}, + response::IntoResponse, + routing::post, + Router, +}; +use serde::{Deserialize, Serialize}; +use std::env; +use std::net::SocketAddr; +use webclaw_core::ExtractionOptions; +use webclaw_fetch::{FetchClient, FetchConfig}; + +#[derive(Deserialize)] +struct ScrapeRequest { + url: String, + // future proofing if user wants to expand features + #[serde(default = "default_format")] + format: String, +} + +fn default_format() -> String { + "json".to_string() +} + +#[derive(Serialize)] +struct ApiError { + error: String, +} + +#[tokio::main] +async fn main() { + dotenvy::dotenv().ok(); + + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .init(); + + // The API key is defined in the Dokploy Environment Settings as API_KEY + let api_key = env::var("API_KEY").unwrap_or_else(|_| { + tracing::warn!("API_KEY environment variable not set! API is running without authentication."); + "".to_string() + }); + + let app = Router::new() + .route("/api/scrape", post(scrape_handler)) + .with_state(api_key); + + let port: u16 = env::var("PORT") + .unwrap_or_else(|_| "3000".to_string()) + .parse() + .expect("PORT must be a number"); + + let addr = SocketAddr::from(([0, 0, 0, 0], port)); + tracing::info!("webclaw-api listening on {}", addr); + + let listener = tokio::net::TcpListener::bind(&addr).await.unwrap(); + axum::serve(listener, app).await.unwrap(); +} + +async fn scrape_handler( + headers: HeaderMap, + State(api_key): State, + Json(payload): Json, +) -> impl IntoResponse { + // Basic Authentication Check + if !api_key.is_empty() { + let auth_header = headers.get("authorization").and_then(|h| h.to_str().ok()); + let expected = format!("Bearer {}", api_key); + if auth_header != Some(&expected) { + let err = ApiError { + error: "Unauthorized: Invalid API_KEY".to_string(), + }; + return ( + StatusCode::UNAUTHORIZED, + Json(serde_json::to_value(err).unwrap()), + ) + .into_response(); + } + } + + // Initialize the fetch client with default webclaw configuration + let client = match FetchClient::new(FetchConfig::default()) { + Ok(c) => c, + Err(e) => { + let err = ApiError { + error: format!("Failed to initialize fetch client: {}", e), + }; + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(serde_json::to_value(err).unwrap()), + ) + .into_response(); + } + }; + + // Perform extraction + let options = ExtractionOptions::default(); + match client.fetch_and_extract_with_options(&payload.url, &options).await { + Ok(result) => { + // Return JSON containing metadata, content, and the structured_data we fixed + match serde_json::to_value(result) { + Ok(val) => (StatusCode::OK, Json(val)).into_response(), + Err(e) => { + let err = ApiError { + error: format!("Failed to serialize result: {}", e), + }; + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(serde_json::to_value(err).unwrap()), + ) + .into_response() + } + } + } + Err(e) => { + let err = ApiError { + error: format!("Extraction failed for {}: {}", payload.url, e), + }; + ( + StatusCode::BAD_REQUEST, + Json(serde_json::to_value(err).unwrap()), + ) + .into_response() + } + } +}