mirror of
https://github.com/0xMassi/webclaw.git
synced 2026-04-25 00:06:21 +02:00
2ba682adf3
Self-hosters hitting docs/self-hosting were promised three binaries
but the OSS Docker image only shipped two. webclaw-server lived in
the closed-source hosted-platform repo, which couldn't be opened. This
adds a minimal axum REST API in the OSS repo so self-hosting actually
works without pretending to ship the cloud platform.
Crate at crates/webclaw-server/. Stateless, no database, no job queue,
single binary. Endpoints: GET /health, POST /v1/{scrape, crawl, map,
batch, extract, summarize, diff, brand}. JSON shapes mirror
api.webclaw.io for the endpoints OSS can support, so swapping between
self-hosted and hosted is a base-URL change.
Auth: optional bearer token via WEBCLAW_API_KEY / --api-key. Comparison
is constant-time (subtle::ConstantTimeEq). Open mode (no key) is
allowed and binds 127.0.0.1 by default; the Docker image flips
WEBCLAW_HOST=0.0.0.0 so the container is reachable out of the box.
Hard caps to keep naive callers from OOMing the process: crawl capped
at 500 pages synchronously, batch capped at 100 URLs / 20 concurrent.
For unbounded crawls or anti-bot bypass the docs point users at the
hosted API.
Dockerfile + Dockerfile.ci updated to copy webclaw-server into
/usr/local/bin and EXPOSE 3000. Workspace version bumped to 0.4.0
(new public binary).
29 lines
1.1 KiB
TOML
29 lines
1.1 KiB
TOML
[package]
|
|
name = "webclaw-server"
|
|
version.workspace = true
|
|
edition.workspace = true
|
|
license.workspace = true
|
|
repository.workspace = true
|
|
description = "Minimal REST API server for self-hosting webclaw extraction. Wraps the OSS extraction crates with HTTP endpoints. NOT the production hosted API at api.webclaw.io — this is a stateless, single-binary reference server for local + self-hosted deployments."
|
|
|
|
[[bin]]
|
|
name = "webclaw-server"
|
|
path = "src/main.rs"
|
|
|
|
[dependencies]
|
|
webclaw-core = { workspace = true }
|
|
webclaw-fetch = { workspace = true }
|
|
webclaw-llm = { workspace = true }
|
|
webclaw-pdf = { workspace = true }
|
|
|
|
axum = { version = "0.8", features = ["macros"] }
|
|
tokio = { workspace = true }
|
|
tower-http = { version = "0.6", features = ["trace", "cors"] }
|
|
clap = { workspace = true, features = ["derive", "env"] }
|
|
serde = { workspace = true }
|
|
serde_json = { workspace = true }
|
|
tracing = { workspace = true }
|
|
tracing-subscriber = { workspace = true, features = ["env-filter"] }
|
|
anyhow = "1"
|
|
thiserror = { workspace = true }
|
|
subtle = "2.6"
|