apunkt/webclaw
1
0
Fork 0
mirror of https://github.com/0xMassi/webclaw.git synced 2026-06-17 23:55:13 +02:00
Code Issues Projects Releases Packages Wiki Activity

fix: harden fetch URL validation

Browse source
This commit is contained in:
Valerio 2026-05-04 11:50:57 +02:00
parent 23544f8fac
commit bdf81fe6bf
10 changed files with 284 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

10
crates/webclaw-server/src/routes/batch.rs
View file

@ -37,6 +37,14 @@ pub async fn batch(
req.urls.len()
)));
}
let mut safe_urls = Vec::with_capacity(req.urls.len());
for url in &req.urls {
safe_urls.push(
webclaw_fetch::url_security::validate_public_http_url(url)
.await?
.to_string(),
);
}
let concurrency = req.concurrency.unwrap_or(5).clamp(1, HARD_MAX_CONCURRENCY);
@ -47,7 +55,7 @@ pub async fn batch(
include_raw_html: false,
};
let url_refs: Vec<&str> = req.urls.iter().map(|s| s.as_str()).collect();
let url_refs: Vec<&str> = safe_urls.iter().map(|s| s.as_str()).collect();
let results = state
.fetch()
.fetch_and_extract_batch_with_options(&url_refs, concurrency, &options)

5
crates/webclaw-server/src/routes/scrape.rs
View file

@ -52,6 +52,7 @@ pub async fn scrape(
if req.url.trim().is_empty() {
return Err(ApiError::bad_request("`url` is required"));
}
let url = webclaw_fetch::url_security::validate_public_http_url(&req.url).await?;
let formats = req.formats.as_vec();
let options = ExtractionOptions {
@ -63,11 +64,11 @@ pub async fn scrape(
let extraction = state
.fetch()
.fetch_and_extract_with_options(&req.url, &options)
.fetch_and_extract_with_options(url.as_str(), &options)
.await?;
let mut body = json!({
"url": extraction.metadata.url.clone().unwrap_or_else(|| req.url.clone()),
"url": extraction.metadata.url.clone().unwrap_or_else(|| url.to_string()),
"metadata": extraction.metadata,
});
let obj = body.as_object_mut().expect("json::object");