apunkt/webclaw
1
0
Fork 0
mirror of https://github.com/0xMassi/webclaw.git synced 2026-07-24 07:31:01 +02:00
Code Issues Projects Releases Packages Wiki Activity

fix: validate self-host route URLs consistently

Browse source
This commit is contained in:
Valerio 2026-05-04 14:30:06 +02:00
parent eede2f6953
commit 1c9def2fde
8 changed files with 26 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

5
crates/webclaw-server/src/routes/crawl.rs
View file

@ -36,6 +36,7 @@ pub async fn crawl(
if req.url.trim().is_empty() {
return Err(ApiError::bad_request("`url` is required"));
}
let url = webclaw_fetch::url_security::validate_public_http_url(&req.url).await?;
let max_pages = req.max_pages.unwrap_or(50).min(HARD_MAX_PAGES);
let max_depth = req.max_depth.unwrap_or(3);
let concurrency = req.concurrency.unwrap_or(5).min(20);
@ -56,8 +57,8 @@ pub async fn crawl(
cancel_flag: None,
};
let crawler = Crawler::new(&req.url, config).map_err(ApiError::from)?;
let result = crawler.crawl(&req.url, None).await;
let crawler = Crawler::new(url.as_str(), config).map_err(ApiError::from)?;
let result = crawler.crawl(url.as_str(), None).await;
let pages: Vec<Value> = result
.pages