apunkt/webclaw
1
0
Fork 0
mirror of https://github.com/0xMassi/webclaw.git synced 2026-07-03 04:10:59 +02:00
Code Issues Projects Releases Packages Wiki Activity

fix(fetch): reject HTML verification pages served at .json reddit URL

Browse source
This commit is contained in:
Valerio 2026-04-23 15:06:35 +02:00
parent b413d702b2
commit 866fa88aa0
1 changed files with 9 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

12
crates/webclaw-fetch/src/client.rs
View file

@ -277,12 +277,18 @@ impl FetchClient {
// the result; here we just do the URL swap at the fetch layer. // the result; here we just do the URL swap at the fetch layer.
if crate::reddit::is_reddit_url(url) { if crate::reddit::is_reddit_url(url) {
let json_url = crate::reddit::json_url(url); let json_url = crate::reddit::json_url(url);
if let Ok(resp) = self.fetch(&json_url).await { if let Ok(resp) = self.fetch(&json_url).await
if resp.status == 200 && !resp.html.is_empty() { && resp.status == 200
{
// Reddit will serve an HTML verification page at the .json
// URL too when the IP is flagged. Only return if the body
// actually starts with a JSON payload.
let first = resp.html.trim_start().as_bytes().first().copied();
if matches!(first, Some(b'{') | Some(b'[')) {
return Ok(resp); return Ok(resp);
} }
} }
// If the .json fetch failed, fall through to the HTML path. // If the .json fetch failed or returned HTML, fall through.
} }
let resp = self.fetch(url).await?; let resp = self.fetch(url).await?;