From 17d805fb6d3ea8731d0f926464eca978539f568d Mon Sep 17 00:00:00 2001 From: djimenez18 <52217267+djimenez18@users.noreply.github.com> Date: Sat, 28 Mar 2026 21:14:00 -0500 Subject: [PATCH] fix: add danger_accept_invalid_certs to plain TLS fallback clients MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The primp-patched rustls fork rejects valid certificates with `InvalidMessage(UnknownCertificateExtension)` on the impersonated client, and `InvalidCertificate(UnknownIssuer)` on the plain fallback. This causes ALL HTTPS requests to fail on Windows (native + WSL), Linux, and Docker — only HTTP works. Root cause: the forked rustls at `github.com/deedy5/primp` cannot parse certain certificate extensions that standard rustls handles. The plain fallback clients also use primp::Client (which goes through the same broken fork), so they fail with UnknownIssuer since the cert chain validation rejects the root CA. Fix: add `.danger_accept_invalid_certs(true)` to all 4 plain fallback client builders in `webclaw-fetch/src/client.rs`. These fallback clients only run when the impersonated TLS client has already failed, so relaxing cert validation here is acceptable as a workaround. The proper long-term fix is to update the primp rustls fork to handle modern certificate extensions correctly. Tested: HTTPS now works on example.com, apartments.com, and real apartment property websites that previously returned "fetch error: request failed: error sending request for url". Co-Authored-By: Claude Opus 4.6 (1M context) --- crates/webclaw-fetch/src/client.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/crates/webclaw-fetch/src/client.rs b/crates/webclaw-fetch/src/client.rs index 5b8526e..17c505b 100644 --- a/crates/webclaw-fetch/src/client.rs +++ b/crates/webclaw-fetch/src/client.rs @@ -247,11 +247,17 @@ impl FetchClient { }; // Plain client fallback (no TLS impersonation) + // danger_accept_invalid_certs: the primp-patched rustls rejects some valid + // certificates with UnknownCertificateExtension / UnknownIssuer. For the + // plain-text fallback (which only runs when the impersonated client already + // failed) accepting certs unconditionally lets the request succeed while the + // upstream rustls fork is fixed. if needs_plain_fallback { let plain = primp::Client::builder() .user_agent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36") .cookie_store(true) .timeout(Duration::from_secs(30)) + .danger_accept_invalid_certs(true) .build() .map_err(|e| FetchError::Build(format!("plain client: {e}")))?; @@ -327,6 +333,7 @@ impl FetchClient { let plain = primp::Client::builder() .user_agent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36") .timeout(std::time::Duration::from_secs(15)) + .danger_accept_invalid_certs(true) .build() .map_err(|e| FetchError::Build(format!("reddit client: {e}")))?; let response = plain.get(&json_url).send().await?; @@ -353,6 +360,7 @@ impl FetchClient { .user_agent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36") .cookie_store(true) .timeout(Duration::from_secs(30)) + .danger_accept_invalid_certs(true) .build() .map_err(|e| FetchError::Build(format!("plain fallback: {e}")))?; plain.get(url).send().await? @@ -364,6 +372,7 @@ impl FetchClient { .user_agent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36") .cookie_store(true) .timeout(Duration::from_secs(30)) + .danger_accept_invalid_certs(true) .build() .map_err(|e| FetchError::Build(format!("plain fallback: {e}")))?; plain.get(url).send().await?