diff --git a/Cargo.lock b/Cargo.lock index ecd5f5a..bd7ff1d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,18 +28,6 @@ dependencies = [ "cpufeatures", ] -[[package]] -name = "ahash" -version = "0.8.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" -dependencies = [ - "cfg-if", - "once_cell", - "version_check", - "zerocopy", -] - [[package]] name = "aho-corasick" version = "1.1.4" @@ -64,6 +52,12 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -272,9 +266,9 @@ dependencies = [ [[package]] name = "bitflags" -version = "2.11.0" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" [[package]] name = "block-buffer" @@ -285,31 +279,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "boring-sys2" -version = "5.0.0-alpha.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "455d79965f5155dcc88a7abce112c3590883889131b799beda10bf9a813ed669" -dependencies = [ - "bindgen", - "cmake", - "fs_extra", - "fslock", -] - -[[package]] -name = "boring2" -version = "5.0.0-alpha.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "183ccc3854411c035410dcdbffafca62084f3a6c33f013c77e83c025d2a08a28" -dependencies = [ - "bitflags", - "boring-sys2", - "foreign-types", - "libc", - "openssl-macros", -] - [[package]] name = "brotli" version = "8.0.2" @@ -331,6 +300,31 @@ dependencies = [ "alloc-stdlib", ] +[[package]] +name = "btls" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c5e60b8c8d282c86360cab651ded04ab0335a7b5390c8d34145cbeab8cacf5f" +dependencies = [ + "bitflags", + "btls-sys", + "foreign-types", + "libc", + "openssl-macros", +] + +[[package]] +name = "btls-sys" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b1b8638a2e1c38a5ae4efa90ae57e643baec35a30d03fc5b399b893adc4954b" +dependencies = [ + "bindgen", + "cmake", + "fs_extra", + "fslock", +] + [[package]] name = "bumpalo" version = "3.20.2" @@ -865,6 +859,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "foreign-types" version = "0.5.0" @@ -1089,19 +1089,13 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" -[[package]] -name = "hashbrown" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" - [[package]] name = "hashbrown" version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "foldhash", + "foldhash 0.1.5", ] [[package]] @@ -1110,6 +1104,17 @@ version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + [[package]] name = "heck" version = "0.5.0" @@ -1172,9 +1177,9 @@ dependencies = [ [[package]] name = "http2" -version = "0.5.15" +version = "0.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c45c6490693ee8a8d0d95fdbdf76fead9fb87548f7894137259a7c6d22821948" +checksum = "569ef7a780e853c4e1768f58a3c8168193b82cdcbab66638a0b1c6583ec5995e" dependencies = [ "atomic-waker", "bytes", @@ -1183,7 +1188,6 @@ dependencies = [ "futures-sink", "http", "indexmap", - "parking_lot", "slab", "smallvec", "tokio", @@ -1495,9 +1499,9 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "libc" -version = "0.2.183" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libloading" @@ -1563,6 +1567,15 @@ dependencies = [ "weezl", ] +[[package]] +name = "lru" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a860605968fce16869fd239cf4237a82f3ac470723415db603b0e8b6c8d4fb9" +dependencies = [ + "hashbrown 0.17.1", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -2375,17 +2388,6 @@ dependencies = [ "syn", ] -[[package]] -name = "schnellru" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "356285bbf17bea63d9e52e96bd18f039672ac92b55b8cb997d6162a2a37d1649" -dependencies = [ - "ahash", - "cfg-if", - "hashbrown 0.13.2", -] - [[package]] name = "scopeguard" version = "1.2.0" @@ -2779,9 +2781,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.50.0" +version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ "bytes", "libc", @@ -2795,20 +2797,20 @@ dependencies = [ ] [[package]] -name = "tokio-boring2" -version = "5.0.0-alpha.13" +name = "tokio-btls" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f81df1210d791f31d72d840de8fbd80b9c3cb324956523048b1413e2bd55756" +checksum = "2e1fd638ec35427faf3b8f412e0fdd6fae76591d79dba40f38fa667d22bc44dd" dependencies = [ - "boring2", + "btls", "tokio", ] [[package]] name = "tokio-macros" -version = "2.6.1" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", @@ -3263,6 +3265,7 @@ dependencies = [ "async-trait", "bytes", "calamine", + "futures-util", "http", "quick-xml 0.37.5", "rand 0.8.5", @@ -3347,9 +3350,9 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "804f18a4ac2676ffb4e8b5b5fa9ae38af06df08162314f96a68d2a363e21a8ca" +checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c" dependencies = [ "rustls-pki-types", ] @@ -3696,17 +3699,14 @@ dependencies = [ [[package]] name = "wreq" -version = "6.0.0-rc.28" +version = "6.0.0-rc.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f79937f6c4df65b3f6f78715b9de2977afe9ee3b3436483c7949a24511e25935" +checksum = "3f0eba5f5814a94e5f1a99156f187133464e525b66bdbc69a9627d46530af2e1" dependencies = [ - "ahash", - "boring2", - "brotli", + "btls", + "btls-sys", "bytes", "cookie", - "flate2", - "futures-channel", "futures-util", "http", "http-body", @@ -3715,29 +3715,64 @@ dependencies = [ "httparse", "ipnet", "libc", + "lru", "percent-encoding", "pin-project-lite", - "schnellru", - "smallvec", "socket2", + "sync_wrapper", "tokio", - "tokio-boring2", + "tokio-btls", + "tokio-util", "tower", "tower-http", "url", - "want", "webpki-root-certs", - "zstd", + "wreq-proto", + "wreq-rt", +] + +[[package]] +name = "wreq-proto" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a43942f024bb303f1042c9aa3c87fa1d9149f507c65db6e5220a11ccdb207387" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "http2", + "httparse", + "pin-project-lite", + "smallvec", + "tokio", + "tokio-util", + "want", +] + +[[package]] +name = "wreq-rt" +version = "0.2.2-rc.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99e9bce67a3fa3dd3f1503f066d86661c9caf399a763d3bd184da7afaf886c8b" +dependencies = [ + "pin-project-lite", + "tokio", + "wreq-proto", ] [[package]] name = "wreq-util" -version = "3.0.0-rc.10" +version = "3.0.0-rc.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c6bbe24d28beb9ceb58b514bd6a613c759d3b706f768b9d2950d5d35b543c04" +checksum = "baa5d2ab72139256916ca352a3d05c53d74e1dd360052eb5ba7691033c417c65" dependencies = [ + "brotli", + "flate2", "typed-builder", "wreq", + "zstd", ] [[package]] diff --git a/crates/webclaw-fetch/Cargo.toml b/crates/webclaw-fetch/Cargo.toml index de3036b..4671dc1 100644 --- a/crates/webclaw-fetch/Cargo.toml +++ b/crates/webclaw-fetch/Cargo.toml @@ -14,13 +14,16 @@ tracing = { workspace = true } tokio = { workspace = true } async-trait = "0.1" # Pinned to exact pre-release versions: wreq/wreq-util are release candidates -# with no semver stability between rc.N builds (rc.29 broke the TLS + Response -# API). An exact pin keeps `cargo build`, `cargo install` (which ignores -# Cargo.lock), and the release workflow all on the version that compiles. -wreq = { version = "=6.0.0-rc.28", features = ["cookies", "gzip", "brotli", "zstd", "deflate"] } -wreq-util = "=3.0.0-rc.10" +# with no semver stability between rc.N builds. An exact pin keeps `cargo build`, +# `cargo install` (which ignores Cargo.lock), and the release workflow all on the +# version that compiles. +wreq = { version = "=6.0.0-rc.29", features = ["cookies", "gzip", "brotli", "zstd", "deflate", "stream"] } +wreq-util = "=3.0.0-rc.12" http = "1" bytes = "1" +# Stream adapter for `wreq::Response::bytes_stream()` (wreq 6.0.0-rc.29 dropped +# `Response::chunk()`); used to buffer bodies under the running size ceiling. +futures-util = "0.3" url = "2" rand = "0.8" quick-xml = { version = "0.37", features = ["serde"] } diff --git a/crates/webclaw-fetch/src/client.rs b/crates/webclaw-fetch/src/client.rs index 2bfd8c5..0724cec 100644 --- a/crates/webclaw-fetch/src/client.rs +++ b/crates/webclaw-fetch/src/client.rs @@ -12,6 +12,7 @@ use std::hash::{Hash, Hasher}; use std::sync::Arc; use std::time::{Duration, Instant}; +use futures_util::StreamExt; use rand::seq::SliceRandom; use tokio::sync::Semaphore; use tracing::{debug, instrument, warn}; @@ -118,7 +119,7 @@ impl Response { /// negotiated), so a tiny compressed payload that inflates to /// gigabytes is aborted as soon as the accumulated size crosses the /// cap — it never gets fully buffered in memory. - async fn from_wreq(mut resp: wreq::Response) -> Result { + async fn from_wreq(resp: wreq::Response) -> Result { if let Some(len) = resp.content_length() && len > MAX_BODY_BYTES { @@ -130,12 +131,13 @@ impl Response { let url = resp.uri().to_string(); let headers = resp.headers().clone(); + // wreq 6.0.0-rc.29 dropped `Response::chunk()`. Stream post-decompression + // bytes via `bytes_stream()` and keep enforcing the running ceiling so a + // compression bomb is aborted before it is fully buffered in memory. let mut buf = bytes::BytesMut::new(); - while let Some(chunk) = resp - .chunk() - .await - .map_err(|e| FetchError::BodyDecode(e.to_string()))? - { + let mut stream = resp.bytes_stream(); + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|e| FetchError::BodyDecode(e.to_string()))?; check_body_ceiling(buf.len(), chunk.len())?; buf.extend_from_slice(&chunk); } diff --git a/crates/webclaw-fetch/src/tls.rs b/crates/webclaw-fetch/src/tls.rs index c6c2955..02209b3 100644 --- a/crates/webclaw-fetch/src/tls.rs +++ b/crates/webclaw-fetch/src/tls.rs @@ -10,15 +10,24 @@ use std::{borrow::Cow, io, time::Duration}; use wreq::http2::{ Http2Options, PseudoId, PseudoOrder, SettingId, SettingsOrder, StreamDependency, StreamId, }; -use wreq::tls::{ - AlpnProtocol, AlpsProtocol, CertificateCompressionAlgorithm, ExtensionType, TlsOptions, - TlsVersion, -}; -use wreq::{Client, Emulation}; +use wreq::tls::compress::CertificateCompressor; +use wreq::tls::{AlpnProtocol, AlpsProtocol, ExtensionType, TlsOptions, TlsVersion}; +use wreq::{Client, Emulation, Group, IntoEmulation}; +use wreq_util::emulate::compress::{BrotliCompressor, ZlibCompressor}; use crate::browser::BrowserVariant; use crate::error::FetchError; +// Certificate-compression advertisement per profile. wreq 6.0.0-rc.29 replaced +// the `CertificateCompressionAlgorithm` enum argument with `&dyn +// CertificateCompressor` trait objects; wreq-util ships the concrete zlib/brotli +// implementations. The advertised set (and order) is a TLS fingerprint signal, +// so these mirror the previous enum lists exactly. +static CHROME_CERT_COMPRESSORS: &[&'static dyn CertificateCompressor] = &[&BrotliCompressor]; +static FIREFOX_CERT_COMPRESSORS: &[&'static dyn CertificateCompressor] = + &[&ZlibCompressor, &BrotliCompressor]; +static SAFARI_CERT_COMPRESSORS: &[&'static dyn CertificateCompressor] = &[&ZlibCompressor]; + #[derive(Clone, Default)] struct PublicDnsResolver; @@ -119,14 +128,14 @@ fn chrome_extensions() -> Vec { ExtensionType::PSK_KEY_EXCHANGE_MODES, // 45 ExtensionType::EC_POINT_FORMATS, // 11 ExtensionType::CERT_COMPRESSION, // 27 - ExtensionType::APPLICATION_SETTINGS_NEW, // 17613 (new codepoint, matches alps_use_new_codepoint) - ExtensionType::SUPPORTED_VERSIONS, // 43 - ExtensionType::SIGNATURE_ALGORITHMS, // 13 - ExtensionType::SERVER_NAME, // 0 + ExtensionType::APPLICATION_SETTINGS, // 17613 (new codepoint, matches alps_use_new_codepoint) + ExtensionType::SUPPORTED_VERSIONS, // 43 + ExtensionType::SIGNATURE_ALGORITHMS, // 13 + ExtensionType::SERVER_NAME, // 0 ExtensionType::APPLICATION_LAYER_PROTOCOL_NEGOTIATION, // 16 - ExtensionType::ENCRYPTED_CLIENT_HELLO, // 65037 - ExtensionType::RENEGOTIATE, // 65281 - ExtensionType::EXTENDED_MASTER_SECRET, // 23 + ExtensionType::ENCRYPTED_CLIENT_HELLO, // 65037 + ExtensionType::RENEGOTIATE, // 65281 + ExtensionType::EXTENDED_MASTER_SECRET, // 23 ] } @@ -287,7 +296,7 @@ fn chrome_tls() -> TlsOptions { .alps_protocols([AlpsProtocol::HTTP3, AlpsProtocol::HTTP2]) .alps_use_new_codepoint(true) .aes_hw_override(true) - .certificate_compression_algorithms(&[CertificateCompressionAlgorithm::BROTLI]) + .certificate_compressors(CHROME_CERT_COMPRESSORS) .build() } @@ -304,10 +313,7 @@ fn firefox_tls() -> TlsOptions { .pre_shared_key(true) .enable_ocsp_stapling(true) .enable_signed_cert_timestamps(true) - .certificate_compression_algorithms(&[ - CertificateCompressionAlgorithm::ZLIB, - CertificateCompressionAlgorithm::BROTLI, - ]) + .certificate_compressors(FIREFOX_CERT_COMPRESSORS) .build() } @@ -324,7 +330,7 @@ fn safari_tls() -> TlsOptions { .pre_shared_key(false) .enable_ocsp_stapling(true) .enable_signed_cert_timestamps(true) - .certificate_compression_algorithms(&[CertificateCompressionAlgorithm::ZLIB]) + .certificate_compressors(SAFARI_CERT_COMPRESSORS) .build() } @@ -345,21 +351,23 @@ fn safari_tls() -> TlsOptions { /// `priority: u=0, i`, zstd), replace with the real iOS 26 set. /// 4. `accept-language` preserved from config.extra_headers for locale. fn safari_ios_emulation() -> wreq::Emulation { - use wreq::EmulationFactory; - let mut em = wreq_util::Emulation::SafariIos26.emulation(); + // wreq 6.0.0-rc.29 exposes the `Emulation` fields directly (no `*_mut()` + // accessors) and wreq-util 3.0.0-rc.12 renamed the enum to `Profile` with + // `IntoEmulation::into_emulation` replacing `EmulationFactory::emulation`. + let mut em = wreq_util::Profile::SafariIos26.into_emulation(); - if let Some(tls) = em.tls_options_mut().as_mut() { + if let Some(tls) = em.tls_options.as_mut() { tls.extension_permutation = Some(Cow::Owned(safari_ios_extensions())); } // Only override the priority flag — keep wreq-util's SETTINGS, WINDOW_UPDATE, // and pseudo-order intact. Replacing the whole Http2Options resets SETTINGS // to defaults, which sends only INITIAL_WINDOW_SIZE and fails DataDome. - if let Some(h2) = em.http2_options_mut().as_mut() { + if let Some(h2) = em.http2_options.as_mut() { h2.headers_stream_dependency = Some(StreamDependency::new(StreamId::zero(), 255, true)); } - let hm = em.headers_mut(); + let hm = &mut em.headers; hm.clear(); for (k, v) in SAFARI_IOS_HEADERS { if let (Ok(n), Ok(val)) = ( @@ -508,12 +516,12 @@ pub fn build_client( .tls_options(tls) .http2_options(h2) .headers(build_headers(headers)) - .build() + .build(Group::default()) } }; // Append extra headers after profile defaults. - let hm = emulation.headers_mut(); + let hm = &mut emulation.headers; for (k, v) in extra_headers { if let (Ok(n), Ok(val)) = ( http::header::HeaderName::from_bytes(k.as_bytes()),