mirror of
https://github.com/0xMassi/webclaw.git
synced 2026-06-25 03:08:06 +02:00
14ded4b99e
Ports the TLS/Response API breaks in the bump: - certificate_compression_algorithms -> certificate_compressors with wreq-util's BrotliCompressor/ZlibCompressor trait objects - ExtensionType::APPLICATION_SETTINGS_NEW -> APPLICATION_SETTINGS (same codepoint 17613) - wreq_util::Emulation::SafariIos26.emulation() -> Profile::SafariIos26.into_emulation(); Emulation fields are now public so *_mut() accessors become direct field access; build() takes a Group - Response::chunk() removed -> bytes_stream() (wreq 'stream' feature) with the running body-size ceiling preserved; adds futures-util Browser fingerprints verified unchanged on tls.peet.ws: Chrome JA3 43067709b025da334de1279a120f8e14, Safari iOS JA3 8d909525bd5bbb79f133d11cc05159fe.
634 lines
25 KiB
Rust
634 lines
25 KiB
Rust
//! Browser TLS + HTTP/2 fingerprint profiles built on wreq (BoringSSL).
|
|
//!
|
|
//! Replaces the old webclaw-http/webclaw-tls patched rustls stack.
|
|
//! Each profile configures TLS options (cipher suites, curves, extensions,
|
|
//! PSK, ECH GREASE) and HTTP/2 options (SETTINGS order, pseudo-header order,
|
|
//! stream dependency, priorities) to match real browser fingerprints.
|
|
|
|
use std::{borrow::Cow, io, time::Duration};
|
|
|
|
use wreq::http2::{
|
|
Http2Options, PseudoId, PseudoOrder, SettingId, SettingsOrder, StreamDependency, StreamId,
|
|
};
|
|
use wreq::tls::compress::CertificateCompressor;
|
|
use wreq::tls::{AlpnProtocol, AlpsProtocol, ExtensionType, TlsOptions, TlsVersion};
|
|
use wreq::{Client, Emulation, Group, IntoEmulation};
|
|
use wreq_util::emulate::compress::{BrotliCompressor, ZlibCompressor};
|
|
|
|
use crate::browser::BrowserVariant;
|
|
use crate::error::FetchError;
|
|
|
|
// Certificate-compression advertisement per profile. wreq 6.0.0-rc.29 replaced
|
|
// the `CertificateCompressionAlgorithm` enum argument with `&dyn
|
|
// CertificateCompressor` trait objects; wreq-util ships the concrete zlib/brotli
|
|
// implementations. The advertised set (and order) is a TLS fingerprint signal,
|
|
// so these mirror the previous enum lists exactly.
|
|
static CHROME_CERT_COMPRESSORS: &[&'static dyn CertificateCompressor] = &[&BrotliCompressor];
|
|
static FIREFOX_CERT_COMPRESSORS: &[&'static dyn CertificateCompressor] =
|
|
&[&ZlibCompressor, &BrotliCompressor];
|
|
static SAFARI_CERT_COMPRESSORS: &[&'static dyn CertificateCompressor] = &[&ZlibCompressor];
|
|
|
|
#[derive(Clone, Default)]
|
|
struct PublicDnsResolver;
|
|
|
|
impl wreq::dns::Resolve for PublicDnsResolver {
|
|
fn resolve(&self, name: wreq::dns::Name) -> wreq::dns::Resolving {
|
|
Box::pin(async move {
|
|
let addrs = tokio::net::lookup_host((name.as_str(), 0))
|
|
.await
|
|
.map_err(|e| Box::new(e) as Box<dyn std::error::Error + Send + Sync>)?;
|
|
let mut public = Vec::new();
|
|
|
|
for addr in addrs {
|
|
if crate::url_security::is_blocked_ip(addr.ip()) {
|
|
let err: Box<dyn std::error::Error + Send + Sync> = Box::new(io::Error::new(
|
|
io::ErrorKind::PermissionDenied,
|
|
"DNS resolved to a blocked private or internal address",
|
|
));
|
|
return Err(err);
|
|
}
|
|
public.push(addr);
|
|
}
|
|
|
|
if public.is_empty() {
|
|
let err: Box<dyn std::error::Error + Send + Sync> = Box::new(io::Error::new(
|
|
io::ErrorKind::NotFound,
|
|
"host did not resolve to any addresses",
|
|
));
|
|
return Err(err);
|
|
}
|
|
|
|
Ok(Box::new(public.into_iter()) as wreq::dns::Addrs)
|
|
})
|
|
}
|
|
}
|
|
|
|
/// Chrome cipher list (TLS 1.3 + TLS 1.2 in Chrome's exact order).
|
|
const CHROME_CIPHERS: &str = "TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256:TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256:TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384:TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384:TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256:TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256:TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA:TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA:TLS_RSA_WITH_AES_128_GCM_SHA256:TLS_RSA_WITH_AES_256_GCM_SHA384:TLS_RSA_WITH_AES_128_CBC_SHA:TLS_RSA_WITH_AES_256_CBC_SHA";
|
|
|
|
/// Chrome signature algorithms.
|
|
const CHROME_SIGALGS: &str = "ecdsa_secp256r1_sha256:rsa_pss_rsae_sha256:rsa_pkcs1_sha256:ecdsa_secp384r1_sha384:rsa_pss_rsae_sha384:rsa_pkcs1_sha384:rsa_pss_rsae_sha512:rsa_pkcs1_sha512";
|
|
|
|
/// Chrome curves (post-quantum ML-KEM + X25519 + P-256 + P-384).
|
|
const CHROME_CURVES: &str = "X25519MLKEM768:X25519:P-256:P-384";
|
|
|
|
/// Firefox cipher list.
|
|
const FIREFOX_CIPHERS: &str = "TLS_AES_128_GCM_SHA256:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_256_GCM_SHA384:TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256:TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256:TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256:TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256:TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384:TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384:TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA:TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA:TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA:TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA:TLS_RSA_WITH_AES_128_GCM_SHA256:TLS_RSA_WITH_AES_256_GCM_SHA384:TLS_RSA_WITH_AES_128_CBC_SHA:TLS_RSA_WITH_AES_256_CBC_SHA";
|
|
|
|
/// Firefox signature algorithms.
|
|
const FIREFOX_SIGALGS: &str = "ecdsa_secp256r1_sha256:ecdsa_secp384r1_sha384:ecdsa_secp521r1_sha512:rsa_pss_rsae_sha256:rsa_pss_rsae_sha384:rsa_pss_rsae_sha512:rsa_pkcs1_sha256:rsa_pkcs1_sha384:rsa_pkcs1_sha512:ecdsa_sha1:rsa_pkcs1_sha1";
|
|
|
|
/// Firefox curves.
|
|
const FIREFOX_CURVES: &str = "X25519MLKEM768:X25519:P-256:P-384:P-521";
|
|
|
|
/// Safari cipher list.
|
|
const SAFARI_CIPHERS: &str = "TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384:TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256:TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256:TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384:TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256:TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256:TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA:TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA:TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA:TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA:TLS_RSA_WITH_AES_256_GCM_SHA384:TLS_RSA_WITH_AES_128_GCM_SHA256:TLS_RSA_WITH_AES_256_CBC_SHA:TLS_RSA_WITH_AES_128_CBC_SHA";
|
|
|
|
/// Safari signature algorithms.
|
|
const SAFARI_SIGALGS: &str = "ecdsa_secp256r1_sha256:rsa_pss_rsae_sha256:rsa_pkcs1_sha256:ecdsa_secp384r1_sha384:rsa_pss_rsae_sha384:ecdsa_secp521r1_sha512:rsa_pss_rsae_sha512:rsa_pkcs1_sha384:rsa_pkcs1_sha512";
|
|
|
|
/// Safari curves.
|
|
const SAFARI_CURVES: &str = "X25519:P-256:P-384:P-521";
|
|
|
|
/// Safari iOS 26 TLS extension order, matching bogdanfinn's
|
|
/// `safari_ios_26_0` wire format. GREASE slots are omitted. wreq
|
|
/// inserts them itself. Diverges from wreq-util's default SafariIos26
|
|
/// extension order, which DataDome's immobiliare.it ruleset flags.
|
|
fn safari_ios_extensions() -> Vec<ExtensionType> {
|
|
vec![
|
|
ExtensionType::CERTIFICATE_TIMESTAMP,
|
|
ExtensionType::APPLICATION_LAYER_PROTOCOL_NEGOTIATION,
|
|
ExtensionType::SERVER_NAME,
|
|
ExtensionType::CERT_COMPRESSION,
|
|
ExtensionType::KEY_SHARE,
|
|
ExtensionType::SUPPORTED_VERSIONS,
|
|
ExtensionType::PSK_KEY_EXCHANGE_MODES,
|
|
ExtensionType::SUPPORTED_GROUPS,
|
|
ExtensionType::RENEGOTIATE,
|
|
ExtensionType::SIGNATURE_ALGORITHMS,
|
|
ExtensionType::STATUS_REQUEST,
|
|
ExtensionType::EC_POINT_FORMATS,
|
|
ExtensionType::EXTENDED_MASTER_SECRET,
|
|
]
|
|
}
|
|
|
|
/// Chrome 133 TLS extension order, matching bogdanfinn's stable JA3
|
|
/// (`43067709b025da334de1279a120f8e14`). Real Chrome permutes extensions
|
|
/// per handshake, but indeed.com's WAF allowlists this specific wire order
|
|
/// and rejects permuted ones. GREASE slots are inserted by wreq.
|
|
///
|
|
/// JA3 extension field from peet.ws: 18-5-35-51-10-45-11-27-17613-43-13-0-16-65037-65281-23
|
|
fn chrome_extensions() -> Vec<ExtensionType> {
|
|
vec![
|
|
ExtensionType::CERTIFICATE_TIMESTAMP, // 18
|
|
ExtensionType::STATUS_REQUEST, // 5
|
|
ExtensionType::SESSION_TICKET, // 35
|
|
ExtensionType::KEY_SHARE, // 51
|
|
ExtensionType::SUPPORTED_GROUPS, // 10
|
|
ExtensionType::PSK_KEY_EXCHANGE_MODES, // 45
|
|
ExtensionType::EC_POINT_FORMATS, // 11
|
|
ExtensionType::CERT_COMPRESSION, // 27
|
|
ExtensionType::APPLICATION_SETTINGS, // 17613 (new codepoint, matches alps_use_new_codepoint)
|
|
ExtensionType::SUPPORTED_VERSIONS, // 43
|
|
ExtensionType::SIGNATURE_ALGORITHMS, // 13
|
|
ExtensionType::SERVER_NAME, // 0
|
|
ExtensionType::APPLICATION_LAYER_PROTOCOL_NEGOTIATION, // 16
|
|
ExtensionType::ENCRYPTED_CLIENT_HELLO, // 65037
|
|
ExtensionType::RENEGOTIATE, // 65281
|
|
ExtensionType::EXTENDED_MASTER_SECRET, // 23
|
|
]
|
|
}
|
|
|
|
// --- Chrome HTTP headers in correct wire order ---
|
|
|
|
const CHROME_HEADERS: &[(&str, &str)] = &[
|
|
(
|
|
"sec-ch-ua",
|
|
r#""Google Chrome";v="145", "Chromium";v="145", "Not/A)Brand";v="24""#,
|
|
),
|
|
("sec-ch-ua-mobile", "?0"),
|
|
("sec-ch-ua-platform", "\"Windows\""),
|
|
("upgrade-insecure-requests", "1"),
|
|
(
|
|
"user-agent",
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36",
|
|
),
|
|
(
|
|
"accept",
|
|
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
|
|
),
|
|
("sec-fetch-site", "none"),
|
|
("sec-fetch-mode", "navigate"),
|
|
("sec-fetch-user", "?1"),
|
|
("sec-fetch-dest", "document"),
|
|
("accept-encoding", "gzip, deflate, br, zstd"),
|
|
("accept-language", "en-US,en;q=0.9"),
|
|
("priority", "u=0, i"),
|
|
];
|
|
|
|
const CHROME_MACOS_HEADERS: &[(&str, &str)] = &[
|
|
(
|
|
"sec-ch-ua",
|
|
r#""Google Chrome";v="145", "Chromium";v="145", "Not/A)Brand";v="24""#,
|
|
),
|
|
("sec-ch-ua-mobile", "?0"),
|
|
("sec-ch-ua-platform", "\"macOS\""),
|
|
("upgrade-insecure-requests", "1"),
|
|
(
|
|
"user-agent",
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36",
|
|
),
|
|
(
|
|
"accept",
|
|
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
|
|
),
|
|
("sec-fetch-site", "none"),
|
|
("sec-fetch-mode", "navigate"),
|
|
("sec-fetch-user", "?1"),
|
|
("sec-fetch-dest", "document"),
|
|
("accept-encoding", "gzip, deflate, br, zstd"),
|
|
("accept-language", "en-US,en;q=0.9"),
|
|
("priority", "u=0, i"),
|
|
];
|
|
|
|
const FIREFOX_HEADERS: &[(&str, &str)] = &[
|
|
(
|
|
"user-agent",
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:135.0) Gecko/20100101 Firefox/135.0",
|
|
),
|
|
(
|
|
"accept",
|
|
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
),
|
|
("accept-language", "en-US,en;q=0.5"),
|
|
("accept-encoding", "gzip, deflate, br, zstd"),
|
|
("upgrade-insecure-requests", "1"),
|
|
("sec-fetch-dest", "document"),
|
|
("sec-fetch-mode", "navigate"),
|
|
("sec-fetch-site", "none"),
|
|
("sec-fetch-user", "?1"),
|
|
("priority", "u=0, i"),
|
|
];
|
|
|
|
const SAFARI_HEADERS: &[(&str, &str)] = &[
|
|
(
|
|
"user-agent",
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.3.1 Safari/605.1.15",
|
|
),
|
|
(
|
|
"accept",
|
|
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
),
|
|
("sec-fetch-site", "none"),
|
|
("accept-language", "en-US,en;q=0.9"),
|
|
("sec-fetch-mode", "navigate"),
|
|
("accept-encoding", "gzip, deflate, br"),
|
|
("sec-fetch-dest", "document"),
|
|
];
|
|
|
|
/// Safari iOS 26 headers, in the wire order real Safari emits. Critically:
|
|
/// NO `sec-fetch-*`, NO `priority: u=0, i` (both Chromium-only leaks), but
|
|
/// `upgrade-insecure-requests: 1` is present. `accept-encoding` does not
|
|
/// include zstd (Safari can't decode it). Verified against bogdanfinn on
|
|
/// 2026-04-22: this header set is what DataDome's immobiliare ruleset
|
|
/// expects for a real iPhone.
|
|
const SAFARI_IOS_HEADERS: &[(&str, &str)] = &[
|
|
(
|
|
"accept",
|
|
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
),
|
|
("accept-language", "en-US,en;q=0.9"),
|
|
("accept-encoding", "gzip, deflate, br"),
|
|
(
|
|
"user-agent",
|
|
"Mozilla/5.0 (iPhone; CPU iPhone OS 26_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.0 Mobile/15E148 Safari/604.1",
|
|
),
|
|
("upgrade-insecure-requests", "1"),
|
|
];
|
|
|
|
const EDGE_HEADERS: &[(&str, &str)] = &[
|
|
(
|
|
"sec-ch-ua",
|
|
r#""Microsoft Edge";v="145", "Chromium";v="145", "Not/A)Brand";v="24""#,
|
|
),
|
|
("sec-ch-ua-mobile", "?0"),
|
|
("sec-ch-ua-platform", "\"Windows\""),
|
|
("upgrade-insecure-requests", "1"),
|
|
(
|
|
"user-agent",
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36 Edg/145.0.0.0",
|
|
),
|
|
(
|
|
"accept",
|
|
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
|
|
),
|
|
("sec-fetch-site", "none"),
|
|
("sec-fetch-mode", "navigate"),
|
|
("sec-fetch-user", "?1"),
|
|
("sec-fetch-dest", "document"),
|
|
("accept-encoding", "gzip, deflate, br, zstd"),
|
|
("accept-language", "en-US,en;q=0.9"),
|
|
("priority", "u=0, i"),
|
|
];
|
|
|
|
fn chrome_tls() -> TlsOptions {
|
|
// permute_extensions is off so the explicit extension_permutation sticks.
|
|
// Real Chrome permutes, but indeed.com's WAF allowlists bogdanfinn's
|
|
// fixed order, so matching that gets us through.
|
|
TlsOptions::builder()
|
|
.cipher_list(CHROME_CIPHERS)
|
|
.sigalgs_list(CHROME_SIGALGS)
|
|
.curves_list(CHROME_CURVES)
|
|
.min_tls_version(TlsVersion::TLS_1_2)
|
|
.max_tls_version(TlsVersion::TLS_1_3)
|
|
.grease_enabled(true)
|
|
.permute_extensions(false)
|
|
.extension_permutation(chrome_extensions())
|
|
.enable_ech_grease(true)
|
|
.pre_shared_key(true)
|
|
.enable_ocsp_stapling(true)
|
|
.enable_signed_cert_timestamps(true)
|
|
.alpn_protocols([
|
|
AlpnProtocol::HTTP3,
|
|
AlpnProtocol::HTTP2,
|
|
AlpnProtocol::HTTP1,
|
|
])
|
|
.alps_protocols([AlpsProtocol::HTTP3, AlpsProtocol::HTTP2])
|
|
.alps_use_new_codepoint(true)
|
|
.aes_hw_override(true)
|
|
.certificate_compressors(CHROME_CERT_COMPRESSORS)
|
|
.build()
|
|
}
|
|
|
|
fn firefox_tls() -> TlsOptions {
|
|
TlsOptions::builder()
|
|
.cipher_list(FIREFOX_CIPHERS)
|
|
.sigalgs_list(FIREFOX_SIGALGS)
|
|
.curves_list(FIREFOX_CURVES)
|
|
.min_tls_version(TlsVersion::TLS_1_2)
|
|
.max_tls_version(TlsVersion::TLS_1_3)
|
|
.grease_enabled(true)
|
|
.permute_extensions(false)
|
|
.enable_ech_grease(true)
|
|
.pre_shared_key(true)
|
|
.enable_ocsp_stapling(true)
|
|
.enable_signed_cert_timestamps(true)
|
|
.certificate_compressors(FIREFOX_CERT_COMPRESSORS)
|
|
.build()
|
|
}
|
|
|
|
fn safari_tls() -> TlsOptions {
|
|
TlsOptions::builder()
|
|
.cipher_list(SAFARI_CIPHERS)
|
|
.sigalgs_list(SAFARI_SIGALGS)
|
|
.curves_list(SAFARI_CURVES)
|
|
.min_tls_version(TlsVersion::TLS_1_2)
|
|
.max_tls_version(TlsVersion::TLS_1_3)
|
|
.grease_enabled(true)
|
|
.permute_extensions(false)
|
|
.enable_ech_grease(false)
|
|
.pre_shared_key(false)
|
|
.enable_ocsp_stapling(true)
|
|
.enable_signed_cert_timestamps(true)
|
|
.certificate_compressors(SAFARI_CERT_COMPRESSORS)
|
|
.build()
|
|
}
|
|
|
|
/// Safari iOS 26 emulation — composed on top of `wreq_util::Emulation::SafariIos26`
|
|
/// with four targeted overrides. We don't hand-roll this one like Chrome/Firefox
|
|
/// because the wire-level defaults from wreq-util are already correct for ciphers,
|
|
/// sigalgs, curves, and GREASE — the four things wreq-util gets *wrong* for
|
|
/// DataDome compatibility are overridden here:
|
|
///
|
|
/// 1. TLS extension order: match bogdanfinn `safari_ios_26_0` exactly (JA3
|
|
/// ends up `8d909525bd5bbb79f133d11cc05159fe`).
|
|
/// 2. HTTP/2 HEADERS priority flag: weight=256, exclusive=1, depends_on=0.
|
|
/// wreq-util omits this frame; real Safari and bogdanfinn include it.
|
|
/// This flip is the thing DataDome actually reads — the akamai_fingerprint
|
|
/// hash changes from `c52879e43202aeb92740be6e8c86ea96` to
|
|
/// `d1294410a06522e37a5c5e3f0a45a705`, which is the winning signature.
|
|
/// 3. Headers: strip wreq-util's Chromium defaults (`sec-fetch-*`,
|
|
/// `priority: u=0, i`, zstd), replace with the real iOS 26 set.
|
|
/// 4. `accept-language` preserved from config.extra_headers for locale.
|
|
fn safari_ios_emulation() -> wreq::Emulation {
|
|
// wreq 6.0.0-rc.29 exposes the `Emulation` fields directly (no `*_mut()`
|
|
// accessors) and wreq-util 3.0.0-rc.12 renamed the enum to `Profile` with
|
|
// `IntoEmulation::into_emulation` replacing `EmulationFactory::emulation`.
|
|
let mut em = wreq_util::Profile::SafariIos26.into_emulation();
|
|
|
|
if let Some(tls) = em.tls_options.as_mut() {
|
|
tls.extension_permutation = Some(Cow::Owned(safari_ios_extensions()));
|
|
}
|
|
|
|
// Only override the priority flag — keep wreq-util's SETTINGS, WINDOW_UPDATE,
|
|
// and pseudo-order intact. Replacing the whole Http2Options resets SETTINGS
|
|
// to defaults, which sends only INITIAL_WINDOW_SIZE and fails DataDome.
|
|
if let Some(h2) = em.http2_options.as_mut() {
|
|
h2.headers_stream_dependency = Some(StreamDependency::new(StreamId::zero(), 255, true));
|
|
}
|
|
|
|
let hm = &mut em.headers;
|
|
hm.clear();
|
|
for (k, v) in SAFARI_IOS_HEADERS {
|
|
if let (Ok(n), Ok(val)) = (
|
|
http::header::HeaderName::from_bytes(k.as_bytes()),
|
|
http::header::HeaderValue::from_str(v),
|
|
) {
|
|
hm.append(n, val);
|
|
}
|
|
}
|
|
|
|
em
|
|
}
|
|
|
|
fn chrome_h2() -> Http2Options {
|
|
// SETTINGS frame matches bogdanfinn `chrome_133`: HEADER_TABLE_SIZE,
|
|
// ENABLE_PUSH=0, INITIAL_WINDOW_SIZE, MAX_HEADER_LIST_SIZE. No
|
|
// MAX_CONCURRENT_STREAMS — real Chrome 133 and bogdanfinn both omit it,
|
|
// and indeed.com's WAF reads this as a bot signal when present. Priority
|
|
// weight 256 (encoded as 255 + 1) matches bogdanfinn's HEADERS frame.
|
|
Http2Options::builder()
|
|
.initial_window_size(6_291_456)
|
|
.initial_connection_window_size(15_728_640)
|
|
.max_header_list_size(262_144)
|
|
.header_table_size(65_536)
|
|
.enable_push(false)
|
|
.settings_order(
|
|
SettingsOrder::builder()
|
|
.extend([
|
|
SettingId::HeaderTableSize,
|
|
SettingId::EnablePush,
|
|
SettingId::InitialWindowSize,
|
|
SettingId::MaxHeaderListSize,
|
|
])
|
|
.build(),
|
|
)
|
|
.headers_pseudo_order(
|
|
PseudoOrder::builder()
|
|
.extend([
|
|
PseudoId::Method,
|
|
PseudoId::Authority,
|
|
PseudoId::Scheme,
|
|
PseudoId::Path,
|
|
])
|
|
.build(),
|
|
)
|
|
.headers_stream_dependency(StreamDependency::new(StreamId::zero(), 255, true))
|
|
.build()
|
|
}
|
|
|
|
fn firefox_h2() -> Http2Options {
|
|
Http2Options::builder()
|
|
.initial_window_size(131_072)
|
|
.initial_connection_window_size(12_517_377)
|
|
.max_header_list_size(65_536)
|
|
.header_table_size(65_536)
|
|
.settings_order(
|
|
SettingsOrder::builder()
|
|
.extend([
|
|
SettingId::HeaderTableSize,
|
|
SettingId::InitialWindowSize,
|
|
SettingId::MaxFrameSize,
|
|
])
|
|
.build(),
|
|
)
|
|
.headers_pseudo_order(
|
|
PseudoOrder::builder()
|
|
.extend([
|
|
PseudoId::Method,
|
|
PseudoId::Path,
|
|
PseudoId::Authority,
|
|
PseudoId::Scheme,
|
|
])
|
|
.build(),
|
|
)
|
|
.build()
|
|
}
|
|
|
|
fn safari_h2() -> Http2Options {
|
|
Http2Options::builder()
|
|
.initial_window_size(2_097_152)
|
|
.initial_connection_window_size(10_420_225)
|
|
.max_header_list_size(0)
|
|
.header_table_size(4_096)
|
|
.enable_push(false)
|
|
.max_concurrent_streams(100u32)
|
|
.settings_order(
|
|
SettingsOrder::builder()
|
|
.extend([
|
|
SettingId::EnablePush,
|
|
SettingId::MaxConcurrentStreams,
|
|
SettingId::InitialWindowSize,
|
|
SettingId::MaxFrameSize,
|
|
])
|
|
.build(),
|
|
)
|
|
.headers_pseudo_order(
|
|
PseudoOrder::builder()
|
|
.extend([
|
|
PseudoId::Method,
|
|
PseudoId::Scheme,
|
|
PseudoId::Authority,
|
|
PseudoId::Path,
|
|
])
|
|
.build(),
|
|
)
|
|
.headers_stream_dependency(StreamDependency::new(StreamId::zero(), 255, false))
|
|
.build()
|
|
}
|
|
|
|
fn build_headers(pairs: &[(&str, &str)]) -> http::HeaderMap {
|
|
let mut map = http::HeaderMap::with_capacity(pairs.len());
|
|
for (name, value) in pairs {
|
|
if let (Ok(n), Ok(v)) = (
|
|
http::header::HeaderName::from_bytes(name.as_bytes()),
|
|
http::header::HeaderValue::from_str(value),
|
|
) {
|
|
map.insert(n, v);
|
|
}
|
|
}
|
|
map
|
|
}
|
|
|
|
/// Build a wreq Client for a specific browser variant.
|
|
pub fn build_client(
|
|
variant: BrowserVariant,
|
|
timeout: Duration,
|
|
extra_headers: &std::collections::HashMap<String, String>,
|
|
proxy: Option<&str>,
|
|
follow_redirects: bool,
|
|
max_redirects: u32,
|
|
) -> Result<Client, FetchError> {
|
|
// SafariIos26 builds its Emulation on top of wreq-util's base instead
|
|
// of from scratch. See `safari_ios_emulation` for why.
|
|
let mut emulation = match variant {
|
|
BrowserVariant::SafariIos26 => safari_ios_emulation(),
|
|
other => {
|
|
let (tls, h2, headers) = match other {
|
|
BrowserVariant::Chrome => (chrome_tls(), chrome_h2(), CHROME_HEADERS),
|
|
BrowserVariant::ChromeMacos => (chrome_tls(), chrome_h2(), CHROME_MACOS_HEADERS),
|
|
BrowserVariant::Firefox => (firefox_tls(), firefox_h2(), FIREFOX_HEADERS),
|
|
BrowserVariant::Safari => (safari_tls(), safari_h2(), SAFARI_HEADERS),
|
|
BrowserVariant::Edge => (chrome_tls(), chrome_h2(), EDGE_HEADERS),
|
|
BrowserVariant::SafariIos26 => unreachable!("handled above"),
|
|
};
|
|
Emulation::builder()
|
|
.tls_options(tls)
|
|
.http2_options(h2)
|
|
.headers(build_headers(headers))
|
|
.build(Group::default())
|
|
}
|
|
};
|
|
|
|
// Append extra headers after profile defaults.
|
|
let hm = &mut emulation.headers;
|
|
for (k, v) in extra_headers {
|
|
if let (Ok(n), Ok(val)) = (
|
|
http::header::HeaderName::from_bytes(k.as_bytes()),
|
|
http::header::HeaderValue::from_str(v),
|
|
) {
|
|
hm.insert(n, val);
|
|
}
|
|
}
|
|
|
|
let mut builder = Client::builder()
|
|
.emulation(emulation)
|
|
.redirect(ssrf_safe_redirect_policy(
|
|
follow_redirects,
|
|
max_redirects as usize,
|
|
))
|
|
.cookie_store(true)
|
|
.timeout(timeout);
|
|
|
|
if let Some(proxy_url) = proxy {
|
|
let proxy = wreq::Proxy::all(proxy_url).map_err(|_| {
|
|
FetchError::Build(format!("invalid proxy {}", redact_proxy_url(proxy_url)))
|
|
})?;
|
|
builder = builder.proxy(proxy);
|
|
} else {
|
|
builder = builder.dns_resolver(PublicDnsResolver);
|
|
}
|
|
|
|
builder
|
|
.build()
|
|
.map_err(|e| FetchError::Build(e.to_string()))
|
|
}
|
|
|
|
/// Render a proxy URL safe to log: drop any `user:pass@` userinfo so
|
|
/// rotating-proxy credentials never reach error strings or tracing.
|
|
/// Falls back to a constant placeholder when the input does not parse.
|
|
fn redact_proxy_url(raw: &str) -> String {
|
|
match url::Url::parse(raw) {
|
|
Ok(mut u) => {
|
|
// Best-effort: opaque URLs (e.g. no host) reject these setters;
|
|
// in that case fall through to the placeholder rather than risk
|
|
// returning the raw string with credentials.
|
|
if u.set_username("").is_err() || u.set_password(None).is_err() {
|
|
return "<proxy redacted>".to_string();
|
|
}
|
|
u.to_string()
|
|
}
|
|
Err(_) => "<proxy redacted>".to_string(),
|
|
}
|
|
}
|
|
|
|
fn ssrf_safe_redirect_policy(
|
|
follow_redirects: bool,
|
|
max_redirects: usize,
|
|
) -> wreq::redirect::Policy {
|
|
if !follow_redirects {
|
|
return wreq::redirect::Policy::none();
|
|
}
|
|
|
|
wreq::redirect::Policy::custom(move |attempt| {
|
|
if attempt.previous.len() > max_redirects {
|
|
return attempt.error("too many redirects");
|
|
}
|
|
|
|
attempt.pending(|attempt| async move {
|
|
let next_url = attempt.uri.to_string();
|
|
match crate::url_security::validate_public_http_url(&next_url).await {
|
|
Ok(_) => attempt.follow(),
|
|
Err(e) => attempt.error(e.to_string()),
|
|
}
|
|
})
|
|
})
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::redact_proxy_url;
|
|
|
|
#[test]
|
|
fn redacts_userinfo_from_proxy_url() {
|
|
let red = redact_proxy_url("http://user123:s3cr3tPass@proxy.example.com:8080");
|
|
assert!(!red.contains("user123"), "username leaked: {red}");
|
|
assert!(!red.contains("s3cr3tPass"), "password leaked: {red}");
|
|
assert!(red.contains("proxy.example.com"), "host lost: {red}");
|
|
assert!(red.contains("8080"), "port lost: {red}");
|
|
}
|
|
|
|
#[test]
|
|
fn redacts_long_token_residential_proxy() {
|
|
// Residential-style: long structured credential with embedded
|
|
// tokens in the username and special chars in the password.
|
|
let red =
|
|
redact_proxy_url("http://acct-zone-resi-country-xx:p@ss-word@gw.proxy.example:12321");
|
|
assert!(!red.contains("acct-zone-resi"), "username leaked: {red}");
|
|
assert!(!red.contains("p@ss-word"), "password leaked: {red}");
|
|
assert!(red.contains("gw.proxy.example"));
|
|
}
|
|
|
|
#[test]
|
|
fn unparseable_proxy_does_not_echo_input() {
|
|
let red = redact_proxy_url("user:pass@not a url");
|
|
assert_eq!(red, "<proxy redacted>");
|
|
}
|
|
|
|
#[test]
|
|
fn proxy_without_credentials_is_preserved() {
|
|
let red = redact_proxy_url("http://proxy.example.com:3128");
|
|
assert!(red.contains("proxy.example.com"));
|
|
assert!(red.contains("3128"));
|
|
}
|
|
}
|