apunkt/webclaw
1
0
Fork 0
mirror of https://github.com/0xMassi/webclaw.git synced 2026-04-25 00:06:21 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: replace primp with webclaw-tls, bump to v0.3.0

Browse source 
Replace primp dependency with our own TLS fingerprinting stack
(webclaw-tls). Perfect Chrome 146 JA4 + Akamai hash match.

- Remove primp entirely (zero references remaining)
- webclaw-fetch now uses webclaw-http from github.com/0xMassi/webclaw-tls
- Native + Mozilla root CAs (fixes HTTPS on cross-signed cert chains)
- Skip unknown certificate extensions (SCT tolerance)
- 99% bypass rate on 102 sites (was ~85% with primp)
- Fixes #5 (HTTPS broken — example.com and similar sites now work)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Valerio 2026-03-29 16:40:10 +02:00
parent 77e93441c0
commit f13cb83c73
8 changed files with 204 additions and 599 deletions
Download patch file Download diff file Expand all files Collapse all files

236
Cargo.lock generated
View file

@ -25,7 +25,7 @@ checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0"
dependencies = [
"cfg-if",
"cipher",
"cpufeatures 0.2.17",
"cpufeatures",
]
[[package]]
@ -183,9 +183,9 @@ dependencies = [
[[package]]
name = "aws-lc-sys"
version = "0.39.0"
version = "0.39.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fa7e52a4c5c547c741610a2c6f123f3881e409b714cd27e6798ef020c514f0a"
checksum = "83a25cf98105baa966497416dbd42565ce3a8cf8dbfd59803ec9ad46f3126399"
dependencies = [
"cc",
"cmake",
@ -313,17 +313,6 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
[[package]]
name = "chacha20"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601"
dependencies = [
"cfg-if",
"cpufeatures 0.3.0",
"rand_core 0.10.0",
]
[[package]]
name = "chrono"
version = "0.4.44"
@ -467,16 +456,6 @@ dependencies = [
"url",
]
[[package]]
name = "core-foundation"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f"
dependencies = [
"core-foundation-sys",
"libc",
]
[[package]]
name = "core-foundation"
version = "0.10.1"
@ -502,15 +481,6 @@ dependencies = [
"libc",
]
[[package]]
name = "cpufeatures"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201"
dependencies = [
"libc",
]
[[package]]
name = "crc"
version = "3.4.0"
@ -973,7 +943,6 @@ dependencies = [
"cfg-if",
"libc",
"r-efi 6.0.0",
"rand_core 0.10.0",
"wasip2",
"wasip3",
]
@ -981,7 +950,7 @@ dependencies = [
[[package]]
name = "h2"
version = "0.4.13"
source = "git+https://github.com/deedy5/primp#b1d34a7b9fc7c24be515b1b74a469060b12fe137"
source = "git+https://github.com/0xMassi/webclaw-tls#fcbd389f90994fc1be1efdde1065713d0ef562d5"
dependencies = [
"atomic-waker",
"bytes",
@ -1081,7 +1050,7 @@ checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"
[[package]]
name = "hyper"
version = "1.8.1"
source = "git+https://github.com/deedy5/primp#b1d34a7b9fc7c24be515b1b74a469060b12fe137"
source = "git+https://github.com/0xMassi/webclaw-tls#fcbd389f90994fc1be1efdde1065713d0ef562d5"
dependencies = [
"atomic-waker",
"bytes",
@ -1121,7 +1090,7 @@ dependencies = [
[[package]]
name = "hyper-util"
version = "0.1.20"
source = "git+https://github.com/deedy5/primp#b1d34a7b9fc7c24be515b1b74a469060b12fe137"
source = "git+https://github.com/0xMassi/webclaw-tls#fcbd389f90994fc1be1efdde1065713d0ef562d5"
dependencies = [
"base64",
"bytes",
@ -1136,11 +1105,9 @@ dependencies = [
"percent-encoding",
"pin-project-lite",
"socket2",
"system-configuration",
"tokio",
"tower-service",
"tracing",
"windows-registry",
]
[[package]]
@ -1342,10 +1309,12 @@ dependencies = [
[[package]]
name = "js-sys"
version = "0.3.91"
version = "0.3.92"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c"
checksum = "cc4c90f45aa2e6eacbe8645f77fdea542ac97a494bcd117a67df9ff4d611f995"
dependencies = [
"cfg-if",
"futures-util",
"once_cell",
"wasm-bindgen",
]
@ -1502,22 +1471,6 @@ version = "2.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
[[package]]
name = "mime"
version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
[[package]]
name = "mime_guess"
version = "2.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e"
dependencies = [
"mime",
"unicase",
]
[[package]]
name = "minimal-lexical"
version = "0.2.1"
@ -1785,20 +1738,6 @@ dependencies = [
"syn",
]
[[package]]
name = "primp"
version = "1.2.0"
source = "git+https://github.com/deedy5/primp#b1d34a7b9fc7c24be515b1b74a469060b12fe137"
dependencies = [
"h2",
"http",
"rand 0.10.0",
"reqwest 0.13.2",
"rustls",
"url",
"webpki-roots",
]
[[package]]
name = "proc-macro2"
version = "1.0.106"
@ -1942,17 +1881,6 @@ dependencies = [
"rand_core 0.9.5",
]
[[package]]
name = "rand"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc266eb313df6c5c09c1c7b1fbe2510961e5bcd3add930c1e31f7ed9da0feff8"
dependencies = [
"chacha20",
"getrandom 0.4.2",
"rand_core 0.10.0",
]
[[package]]
name = "rand_chacha"
version = "0.3.1"
@ -1991,12 +1919,6 @@ dependencies = [
"getrandom 0.3.4",
]
[[package]]
name = "rand_core"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba"
[[package]]
name = "rangemap"
version = "1.7.1"
@ -2102,14 +2024,13 @@ dependencies = [
[[package]]
name = "reqwest"
version = "0.13.2"
source = "git+https://github.com/deedy5/primp#b1d34a7b9fc7c24be515b1b74a469060b12fe137"
source = "git+https://github.com/0xMassi/webclaw-tls#fcbd389f90994fc1be1efdde1065713d0ef562d5"
dependencies = [
"base64",
"bytes",
"cookie",
"cookie_store",
"futures-core",
"futures-util",
"h2",
"http",
"http-body",
@ -2119,7 +2040,6 @@ dependencies = [
"hyper-util",
"js-sys",
"log",
"mime_guess",
"percent-encoding",
"pin-project-lite",
"quinn",
@ -2127,18 +2047,15 @@ dependencies = [
"rustls-pki-types",
"serde",
"serde_json",
"serde_urlencoded",
"sync_wrapper",
"tokio",
"tokio-rustls",
"tokio-util",
"tower",
"tower-http",
"tower-service",
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
"wasm-streams",
"web-sys",
"webpki-roots",
]
@ -2221,9 +2138,9 @@ dependencies = [
[[package]]
name = "rustc-hash"
version = "2.1.1"
version = "2.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
[[package]]
name = "rustix"
@ -2241,7 +2158,7 @@ dependencies = [
[[package]]
name = "rustls"
version = "0.23.37"
source = "git+https://github.com/deedy5/primp#b1d34a7b9fc7c24be515b1b74a469060b12fe137"
source = "git+https://github.com/0xMassi/webclaw-tls#fcbd389f90994fc1be1efdde1065713d0ef562d5"
dependencies = [
"aws-lc-rs",
"brotli",
@ -2253,7 +2170,6 @@ dependencies = [
"rustls-webpki",
"subtle",
"zeroize",
"zlib-rs",
"zstd",
]
@ -2366,7 +2282,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"
dependencies = [
"bitflags",
"core-foundation 0.10.1",
"core-foundation",
"core-foundation-sys",
"libc",
"security-framework-sys",
@ -2489,7 +2405,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba"
dependencies = [
"cfg-if",
"cpufeatures 0.2.17",
"cpufeatures",
"digest",
]
@ -2632,27 +2548,6 @@ dependencies = [
"syn",
]
[[package]]
name = "system-configuration"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b"
dependencies = [
"bitflags",
"core-foundation 0.9.4",
"system-configuration-sys",
]
[[package]]
name = "system-configuration-sys"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4"
dependencies = [
"core-foundation-sys",
"libc",
]
[[package]]
name = "tempfile"
version = "3.27.0"
@ -2951,12 +2846,6 @@ version = "1.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb"
[[package]]
name = "unicase"
version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142"
[[package]]
name = "unicode-ident"
version = "1.0.24"
@ -3074,9 +2963,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen"
version = "0.2.114"
version = "0.2.115"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e"
checksum = "6523d69017b7633e396a89c5efab138161ed5aafcbc8d3e5c5a42ae38f50495a"
dependencies = [
"cfg-if",
"once_cell",
@ -3087,23 +2976,19 @@ dependencies = [
[[package]]
name = "wasm-bindgen-futures"
version = "0.4.64"
version = "0.4.65"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e9c5522b3a28661442748e09d40924dfb9ca614b21c00d3fd135720e48b67db8"
checksum = "2d1faf851e778dfa54db7cd438b70758eba9755cb47403f3496edd7c8fc212f0"
dependencies = [
"cfg-if",
"futures-util",
"js-sys",
"once_cell",
"wasm-bindgen",
"web-sys",
]
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.114"
version = "0.2.115"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6"
checksum = "4e3a6c758eb2f701ed3d052ff5737f5bfe6614326ea7f3bbac7156192dc32e67"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@ -3111,9 +2996,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.114"
version = "0.2.115"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3"
checksum = "921de2737904886b52bcbb237301552d05969a6f9c40d261eb0533c8b055fedf"
dependencies = [
"bumpalo",
"proc-macro2",
@ -3124,9 +3009,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.114"
version = "0.2.115"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16"
checksum = "a93e946af942b58934c604527337bad9ae33ba1d5c6900bbb41c2c07c2364a93"
dependencies = [
"unicode-ident",
]
@ -3153,19 +3038,6 @@ dependencies = [
"wasmparser",
]
[[package]]
name = "wasm-streams"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb"
dependencies = [
"futures-util",
"js-sys",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
]
[[package]]
name = "wasmparser"
version = "0.244.0"
@ -3180,9 +3052,9 @@ dependencies = [
[[package]]
name = "web-sys"
version = "0.3.91"
version = "0.3.92"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9"
checksum = "84cde8507f4d7cfcb1185b8cb5890c494ffea65edbe1ba82cfd63661c805ed94"
dependencies = [
"js-sys",
"wasm-bindgen",
@ -3200,11 +3072,10 @@ dependencies = [
[[package]]
name = "webclaw-cli"
version = "0.2.3"
version = "0.3.0"
dependencies = [
"clap",
"dotenvy",
"rand 0.8.5",
"regex",
"reqwest 0.12.28",
"serde_json",
@ -3220,10 +3091,9 @@ dependencies = [
[[package]]
name = "webclaw-core"
version = "0.2.3"
version = "0.3.0"
dependencies = [
"ego-tree",
"once_cell",
"regex",
"rquickjs",
"scraper",
@ -3238,10 +3108,9 @@ dependencies = [
[[package]]
name = "webclaw-fetch"
version = "0.2.3"
version = "0.3.0"
dependencies = [
"calamine",
"primp",
"quick-xml 0.37.5",
"rand 0.8.5",
"serde",
@ -3252,13 +3121,31 @@ dependencies = [
"tracing",
"url",
"webclaw-core",
"webclaw-http",
"webclaw-pdf",
"zip 2.4.2",
]
[[package]]
name = "webclaw-http"
version = "0.1.0"
source = "git+https://github.com/0xMassi/webclaw-tls#fcbd389f90994fc1be1efdde1065713d0ef562d5"
dependencies = [
"bytes",
"h2",
"http",
"reqwest 0.13.2",
"rustls",
"rustls-native-certs",
"tokio",
"tracing",
"url",
"webpki-roots",
]
[[package]]
name = "webclaw-llm"
version = "0.2.3"
version = "0.3.0"
dependencies = [
"async-trait",
"reqwest 0.12.28",
@ -3271,7 +3158,7 @@ dependencies = [
[[package]]
name = "webclaw-mcp"
version = "0.2.3"
version = "0.3.0"
dependencies = [
"dotenvy",
"reqwest 0.12.28",
@ -3291,7 +3178,7 @@ dependencies = [
[[package]]
name = "webclaw-pdf"
version = "0.2.3"
version = "0.3.0"
dependencies = [
"pdf-extract",
"thiserror",
@ -3354,17 +3241,6 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
[[package]]
name = "windows-registry"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720"
dependencies = [
"windows-link",
"windows-result",
"windows-strings",
]
[[package]]
name = "windows-result"
version = "0.4.1"
@ -3667,18 +3543,18 @@ dependencies = [
[[package]]
name = "zerocopy"
version = "0.8.47"
version = "0.8.48"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "efbb2a062be311f2ba113ce66f697a4dc589f85e78a4aea276200804cea0ed87"
checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
version = "0.8.47"
version = "0.8.48"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e8bc7269b54418e7aeeef514aa68f8690b8c0489a06b0136e5f57c4c5ccab89"
checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4"
dependencies = [
"proc-macro2",
"quote",