apunkt/webclaw
1
0
Fork 0
mirror of https://github.com/0xMassi/webclaw.git synced 2026-07-20 06:51:02 +02:00
Code Issues Projects Releases Packages Wiki Activity

Initial release: webclaw v0.1.0 — web content extraction for LLMs

Browse source 
CLI + MCP server for extracting clean, structured content from any URL.
6 Rust crates, 10 MCP tools, TLS fingerprinting, 5 output formats.

MIT Licensed | https://webclaw.io
This commit is contained in:
Valerio 2026-03-23 18:31:11 +01:00
commit c99ec684fa
79 changed files with 24074 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

96
crates/webclaw-fetch/src/browser.rs Normal file
View file

@ -0,0 +1,96 @@
/// Browser fingerprint selection and rotation.
/// Maps our simple `BrowserProfile` enum to primp's impersonation profiles.
use primp::{Impersonate, ImpersonateOS};
/// Which browser identity to present at the TLS/HTTP layer.
#[derive(Debug, Clone, Default)]
pub enum BrowserProfile {
#[default]
Chrome,
Firefox,
/// Randomly pick from all available profiles on each request.
Random,
}
/// A complete impersonation profile: browser + OS.
#[derive(Debug, Clone)]
pub struct ImpersonateProfile {
pub browser: Impersonate,
pub os: ImpersonateOS,
}
/// All Chrome profiles we ship, newest first.
pub fn chrome_profiles() -> Vec<ImpersonateProfile> {
vec![
ImpersonateProfile {
browser: Impersonate::ChromeV145,
os: ImpersonateOS::Windows,
},
ImpersonateProfile {
browser: Impersonate::ChromeV145,
os: ImpersonateOS::MacOS,
},
ImpersonateProfile {
browser: Impersonate::ChromeV144,
os: ImpersonateOS::Windows,
},
ImpersonateProfile {
browser: Impersonate::ChromeV144,
os: ImpersonateOS::Linux,
},
]
}
/// All Firefox profiles we ship, newest first.
pub fn firefox_profiles() -> Vec<ImpersonateProfile> {
vec![
ImpersonateProfile {
browser: Impersonate::FirefoxV146,
os: ImpersonateOS::Windows,
},
ImpersonateProfile {
browser: Impersonate::FirefoxV146,
os: ImpersonateOS::Linux,
},
ImpersonateProfile {
browser: Impersonate::FirefoxV140,
os: ImpersonateOS::Windows,
},
]
}
/// Safari + Edge + Opera profiles for maximum diversity in Random mode.
pub fn extra_profiles() -> Vec<ImpersonateProfile> {
vec![
ImpersonateProfile {
browser: Impersonate::SafariV18_5,
os: ImpersonateOS::MacOS,
},
ImpersonateProfile {
browser: Impersonate::SafariV26,
os: ImpersonateOS::MacOS,
},
ImpersonateProfile {
browser: Impersonate::EdgeV145,
os: ImpersonateOS::Windows,
},
ImpersonateProfile {
browser: Impersonate::OperaV127,
os: ImpersonateOS::Windows,
},
]
}
pub fn latest_chrome() -> ImpersonateProfile {
ImpersonateProfile {
browser: Impersonate::ChromeV145,
os: ImpersonateOS::Windows,
}
}
pub fn latest_firefox() -> ImpersonateProfile {
ImpersonateProfile {
browser: Impersonate::FirefoxV146,
os: ImpersonateOS::Windows,
}
}