mirror of
https://github.com/0xMassi/webclaw.git
synced 2026-06-16 23:45:13 +02:00
de899ab3ba
OSS surfaces can now search without the hosted webclaw API. New webclaw-fetch::search() calls Serper.dev directly with a user-supplied key and optionally fetches + extracts the result pages. Wired into the CLI (webclaw search, --serper-key / SERPER_API_KEY), the MCP search tool (local-first when SERPER_API_KEY is set, cloud fallback otherwise), and the OSS reference server (POST /v1/search). Adds futures for concurrent result page scraping.
38 lines
1.1 KiB
TOML
38 lines
1.1 KiB
TOML
[package]
|
|
name = "webclaw-fetch"
|
|
description = "HTTP client with browser TLS fingerprint impersonation via wreq"
|
|
version.workspace = true
|
|
edition.workspace = true
|
|
rust-version.workspace = true
|
|
license.workspace = true
|
|
|
|
[lints]
|
|
workspace = true
|
|
|
|
[dependencies]
|
|
webclaw-core = { workspace = true }
|
|
webclaw-pdf = { path = "../webclaw-pdf" }
|
|
serde = { workspace = true }
|
|
thiserror = { workspace = true }
|
|
tracing = { workspace = true }
|
|
tokio = { workspace = true }
|
|
async-trait = "0.1"
|
|
wreq = { version = "6.0.0-rc.28", features = ["cookies", "gzip", "brotli", "zstd", "deflate"] }
|
|
wreq-util = "3.0.0-rc.10"
|
|
http = "1"
|
|
bytes = "1"
|
|
url = "2"
|
|
rand = "0.8"
|
|
quick-xml = { version = "0.37", features = ["serde"] }
|
|
regex = "1"
|
|
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
|
|
serde_json.workspace = true
|
|
calamine = "0.34"
|
|
zip = "2"
|
|
flate2 = "1"
|
|
# Already in the dependency tree (transitive); used directly here for
|
|
# `join_all` to drive bounded-concurrent result-page scrapes in search.
|
|
futures = { version = "0.3", default-features = false }
|
|
|
|
[dev-dependencies]
|
|
tempfile = "3"
|