mirror of
https://github.com/0xMassi/webclaw.git
synced 2026-06-10 22:45:13 +02:00
Compare commits
14 commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
df7336d55b | ||
|
|
acd3021f38 | ||
|
|
bcc58dbadd | ||
|
|
8015de7db5 | ||
|
|
be64409d62 | ||
|
|
2773474984 | ||
|
|
7dfa180e86 | ||
|
|
598f319bf3 | ||
|
|
fae2766db1 | ||
|
|
d0909a25e3 | ||
|
|
499345046c | ||
|
|
d0d7b835f2 | ||
|
|
6519ac2a8b | ||
|
|
14ded4b99e |
19 changed files with 449 additions and 223 deletions
BIN
.github/banner.png
vendored
BIN
.github/banner.png
vendored
Binary file not shown.
|
Before Width: | Height: | Size: 44 KiB After Width: | Height: | Size: 48 KiB |
74
.github/workflows/release.yml
vendored
74
.github/workflows/release.yml
vendored
|
|
@ -3,6 +3,15 @@ name: Release
|
|||
on:
|
||||
push:
|
||||
tags: ["v*"]
|
||||
# Manual re-publish of the Docker image for an existing release, without
|
||||
# rebuilding binaries or cutting a new version. Runs only the docker (+
|
||||
# homebrew) jobs against the given tag's already-published release assets.
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tag:
|
||||
description: "Existing release tag to (re)build + push the Docker image for, e.g. v0.6.9"
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
|
@ -12,6 +21,9 @@ env:
|
|||
|
||||
jobs:
|
||||
build:
|
||||
# Binaries are only built when a tag is pushed. A manual dispatch reuses
|
||||
# the existing release's binaries, so it skips this job entirely.
|
||||
if: github.event_name == 'push'
|
||||
permissions:
|
||||
contents: read
|
||||
name: Build ${{ matrix.target }}
|
||||
|
|
@ -105,6 +117,7 @@ jobs:
|
|||
|
||||
release:
|
||||
name: Release
|
||||
if: github.event_name == 'push'
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
|
|
@ -137,6 +150,10 @@ jobs:
|
|||
docker:
|
||||
name: Docker
|
||||
needs: release
|
||||
# Runs after a successful release on tag push, or standalone via
|
||||
# workflow_dispatch to (re)publish an existing tag's image. `always()` lets
|
||||
# it run even though `release` is skipped on a manual dispatch.
|
||||
if: ${{ always() && (github.event_name == 'workflow_dispatch' || needs.release.result == 'success') }}
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
|
|
@ -156,49 +173,48 @@ jobs:
|
|||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
# Download pre-built binaries for both architectures
|
||||
# The pushed tag, or the workflow_dispatch input for a manual re-publish.
|
||||
- name: Resolve tag
|
||||
id: tag
|
||||
run: echo "tag=${{ github.event.inputs.tag || github.ref_name }}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Download pre-built binaries into TARGETARCH-named dirs (amd64/arm64) so
|
||||
# a single multi-platform build picks the matching binary per platform.
|
||||
- name: Download release binaries
|
||||
run: |
|
||||
tag="${GITHUB_REF#refs/tags/}"
|
||||
tag="${{ steps.tag.outputs.tag }}"
|
||||
declare -A arch=( [x86_64-unknown-linux-gnu]=amd64 [aarch64-unknown-linux-gnu]=arm64 )
|
||||
for target in x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu; do
|
||||
dir="webclaw-${tag}-${target}"
|
||||
curl -sSL "https://github.com/0xMassi/webclaw/releases/download/${tag}/${dir}.tar.gz" -o "${target}.tar.gz"
|
||||
tar xzf "${target}.tar.gz"
|
||||
mkdir -p "binaries-${target}"
|
||||
cp "${dir}/webclaw" "binaries-${target}/webclaw"
|
||||
cp "${dir}/webclaw-mcp" "binaries-${target}/webclaw-mcp"
|
||||
cp "${dir}/webclaw-server" "binaries-${target}/webclaw-server"
|
||||
chmod +x "binaries-${target}"/*
|
||||
a="${arch[$target]}"
|
||||
mkdir -p "binaries-${a}"
|
||||
cp "${dir}/webclaw" "${dir}/webclaw-mcp" "${dir}/webclaw-server" "binaries-${a}/"
|
||||
chmod +x "binaries-${a}"/*
|
||||
done
|
||||
ls -laR binaries-*/
|
||||
|
||||
# Build per-arch images with plain docker build (no buildx manifest nesting)
|
||||
# One atomic multi-platform build + push. buildx assembles a single
|
||||
# manifest list and pushes it in one shot, so there is no separate
|
||||
# `imagetools create` step to race GHCR's read-after-write (that is what
|
||||
# failed before: "v0.6.9-arm64: not found"). Provenance/SBOM attestations
|
||||
# are disabled so each platform entry stays a plain image manifest.
|
||||
- name: Build and push
|
||||
run: |
|
||||
tag="${GITHUB_REF#refs/tags/}"
|
||||
|
||||
# amd64
|
||||
docker build -f Dockerfile.ci --build-arg BINARY_DIR=binaries-x86_64-unknown-linux-gnu \
|
||||
--platform linux/amd64 -t ghcr.io/0xmassi/webclaw:${tag}-amd64 --push .
|
||||
|
||||
# arm64
|
||||
docker build -f Dockerfile.ci --build-arg BINARY_DIR=binaries-aarch64-unknown-linux-gnu \
|
||||
--platform linux/arm64 -t ghcr.io/0xmassi/webclaw:${tag}-arm64 --push .
|
||||
|
||||
# Multi-arch manifest
|
||||
docker manifest create ghcr.io/0xmassi/webclaw:${tag} \
|
||||
ghcr.io/0xmassi/webclaw:${tag}-amd64 \
|
||||
ghcr.io/0xmassi/webclaw:${tag}-arm64
|
||||
docker manifest push ghcr.io/0xmassi/webclaw:${tag}
|
||||
|
||||
docker manifest create ghcr.io/0xmassi/webclaw:latest \
|
||||
ghcr.io/0xmassi/webclaw:${tag}-amd64 \
|
||||
ghcr.io/0xmassi/webclaw:${tag}-arm64
|
||||
docker manifest push ghcr.io/0xmassi/webclaw:latest
|
||||
tag="${{ steps.tag.outputs.tag }}"
|
||||
docker buildx build -f Dockerfile.ci \
|
||||
--platform linux/amd64,linux/arm64 \
|
||||
--provenance=false --sbom=false \
|
||||
-t "ghcr.io/0xmassi/webclaw:${tag}" \
|
||||
-t ghcr.io/0xmassi/webclaw:latest \
|
||||
--push .
|
||||
|
||||
homebrew:
|
||||
name: Update Homebrew
|
||||
needs: [release, docker]
|
||||
# Runs once Docker succeeds, on both tag push and manual re-publish.
|
||||
if: ${{ always() && needs.docker.result == 'success' }}
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
|
|
@ -207,7 +223,7 @@ jobs:
|
|||
env:
|
||||
COMMITTER_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }}
|
||||
run: |
|
||||
tag="${GITHUB_REF#refs/tags/}"
|
||||
tag="${{ github.event.inputs.tag || github.ref_name }}"
|
||||
base="https://github.com/0xMassi/webclaw/releases/download/${tag}"
|
||||
|
||||
# Download all tarballs (Linux + macOS) and compute SHAs
|
||||
|
|
|
|||
26
CHANGELOG.md
26
CHANGELOG.md
|
|
@ -3,6 +3,32 @@
|
|||
All notable changes to webclaw are documented here.
|
||||
Format follows [Keep a Changelog](https://keepachangelog.com/).
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [0.6.9] - 2026-06-10
|
||||
|
||||
### Fixed
|
||||
- The multi-arch Docker image (linux/amd64 + linux/arm64) now publishes reliably on each release. The build moved to Buildx so registry pushes no longer fail intermittently, and the Homebrew formula update that depends on it is no longer skipped.
|
||||
|
||||
## [0.6.8] - 2026-06-10
|
||||
|
||||
### Fixed
|
||||
- Pages with multibyte text (accented or CJK characters) no longer panic or get mangled during extraction. API-endpoint discovery now cuts oversized scripts on a character boundary instead of crashing mid-character, and structured-data parsing preserves non-ASCII string values instead of turning them into mojibake.
|
||||
- LLM error messages from a provider no longer panic when the error body contains multibyte characters near the truncation point.
|
||||
- LLM provider requests now have explicit connect and overall timeouts, so a stalled or unreachable provider fails fast instead of hanging.
|
||||
- Batch extraction in the MCP server no longer aborts the whole batch when a single URL fails to resolve; bad URLs are reported as individual per-URL errors and the rest still run.
|
||||
- CLI crawl and batch runs now wait for the completion webhook to actually send before exiting, replacing a fixed delay that could cut the request off or waste time.
|
||||
- Homepage warm-up requests now include the port for hosts on a non-default port, so those sites are warmed correctly.
|
||||
|
||||
---
|
||||
|
||||
## [0.6.7] — 2026-06-09
|
||||
|
||||
### Changed
|
||||
- Updated the HTTP/TLS engine (wreq 6.0.0-rc.29, wreq-util 3.0.0-rc.12). This pulls in upstream robustness fixes: no more panic on responses with non-UTF8 header values, a fix for short reads when decoding large compressed bodies, and the TCP nodelay setting is restored. Browser TLS fingerprints are unchanged.
|
||||
|
||||
---
|
||||
|
||||
## [0.6.6] — 2026-06-09
|
||||
|
||||
### Added
|
||||
|
|
|
|||
221
Cargo.lock
generated
221
Cargo.lock
generated
|
|
@ -28,18 +28,6 @@ dependencies = [
|
|||
"cpufeatures",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ahash"
|
||||
version = "0.8.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"once_cell",
|
||||
"version_check",
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
version = "1.1.4"
|
||||
|
|
@ -64,6 +52,12 @@ dependencies = [
|
|||
"alloc-no-stdlib",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "allocator-api2"
|
||||
version = "0.2.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
|
||||
|
||||
[[package]]
|
||||
name = "android_system_properties"
|
||||
version = "0.1.5"
|
||||
|
|
@ -272,9 +266,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.11.0"
|
||||
version = "2.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af"
|
||||
checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8"
|
||||
|
||||
[[package]]
|
||||
name = "block-buffer"
|
||||
|
|
@ -285,31 +279,6 @@ dependencies = [
|
|||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "boring-sys2"
|
||||
version = "5.0.0-alpha.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "455d79965f5155dcc88a7abce112c3590883889131b799beda10bf9a813ed669"
|
||||
dependencies = [
|
||||
"bindgen",
|
||||
"cmake",
|
||||
"fs_extra",
|
||||
"fslock",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "boring2"
|
||||
version = "5.0.0-alpha.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "183ccc3854411c035410dcdbffafca62084f3a6c33f013c77e83c025d2a08a28"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"boring-sys2",
|
||||
"foreign-types",
|
||||
"libc",
|
||||
"openssl-macros",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "brotli"
|
||||
version = "8.0.2"
|
||||
|
|
@ -331,6 +300,31 @@ dependencies = [
|
|||
"alloc-stdlib",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "btls"
|
||||
version = "0.5.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2c5e60b8c8d282c86360cab651ded04ab0335a7b5390c8d34145cbeab8cacf5f"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"btls-sys",
|
||||
"foreign-types",
|
||||
"libc",
|
||||
"openssl-macros",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "btls-sys"
|
||||
version = "0.5.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b1b8638a2e1c38a5ae4efa90ae57e643baec35a30d03fc5b399b893adc4954b"
|
||||
dependencies = [
|
||||
"bindgen",
|
||||
"cmake",
|
||||
"fs_extra",
|
||||
"fslock",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bumpalo"
|
||||
version = "3.20.2"
|
||||
|
|
@ -865,6 +859,12 @@ version = "0.1.5"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
|
||||
|
||||
[[package]]
|
||||
name = "foldhash"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
|
||||
|
||||
[[package]]
|
||||
name = "foreign-types"
|
||||
version = "0.5.0"
|
||||
|
|
@ -1089,19 +1089,13 @@ version = "0.3.3"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.13.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e"
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.15.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
|
||||
dependencies = [
|
||||
"foldhash",
|
||||
"foldhash 0.1.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1110,6 +1104,17 @@ version = "0.16.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.17.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
|
||||
dependencies = [
|
||||
"allocator-api2",
|
||||
"equivalent",
|
||||
"foldhash 0.2.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.5.0"
|
||||
|
|
@ -1172,9 +1177,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "http2"
|
||||
version = "0.5.15"
|
||||
version = "0.5.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c45c6490693ee8a8d0d95fdbdf76fead9fb87548f7894137259a7c6d22821948"
|
||||
checksum = "569ef7a780e853c4e1768f58a3c8168193b82cdcbab66638a0b1c6583ec5995e"
|
||||
dependencies = [
|
||||
"atomic-waker",
|
||||
"bytes",
|
||||
|
|
@ -1183,7 +1188,6 @@ dependencies = [
|
|||
"futures-sink",
|
||||
"http",
|
||||
"indexmap",
|
||||
"parking_lot",
|
||||
"slab",
|
||||
"smallvec",
|
||||
"tokio",
|
||||
|
|
@ -1495,9 +1499,9 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
|
|||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.183"
|
||||
version = "0.2.186"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d"
|
||||
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
|
||||
|
||||
[[package]]
|
||||
name = "libloading"
|
||||
|
|
@ -1563,6 +1567,15 @@ dependencies = [
|
|||
"weezl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lru"
|
||||
version = "0.18.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a860605968fce16869fd239cf4237a82f3ac470723415db603b0e8b6c8d4fb9"
|
||||
dependencies = [
|
||||
"hashbrown 0.17.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lru-slab"
|
||||
version = "0.1.2"
|
||||
|
|
@ -2375,17 +2388,6 @@ dependencies = [
|
|||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "schnellru"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "356285bbf17bea63d9e52e96bd18f039672ac92b55b8cb997d6162a2a37d1649"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"cfg-if",
|
||||
"hashbrown 0.13.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "scopeguard"
|
||||
version = "1.2.0"
|
||||
|
|
@ -2779,9 +2781,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
|
|||
|
||||
[[package]]
|
||||
name = "tokio"
|
||||
version = "1.50.0"
|
||||
version = "1.52.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d"
|
||||
checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"libc",
|
||||
|
|
@ -2795,20 +2797,20 @@ dependencies = [
|
|||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-boring2"
|
||||
version = "5.0.0-alpha.13"
|
||||
name = "tokio-btls"
|
||||
version = "0.5.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0f81df1210d791f31d72d840de8fbd80b9c3cb324956523048b1413e2bd55756"
|
||||
checksum = "2e1fd638ec35427faf3b8f412e0fdd6fae76591d79dba40f38fa667d22bc44dd"
|
||||
dependencies = [
|
||||
"boring2",
|
||||
"btls",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-macros"
|
||||
version = "2.6.1"
|
||||
version = "2.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c"
|
||||
checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
|
@ -3219,7 +3221,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "webclaw-cli"
|
||||
version = "0.6.6"
|
||||
version = "0.6.9"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"dotenvy",
|
||||
|
|
@ -3240,7 +3242,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "webclaw-core"
|
||||
version = "0.6.6"
|
||||
version = "0.6.9"
|
||||
dependencies = [
|
||||
"ego-tree",
|
||||
"once_cell",
|
||||
|
|
@ -3258,11 +3260,12 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "webclaw-fetch"
|
||||
version = "0.6.6"
|
||||
version = "0.6.9"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"calamine",
|
||||
"futures-util",
|
||||
"http",
|
||||
"quick-xml 0.37.5",
|
||||
"rand 0.8.5",
|
||||
|
|
@ -3284,7 +3287,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "webclaw-llm"
|
||||
version = "0.6.6"
|
||||
version = "0.6.9"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"reqwest",
|
||||
|
|
@ -3297,7 +3300,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "webclaw-mcp"
|
||||
version = "0.6.6"
|
||||
version = "0.6.9"
|
||||
dependencies = [
|
||||
"dirs",
|
||||
"dotenvy",
|
||||
|
|
@ -3317,7 +3320,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "webclaw-pdf"
|
||||
version = "0.6.6"
|
||||
version = "0.6.9"
|
||||
dependencies = [
|
||||
"pdf-extract",
|
||||
"thiserror",
|
||||
|
|
@ -3326,7 +3329,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "webclaw-server"
|
||||
version = "0.6.6"
|
||||
version = "0.6.9"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"axum",
|
||||
|
|
@ -3347,9 +3350,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "webpki-root-certs"
|
||||
version = "1.0.6"
|
||||
version = "1.0.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "804f18a4ac2676ffb4e8b5b5fa9ae38af06df08162314f96a68d2a363e21a8ca"
|
||||
checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c"
|
||||
dependencies = [
|
||||
"rustls-pki-types",
|
||||
]
|
||||
|
|
@ -3696,17 +3699,14 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "wreq"
|
||||
version = "6.0.0-rc.28"
|
||||
version = "6.0.0-rc.29"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f79937f6c4df65b3f6f78715b9de2977afe9ee3b3436483c7949a24511e25935"
|
||||
checksum = "3f0eba5f5814a94e5f1a99156f187133464e525b66bdbc69a9627d46530af2e1"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"boring2",
|
||||
"brotli",
|
||||
"btls",
|
||||
"btls-sys",
|
||||
"bytes",
|
||||
"cookie",
|
||||
"flate2",
|
||||
"futures-channel",
|
||||
"futures-util",
|
||||
"http",
|
||||
"http-body",
|
||||
|
|
@ -3715,29 +3715,64 @@ dependencies = [
|
|||
"httparse",
|
||||
"ipnet",
|
||||
"libc",
|
||||
"lru",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"schnellru",
|
||||
"smallvec",
|
||||
"socket2",
|
||||
"sync_wrapper",
|
||||
"tokio",
|
||||
"tokio-boring2",
|
||||
"tokio-btls",
|
||||
"tokio-util",
|
||||
"tower",
|
||||
"tower-http",
|
||||
"url",
|
||||
"want",
|
||||
"webpki-root-certs",
|
||||
"zstd",
|
||||
"wreq-proto",
|
||||
"wreq-rt",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wreq-proto"
|
||||
version = "0.2.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a43942f024bb303f1042c9aa3c87fa1d9149f507c65db6e5220a11ccdb207387"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-channel",
|
||||
"futures-util",
|
||||
"http",
|
||||
"http-body",
|
||||
"http2",
|
||||
"httparse",
|
||||
"pin-project-lite",
|
||||
"smallvec",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"want",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wreq-rt"
|
||||
version = "0.2.2-rc.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "99e9bce67a3fa3dd3f1503f066d86661c9caf399a763d3bd184da7afaf886c8b"
|
||||
dependencies = [
|
||||
"pin-project-lite",
|
||||
"tokio",
|
||||
"wreq-proto",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wreq-util"
|
||||
version = "3.0.0-rc.10"
|
||||
version = "3.0.0-rc.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6c6bbe24d28beb9ceb58b514bd6a613c759d3b706f768b9d2950d5d35b543c04"
|
||||
checksum = "baa5d2ab72139256916ca352a3d05c53d74e1dd360052eb5ba7691033c417c65"
|
||||
dependencies = [
|
||||
"brotli",
|
||||
"flate2",
|
||||
"typed-builder",
|
||||
"wreq",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ resolver = "2"
|
|||
members = ["crates/*"]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.6.6"
|
||||
version = "0.6.9"
|
||||
edition = "2024"
|
||||
license = "AGPL-3.0"
|
||||
repository = "https://github.com/0xMassi/webclaw"
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
# Slim runtime image — uses pre-built binaries from the release.
|
||||
# The full Dockerfile (multi-stage Rust build) is for local development.
|
||||
# CI uses this to avoid 60+ min QEMU cross-compilation.
|
||||
ARG BINARY_DIR=binaries
|
||||
|
||||
FROM ubuntu:24.04
|
||||
|
||||
|
|
@ -10,10 +9,13 @@ FROM ubuntu:24.04
|
|||
# CI runners and breaks the multi-arch release build. No build-time network.
|
||||
COPY --from=gcr.io/distroless/static-debian12 /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||
|
||||
ARG BINARY_DIR
|
||||
COPY ${BINARY_DIR}/webclaw /usr/local/bin/webclaw
|
||||
COPY ${BINARY_DIR}/webclaw-mcp /usr/local/bin/webclaw-mcp
|
||||
COPY ${BINARY_DIR}/webclaw-server /usr/local/bin/webclaw-server
|
||||
# TARGETARCH (amd64 / arm64) is provided automatically by buildx for each
|
||||
# target platform, so one multi-platform build copies the matching binaries.
|
||||
# The release workflow stages them in binaries-amd64 / binaries-arm64.
|
||||
ARG TARGETARCH
|
||||
COPY binaries-${TARGETARCH}/webclaw /usr/local/bin/webclaw
|
||||
COPY binaries-${TARGETARCH}/webclaw-mcp /usr/local/bin/webclaw-mcp
|
||||
COPY binaries-${TARGETARCH}/webclaw-server /usr/local/bin/webclaw-server
|
||||
|
||||
# Default REST API port when running `webclaw-server` inside the container.
|
||||
EXPOSE 3000
|
||||
|
|
@ -25,8 +27,9 @@ ENV WEBCLAW_HOST=0.0.0.0
|
|||
|
||||
# Entrypoint shim: forwards webclaw args/URL to the binary, but exec's other
|
||||
# commands directly so this image can be used as a FROM base with custom CMD.
|
||||
COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
|
||||
RUN chmod +x /usr/local/bin/docker-entrypoint.sh
|
||||
# `--chmod` sets the bit at copy time so the build needs no in-container `RUN`
|
||||
# (and thus no QEMU emulation for the arm64 platform).
|
||||
COPY --chmod=755 docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["docker-entrypoint.sh"]
|
||||
CMD ["webclaw", "--help"]
|
||||
|
|
|
|||
19
README.md
19
README.md
|
|
@ -142,7 +142,7 @@ webclaw https://docs.rust-lang.org --crawl --depth 2 --max-pages 50
|
|||
- [HTML to Markdown for RAG](examples/html-to-markdown-rag/)
|
||||
- [Firecrawl-compatible API](examples/firecrawl-compatible-api/)
|
||||
- [MCP web scraping](examples/mcp-web-scraping/)
|
||||
- [Proxy-backed crawling](examples/proxy-backed-crawling/)
|
||||
- [Proxy-backed crawling with ColdProxy](examples/proxy-backed-crawling/)
|
||||
- [Cloudflare diagnostics](examples/cloudflare-diagnostics/)
|
||||
|
||||
### Extract brand assets
|
||||
|
|
@ -401,6 +401,8 @@ Please remove secrets, cookies, private tokens, and customer data from logs befo
|
|||
residential IPv6, and datacenter IPv6 proxy infrastructure across 195+ countries for public data
|
||||
collection, regional testing, monitoring, and web scraping workflows. Explore
|
||||
<a href="https://coldproxy.com/">ColdProxy</a>'s latest plans and available offers directly on the website.
|
||||
See the <a href="examples/proxy-backed-crawling/#using-coldproxy">proxy-backed crawling guide</a>
|
||||
for a hands-on walkthrough of wiring ColdProxy into webclaw.
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
|
@ -410,6 +412,21 @@ Please remove secrets, cookies, private tokens, and customer data from logs befo
|
|||
## Studio Partners
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<td width="340" align="center">
|
||||
<a href="https://go.nodemaven.com/webclaw">
|
||||
<img src="./assets/sponsors/nodemaven-banner.png" alt="NodeMaven" width="300" />
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
<strong>NodeMaven</strong> is the most reliable proxy provider with the highest-quality IPs on the market.
|
||||
Best solution for automation, web scraping, SEO research, and social media management: 99.9% uptime,
|
||||
sticky sessions up to 7 days, IP filtering (all proxies under a 97% fraud score), no KYC, and cashback up
|
||||
to 10% on traffic. Use <code>WEBCLAW35</code> for 35% off Mobile and Residential proxies, or
|
||||
<code>WEBCLAW40</code> for 40% off ISP (Static) proxies at
|
||||
<a href="https://go.nodemaven.com/webclaw">NodeMaven</a>.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td width="340" align="center">
|
||||
<a href="https://quantumproxies.net/?utm_source=webclaw&utm_medium=github&utm_campaign=sponsor">
|
||||
|
|
|
|||
BIN
assets/sponsors/nodemaven-banner.png
Normal file
BIN
assets/sponsors/nodemaven-banner.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 302 KiB |
|
|
@ -1548,7 +1548,7 @@ async fn run_crawl(cli: &Cli) -> Result<(), String> {
|
|||
// Fire webhook on crawl complete
|
||||
if let Some(ref webhook_url) = cli.webhook {
|
||||
let urls: Vec<&str> = result.pages.iter().map(|p| p.url.as_str()).collect();
|
||||
fire_webhook(
|
||||
let handle = fire_webhook(
|
||||
webhook_url,
|
||||
&serde_json::json!({
|
||||
"event": "crawl_complete",
|
||||
|
|
@ -1559,8 +1559,8 @@ async fn run_crawl(cli: &Cli) -> Result<(), String> {
|
|||
"urls": urls,
|
||||
}),
|
||||
);
|
||||
// Brief pause so the async webhook has time to fire
|
||||
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
|
||||
// Wait for the webhook to finish so the process doesn't exit mid-send.
|
||||
let _ = handle.await;
|
||||
}
|
||||
|
||||
if result.errors > 0 {
|
||||
|
|
@ -1658,7 +1658,7 @@ async fn run_batch(cli: &Cli, entries: &[(String, Option<String>)]) -> Result<()
|
|||
// Fire webhook on batch complete
|
||||
if let Some(ref webhook_url) = cli.webhook {
|
||||
let urls: Vec<&str> = results.iter().map(|r| r.url.as_str()).collect();
|
||||
fire_webhook(
|
||||
let handle = fire_webhook(
|
||||
webhook_url,
|
||||
&serde_json::json!({
|
||||
"event": "batch_complete",
|
||||
|
|
@ -1668,7 +1668,7 @@ async fn run_batch(cli: &Cli, entries: &[(String, Option<String>)]) -> Result<()
|
|||
"urls": urls,
|
||||
}),
|
||||
);
|
||||
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
|
||||
let _ = handle.await;
|
||||
}
|
||||
|
||||
if errors > 0 {
|
||||
|
|
@ -1742,9 +1742,12 @@ async fn spawn_on_change(cmd: &str, stdin_payload: &[u8]) {
|
|||
}
|
||||
}
|
||||
|
||||
/// Fire a webhook POST with a JSON payload. Non-blocking — errors logged to stderr.
|
||||
/// Auto-detects Discord and Slack webhook URLs and wraps the payload accordingly.
|
||||
fn fire_webhook(url: &str, payload: &serde_json::Value) {
|
||||
/// Fire a webhook POST with a JSON payload. Spawns the send on a background task
|
||||
/// and returns its `JoinHandle` so callers that need delivery (e.g. one-shot
|
||||
/// crawl/batch runs that exit immediately after) can `.await` it; long-running
|
||||
/// loops can drop the handle and let it run fire-and-forget. Errors are logged
|
||||
/// to stderr. Auto-detects Discord and Slack webhook URLs and wraps the payload.
|
||||
fn fire_webhook(url: &str, payload: &serde_json::Value) -> tokio::task::JoinHandle<()> {
|
||||
let url = url.to_string();
|
||||
let is_discord = url.contains("discord.com/api/webhooks");
|
||||
let is_slack = url.contains("hooks.slack.com");
|
||||
|
|
@ -1806,7 +1809,7 @@ fn fire_webhook(url: &str, payload: &serde_json::Value) {
|
|||
},
|
||||
Err(e) => eprintln!("[webhook] client error: {e}"),
|
||||
}
|
||||
});
|
||||
})
|
||||
}
|
||||
|
||||
async fn run_watch(cli: &Cli, urls: &[String]) -> Result<(), String> {
|
||||
|
|
@ -2318,7 +2321,7 @@ async fn run_batch_llm(cli: &Cli, entries: &[(String, Option<String>)]) -> Resul
|
|||
eprintln!("Processed {total} URLs ({ok} ok, {errors} errors)");
|
||||
|
||||
if let Some(ref webhook_url) = cli.webhook {
|
||||
fire_webhook(
|
||||
let handle = fire_webhook(
|
||||
webhook_url,
|
||||
&serde_json::json!({
|
||||
"event": "batch_llm_complete",
|
||||
|
|
@ -2327,7 +2330,7 @@ async fn run_batch_llm(cli: &Cli, entries: &[(String, Option<String>)]) -> Resul
|
|||
"errors": errors,
|
||||
}),
|
||||
);
|
||||
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
|
||||
let _ = handle.await;
|
||||
}
|
||||
|
||||
if errors > 0 {
|
||||
|
|
|
|||
|
|
@ -233,7 +233,13 @@ pub fn extract_endpoints(
|
|||
}
|
||||
let slice = if text.len() > *budget {
|
||||
*truncated = true;
|
||||
&text[..*budget]
|
||||
// Snap the cut to a UTF-8 char boundary so non-ASCII content
|
||||
// (multibyte codepoints straddling the budget) can't panic.
|
||||
let mut cut = (*budget).min(text.len());
|
||||
while cut > 0 && !text.is_char_boundary(cut) {
|
||||
cut -= 1;
|
||||
}
|
||||
&text[..cut]
|
||||
} else {
|
||||
text
|
||||
};
|
||||
|
|
@ -512,4 +518,16 @@ mod tests {
|
|||
);
|
||||
assert!(r.hosts.iter().any(|h| h == "pubapi.ticketmaster.co.uk"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scan_truncation_at_non_ascii_boundary_does_not_panic() {
|
||||
// A bundle just over the scan budget, padded with a multibyte char
|
||||
// ('é' is 2 bytes) so the cut lands mid-codepoint. The old
|
||||
// `&text[..budget]` slice panicked here; the boundary snap must not.
|
||||
let pad = "é".repeat(MAX_SCAN_BYTES); // ~2× budget in bytes
|
||||
let bundle = format!("{pad} fetch(\"/api/x\")");
|
||||
let bundles = vec![("big.js".to_string(), bundle)];
|
||||
let r = extract_endpoints("<html></html>", "https://example.com/", &bundles);
|
||||
assert!(r.truncated, "oversized bundle should mark truncated");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -178,7 +178,12 @@ pub fn extract_sveltekit(html: &str) -> Vec<Value> {
|
|||
/// Preserves already-quoted keys and string values.
|
||||
fn js_literal_to_json(input: &str) -> String {
|
||||
let bytes = input.as_bytes();
|
||||
let mut out = String::with_capacity(input.len() + input.len() / 10);
|
||||
// Accumulate raw bytes, not `byte as char`. The input is valid UTF-8 and we
|
||||
// only ever copy its bytes verbatim or insert ASCII quotes, so the result is
|
||||
// guaranteed valid UTF-8 — copying byte-by-byte preserves multibyte
|
||||
// codepoints (e.g. accented/CJK string values) instead of mangling them
|
||||
// into Latin-1 mojibake.
|
||||
let mut out: Vec<u8> = Vec::with_capacity(input.len() + input.len() / 10);
|
||||
let mut i = 0;
|
||||
let len = bytes.len();
|
||||
|
||||
|
|
@ -187,14 +192,14 @@ fn js_literal_to_json(input: &str) -> String {
|
|||
|
||||
// Skip through strings
|
||||
if b == b'"' {
|
||||
out.push('"');
|
||||
out.push(b'"');
|
||||
i += 1;
|
||||
while i < len {
|
||||
let c = bytes[i];
|
||||
out.push(c as char);
|
||||
out.push(c);
|
||||
i += 1;
|
||||
if c == b'\\' && i < len {
|
||||
out.push(bytes[i] as char);
|
||||
out.push(bytes[i]);
|
||||
i += 1;
|
||||
} else if c == b'"' {
|
||||
break;
|
||||
|
|
@ -205,11 +210,11 @@ fn js_literal_to_json(input: &str) -> String {
|
|||
|
||||
// After { or , — look for unquoted key followed by :
|
||||
if (b == b'{' || b == b',' || b == b'[') && i + 1 < len {
|
||||
out.push(b as char);
|
||||
out.push(b);
|
||||
i += 1;
|
||||
// Skip whitespace
|
||||
while i < len && bytes[i].is_ascii_whitespace() {
|
||||
out.push(bytes[i] as char);
|
||||
out.push(bytes[i]);
|
||||
i += 1;
|
||||
}
|
||||
// Check if next is an unquoted identifier (key)
|
||||
|
|
@ -218,29 +223,30 @@ fn js_literal_to_json(input: &str) -> String {
|
|||
while i < len && (bytes[i].is_ascii_alphanumeric() || bytes[i] == b'_') {
|
||||
i += 1;
|
||||
}
|
||||
let key = &input[key_start..i];
|
||||
let key = &bytes[key_start..i];
|
||||
// Skip whitespace after key
|
||||
while i < len && bytes[i].is_ascii_whitespace() {
|
||||
i += 1;
|
||||
}
|
||||
// If followed by :, it's an unquoted key — quote it
|
||||
if i < len && bytes[i] == b':' {
|
||||
out.push('"');
|
||||
out.push_str(key);
|
||||
out.push('"');
|
||||
out.push(b'"');
|
||||
out.extend_from_slice(key);
|
||||
out.push(b'"');
|
||||
} else {
|
||||
// Not a key — might be a bare value like true/false/null
|
||||
out.push_str(key);
|
||||
out.extend_from_slice(key);
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
out.push(b as char);
|
||||
out.push(b);
|
||||
i += 1;
|
||||
}
|
||||
|
||||
out
|
||||
// Safe: we only copied bytes from valid-UTF-8 `input` plus ASCII quotes.
|
||||
String::from_utf8(out).unwrap_or_else(|e| String::from_utf8_lossy(e.as_bytes()).into_owned())
|
||||
}
|
||||
|
||||
/// Replace raw newlines/tabs inside JSON string values with escape sequences.
|
||||
|
|
@ -440,4 +446,17 @@ newline"}"#;
|
|||
assert_eq!(parsed["text"], "line1\nline2");
|
||||
assert_eq!(parsed["raw"], "has\nnewline");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn js_literal_to_json_preserves_multibyte_utf8() {
|
||||
// Unquoted ASCII keys with accented and CJK string values (the shape
|
||||
// SvelteKit emits). The old `byte as char` path turned the multibyte
|
||||
// values into Latin-1 mojibake; they must now survive intact.
|
||||
let input = r#"{name:"déjà vu", city:"東京", emoji:"🌱"}"#;
|
||||
let json = js_literal_to_json(input);
|
||||
let parsed: Value = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(parsed["name"], "déjà vu");
|
||||
assert_eq!(parsed["city"], "東京");
|
||||
assert_eq!(parsed["emoji"], "🌱");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,13 +14,16 @@ tracing = { workspace = true }
|
|||
tokio = { workspace = true }
|
||||
async-trait = "0.1"
|
||||
# Pinned to exact pre-release versions: wreq/wreq-util are release candidates
|
||||
# with no semver stability between rc.N builds (rc.29 broke the TLS + Response
|
||||
# API). An exact pin keeps `cargo build`, `cargo install` (which ignores
|
||||
# Cargo.lock), and the release workflow all on the version that compiles.
|
||||
wreq = { version = "=6.0.0-rc.28", features = ["cookies", "gzip", "brotli", "zstd", "deflate"] }
|
||||
wreq-util = "=3.0.0-rc.10"
|
||||
# with no semver stability between rc.N builds. An exact pin keeps `cargo build`,
|
||||
# `cargo install` (which ignores Cargo.lock), and the release workflow all on the
|
||||
# version that compiles.
|
||||
wreq = { version = "=6.0.0-rc.29", features = ["cookies", "gzip", "brotli", "zstd", "deflate", "stream"] }
|
||||
wreq-util = "=3.0.0-rc.12"
|
||||
http = "1"
|
||||
bytes = "1"
|
||||
# Stream adapter for `wreq::Response::bytes_stream()` (wreq 6.0.0-rc.29 dropped
|
||||
# `Response::chunk()`); used to buffer bodies under the running size ceiling.
|
||||
futures-util = "0.3"
|
||||
url = "2"
|
||||
rand = "0.8"
|
||||
quick-xml = { version = "0.37", features = ["serde"] }
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ use std::hash::{Hash, Hasher};
|
|||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use futures_util::StreamExt;
|
||||
use rand::seq::SliceRandom;
|
||||
use tokio::sync::Semaphore;
|
||||
use tracing::{debug, instrument, warn};
|
||||
|
|
@ -118,7 +119,7 @@ impl Response {
|
|||
/// negotiated), so a tiny compressed payload that inflates to
|
||||
/// gigabytes is aborted as soon as the accumulated size crosses the
|
||||
/// cap — it never gets fully buffered in memory.
|
||||
async fn from_wreq(mut resp: wreq::Response) -> Result<Self, FetchError> {
|
||||
async fn from_wreq(resp: wreq::Response) -> Result<Self, FetchError> {
|
||||
if let Some(len) = resp.content_length()
|
||||
&& len > MAX_BODY_BYTES
|
||||
{
|
||||
|
|
@ -130,12 +131,13 @@ impl Response {
|
|||
let url = resp.uri().to_string();
|
||||
let headers = resp.headers().clone();
|
||||
|
||||
// wreq 6.0.0-rc.29 dropped `Response::chunk()`. Stream post-decompression
|
||||
// bytes via `bytes_stream()` and keep enforcing the running ceiling so a
|
||||
// compression bomb is aborted before it is fully buffered in memory.
|
||||
let mut buf = bytes::BytesMut::new();
|
||||
while let Some(chunk) = resp
|
||||
.chunk()
|
||||
.await
|
||||
.map_err(|e| FetchError::BodyDecode(e.to_string()))?
|
||||
{
|
||||
let mut stream = resp.bytes_stream();
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk.map_err(|e| FetchError::BodyDecode(e.to_string()))?;
|
||||
check_body_ceiling(buf.len(), chunk.len())?;
|
||||
buf.extend_from_slice(&chunk);
|
||||
}
|
||||
|
|
@ -799,11 +801,17 @@ fn is_challenge_html(html: &str) -> bool {
|
|||
false
|
||||
}
|
||||
|
||||
/// Extract the homepage URL (scheme + host) from a full URL.
|
||||
/// Extract the homepage URL (scheme + host[:port]) from a full URL.
|
||||
fn extract_homepage(url: &str) -> Option<String> {
|
||||
url::Url::parse(url)
|
||||
.ok()
|
||||
.map(|u| format!("{}://{}/", u.scheme(), u.host_str().unwrap_or("")))
|
||||
url::Url::parse(url).ok().map(|u| {
|
||||
let host = u.host_str().unwrap_or("");
|
||||
// `port()` is `Some` only for a non-default port; include it so a
|
||||
// host like example.com:8443 is warmed on the right port.
|
||||
match u.port() {
|
||||
Some(port) => format!("{}://{}:{}/", u.scheme(), host, port),
|
||||
None => format!("{}://{}/", u.scheme(), host),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Convert a webclaw-pdf PdfResult into a webclaw-core ExtractionResult.
|
||||
|
|
|
|||
|
|
@ -10,15 +10,24 @@ use std::{borrow::Cow, io, time::Duration};
|
|||
use wreq::http2::{
|
||||
Http2Options, PseudoId, PseudoOrder, SettingId, SettingsOrder, StreamDependency, StreamId,
|
||||
};
|
||||
use wreq::tls::{
|
||||
AlpnProtocol, AlpsProtocol, CertificateCompressionAlgorithm, ExtensionType, TlsOptions,
|
||||
TlsVersion,
|
||||
};
|
||||
use wreq::{Client, Emulation};
|
||||
use wreq::tls::compress::CertificateCompressor;
|
||||
use wreq::tls::{AlpnProtocol, AlpsProtocol, ExtensionType, TlsOptions, TlsVersion};
|
||||
use wreq::{Client, Emulation, Group, IntoEmulation};
|
||||
use wreq_util::emulate::compress::{BrotliCompressor, ZlibCompressor};
|
||||
|
||||
use crate::browser::BrowserVariant;
|
||||
use crate::error::FetchError;
|
||||
|
||||
// Certificate-compression advertisement per profile. wreq 6.0.0-rc.29 replaced
|
||||
// the `CertificateCompressionAlgorithm` enum argument with `&dyn
|
||||
// CertificateCompressor` trait objects; wreq-util ships the concrete zlib/brotli
|
||||
// implementations. The advertised set (and order) is a TLS fingerprint signal,
|
||||
// so these mirror the previous enum lists exactly.
|
||||
static CHROME_CERT_COMPRESSORS: &[&'static dyn CertificateCompressor] = &[&BrotliCompressor];
|
||||
static FIREFOX_CERT_COMPRESSORS: &[&'static dyn CertificateCompressor] =
|
||||
&[&ZlibCompressor, &BrotliCompressor];
|
||||
static SAFARI_CERT_COMPRESSORS: &[&'static dyn CertificateCompressor] = &[&ZlibCompressor];
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct PublicDnsResolver;
|
||||
|
||||
|
|
@ -119,14 +128,14 @@ fn chrome_extensions() -> Vec<ExtensionType> {
|
|||
ExtensionType::PSK_KEY_EXCHANGE_MODES, // 45
|
||||
ExtensionType::EC_POINT_FORMATS, // 11
|
||||
ExtensionType::CERT_COMPRESSION, // 27
|
||||
ExtensionType::APPLICATION_SETTINGS_NEW, // 17613 (new codepoint, matches alps_use_new_codepoint)
|
||||
ExtensionType::SUPPORTED_VERSIONS, // 43
|
||||
ExtensionType::SIGNATURE_ALGORITHMS, // 13
|
||||
ExtensionType::SERVER_NAME, // 0
|
||||
ExtensionType::APPLICATION_SETTINGS, // 17613 (new codepoint, matches alps_use_new_codepoint)
|
||||
ExtensionType::SUPPORTED_VERSIONS, // 43
|
||||
ExtensionType::SIGNATURE_ALGORITHMS, // 13
|
||||
ExtensionType::SERVER_NAME, // 0
|
||||
ExtensionType::APPLICATION_LAYER_PROTOCOL_NEGOTIATION, // 16
|
||||
ExtensionType::ENCRYPTED_CLIENT_HELLO, // 65037
|
||||
ExtensionType::RENEGOTIATE, // 65281
|
||||
ExtensionType::EXTENDED_MASTER_SECRET, // 23
|
||||
ExtensionType::ENCRYPTED_CLIENT_HELLO, // 65037
|
||||
ExtensionType::RENEGOTIATE, // 65281
|
||||
ExtensionType::EXTENDED_MASTER_SECRET, // 23
|
||||
]
|
||||
}
|
||||
|
||||
|
|
@ -287,7 +296,7 @@ fn chrome_tls() -> TlsOptions {
|
|||
.alps_protocols([AlpsProtocol::HTTP3, AlpsProtocol::HTTP2])
|
||||
.alps_use_new_codepoint(true)
|
||||
.aes_hw_override(true)
|
||||
.certificate_compression_algorithms(&[CertificateCompressionAlgorithm::BROTLI])
|
||||
.certificate_compressors(CHROME_CERT_COMPRESSORS)
|
||||
.build()
|
||||
}
|
||||
|
||||
|
|
@ -304,10 +313,7 @@ fn firefox_tls() -> TlsOptions {
|
|||
.pre_shared_key(true)
|
||||
.enable_ocsp_stapling(true)
|
||||
.enable_signed_cert_timestamps(true)
|
||||
.certificate_compression_algorithms(&[
|
||||
CertificateCompressionAlgorithm::ZLIB,
|
||||
CertificateCompressionAlgorithm::BROTLI,
|
||||
])
|
||||
.certificate_compressors(FIREFOX_CERT_COMPRESSORS)
|
||||
.build()
|
||||
}
|
||||
|
||||
|
|
@ -324,7 +330,7 @@ fn safari_tls() -> TlsOptions {
|
|||
.pre_shared_key(false)
|
||||
.enable_ocsp_stapling(true)
|
||||
.enable_signed_cert_timestamps(true)
|
||||
.certificate_compression_algorithms(&[CertificateCompressionAlgorithm::ZLIB])
|
||||
.certificate_compressors(SAFARI_CERT_COMPRESSORS)
|
||||
.build()
|
||||
}
|
||||
|
||||
|
|
@ -345,21 +351,23 @@ fn safari_tls() -> TlsOptions {
|
|||
/// `priority: u=0, i`, zstd), replace with the real iOS 26 set.
|
||||
/// 4. `accept-language` preserved from config.extra_headers for locale.
|
||||
fn safari_ios_emulation() -> wreq::Emulation {
|
||||
use wreq::EmulationFactory;
|
||||
let mut em = wreq_util::Emulation::SafariIos26.emulation();
|
||||
// wreq 6.0.0-rc.29 exposes the `Emulation` fields directly (no `*_mut()`
|
||||
// accessors) and wreq-util 3.0.0-rc.12 renamed the enum to `Profile` with
|
||||
// `IntoEmulation::into_emulation` replacing `EmulationFactory::emulation`.
|
||||
let mut em = wreq_util::Profile::SafariIos26.into_emulation();
|
||||
|
||||
if let Some(tls) = em.tls_options_mut().as_mut() {
|
||||
if let Some(tls) = em.tls_options.as_mut() {
|
||||
tls.extension_permutation = Some(Cow::Owned(safari_ios_extensions()));
|
||||
}
|
||||
|
||||
// Only override the priority flag — keep wreq-util's SETTINGS, WINDOW_UPDATE,
|
||||
// and pseudo-order intact. Replacing the whole Http2Options resets SETTINGS
|
||||
// to defaults, which sends only INITIAL_WINDOW_SIZE and fails DataDome.
|
||||
if let Some(h2) = em.http2_options_mut().as_mut() {
|
||||
if let Some(h2) = em.http2_options.as_mut() {
|
||||
h2.headers_stream_dependency = Some(StreamDependency::new(StreamId::zero(), 255, true));
|
||||
}
|
||||
|
||||
let hm = em.headers_mut();
|
||||
let hm = &mut em.headers;
|
||||
hm.clear();
|
||||
for (k, v) in SAFARI_IOS_HEADERS {
|
||||
if let (Ok(n), Ok(val)) = (
|
||||
|
|
@ -508,12 +516,12 @@ pub fn build_client(
|
|||
.tls_options(tls)
|
||||
.http2_options(h2)
|
||||
.headers(build_headers(headers))
|
||||
.build()
|
||||
.build(Group::default())
|
||||
}
|
||||
};
|
||||
|
||||
// Append extra headers after profile defaults.
|
||||
let hm = emulation.headers_mut();
|
||||
let hm = &mut emulation.headers;
|
||||
for (k, v) in extra_headers {
|
||||
if let (Ok(n), Ok(val)) = (
|
||||
http::header::HeaderName::from_bytes(k.as_bytes()),
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
/// Anthropic provider — Claude models via api.anthropic.com.
|
||||
/// Anthropic's API differs from OpenAI: system message is a top-level param,
|
||||
/// not part of the messages array.
|
||||
use std::time::Duration;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use serde_json::json;
|
||||
|
||||
|
|
@ -35,7 +37,11 @@ impl AnthropicProvider {
|
|||
let key = load_api_key(key_override, "ANTHROPIC_API_KEY")?;
|
||||
|
||||
Some(Self {
|
||||
client: reqwest::Client::new(),
|
||||
client: reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(120))
|
||||
.connect_timeout(Duration::from_secs(10))
|
||||
.build()
|
||||
.unwrap_or_else(|_| reqwest::Client::new()),
|
||||
key,
|
||||
base_url: base_url
|
||||
.or_else(|| std::env::var("ANTHROPIC_BASE_URL").ok())
|
||||
|
|
@ -108,11 +114,7 @@ impl LlmProvider for AnthropicProvider {
|
|||
if !resp.status().is_success() {
|
||||
let status = resp.status();
|
||||
let text = resp.text().await.unwrap_or_default();
|
||||
let safe_text = if text.len() > 500 {
|
||||
&text[..500]
|
||||
} else {
|
||||
&text
|
||||
};
|
||||
let safe_text = text.chars().take(500).collect::<String>();
|
||||
return Err(LlmError::ProviderError(format!(
|
||||
"anthropic returned {status}: {safe_text}"
|
||||
)));
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
/// Ollama provider — talks to a local Ollama instance (default localhost:11434).
|
||||
/// First choice in the provider chain: free, private, fast on Apple Silicon.
|
||||
use std::time::Duration;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use serde_json::json;
|
||||
|
||||
|
|
@ -24,7 +26,11 @@ impl OllamaProvider {
|
|||
.unwrap_or_else(|| "qwen3:8b".into());
|
||||
|
||||
Self {
|
||||
client: reqwest::Client::new(),
|
||||
client: reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(120))
|
||||
.connect_timeout(Duration::from_secs(10))
|
||||
.build()
|
||||
.unwrap_or_else(|_| reqwest::Client::new()),
|
||||
base_url,
|
||||
default_model,
|
||||
}
|
||||
|
|
@ -70,11 +76,7 @@ impl LlmProvider for OllamaProvider {
|
|||
if !resp.status().is_success() {
|
||||
let status = resp.status();
|
||||
let text = resp.text().await.unwrap_or_default();
|
||||
let safe_text = if text.len() > 500 {
|
||||
&text[..500]
|
||||
} else {
|
||||
&text
|
||||
};
|
||||
let safe_text = text.chars().take(500).collect::<String>();
|
||||
return Err(LlmError::ProviderError(format!(
|
||||
"ollama returned {status}: {safe_text}"
|
||||
)));
|
||||
|
|
@ -98,7 +100,8 @@ impl LlmProvider for OllamaProvider {
|
|||
|
||||
async fn is_available(&self) -> bool {
|
||||
let url = format!("{}/api/tags", self.base_url);
|
||||
matches!(self.client.get(&url).send().await, Ok(r) if r.status().is_success())
|
||||
let req = self.client.get(&url).timeout(Duration::from_secs(10));
|
||||
matches!(req.send().await, Ok(r) if r.status().is_success())
|
||||
}
|
||||
|
||||
fn name(&self) -> &str {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
/// OpenAI provider — works with api.openai.com and any OpenAI-compatible endpoint.
|
||||
use std::time::Duration;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use serde_json::json;
|
||||
|
||||
|
|
@ -69,7 +71,11 @@ impl OpenAiProvider {
|
|||
let key = load_api_key(key_override, "OPENAI_API_KEY")?;
|
||||
|
||||
Some(Self {
|
||||
client: reqwest::Client::new(),
|
||||
client: reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(120))
|
||||
.connect_timeout(Duration::from_secs(10))
|
||||
.build()
|
||||
.unwrap_or_else(|_| reqwest::Client::new()),
|
||||
key,
|
||||
base_url: base_url
|
||||
.or_else(|| std::env::var("OPENAI_BASE_URL").ok())
|
||||
|
|
@ -132,11 +138,7 @@ impl LlmProvider for OpenAiProvider {
|
|||
if !resp.status().is_success() {
|
||||
let status = resp.status();
|
||||
let text = resp.text().await.unwrap_or_default();
|
||||
let safe_text = if text.len() > 500 {
|
||||
&text[..500]
|
||||
} else {
|
||||
&text
|
||||
};
|
||||
let safe_text = text.chars().take(500).collect::<String>();
|
||||
return Err(LlmError::ProviderError(format!(
|
||||
"openai returned {status}: {safe_text}"
|
||||
)));
|
||||
|
|
|
|||
|
|
@ -323,9 +323,10 @@ impl WebclawMcp {
|
|||
if params.urls.len() > 100 {
|
||||
return Err("batch is limited to 100 URLs per request".into());
|
||||
}
|
||||
for u in ¶ms.urls {
|
||||
validate_url(u).await?;
|
||||
}
|
||||
// No up-front DNS pre-validation: it aborted the whole batch on a
|
||||
// single unresolvable URL. The fetch layer applies the same SSRF
|
||||
// guard (validate_public_http_url) per URL, so bad entries surface
|
||||
// as individual per-URL errors below instead of failing the batch.
|
||||
|
||||
let format = params.format.as_deref().unwrap_or("markdown");
|
||||
let concurrency = params.concurrency.unwrap_or(5);
|
||||
|
|
|
|||
|
|
@ -1,6 +1,68 @@
|
|||
# Proxy-Backed Crawling
|
||||
|
||||
Use proxy rotation when you need to distribute a crawl across a proxy pool. webclaw supports a single proxy or a proxy file.
|
||||
Use proxy rotation when you need to distribute a crawl across a proxy pool. webclaw supports a single proxy or a proxy file, and accepts any standard HTTP/HTTPS or SOCKS5 proxy URL.
|
||||
|
||||
## Using ColdProxy
|
||||
|
||||
[ColdProxy](https://coldproxy.com/) is webclaw's infrastructure partner, providing residential IPv4, residential IPv6, and datacenter IPv6 proxies across 195+ countries. Use a ColdProxy endpoint as a full URL with `--proxy` / `WEBCLAW_PROXY`, or list several in a `--proxy-file` pool.
|
||||
|
||||
### 1. Get your endpoint
|
||||
|
||||
Sign in to your [ColdProxy dashboard](https://coldproxy.com/) and copy your proxy host, port, and credentials. Assemble them into a standard proxy URL:
|
||||
|
||||
```text
|
||||
http://USERNAME:PASSWORD@HOST:PORT
|
||||
```
|
||||
|
||||
### 2. One ColdProxy endpoint
|
||||
|
||||
```bash
|
||||
export WEBCLAW_PROXY="http://USERNAME:PASSWORD@HOST:PORT"
|
||||
webclaw https://example.com --format markdown
|
||||
```
|
||||
|
||||
Or pass it inline:
|
||||
|
||||
```bash
|
||||
webclaw https://example.com \
|
||||
--proxy "http://USERNAME:PASSWORD@HOST:PORT" \
|
||||
--format markdown
|
||||
```
|
||||
|
||||
### 3. Rotate a ColdProxy pool
|
||||
|
||||
List one ColdProxy endpoint per line in `coldproxy.txt`. Pool files use `host:port:user:pass` (one entry per line; lines starting with `#` are ignored). Mix product types and regions to match your workload:
|
||||
|
||||
```text
|
||||
# residential IPv4
|
||||
HOST:PORT:USERNAME:PASSWORD
|
||||
# residential IPv6
|
||||
HOST:PORT:USERNAME:PASSWORD
|
||||
# datacenter IPv6
|
||||
HOST:PORT:USERNAME:PASSWORD
|
||||
```
|
||||
|
||||
webclaw rotates across the pool per request:
|
||||
|
||||
```bash
|
||||
webclaw https://docs.example.com \
|
||||
--crawl \
|
||||
--depth 2 \
|
||||
--max-pages 200 \
|
||||
--concurrency 10 \
|
||||
--delay 200 \
|
||||
--proxy-file coldproxy.txt \
|
||||
--format markdown
|
||||
```
|
||||
|
||||
### 4. Target a country
|
||||
|
||||
ColdProxy offers access across 195+ countries. Use the country-specific endpoint from your ColdProxy dashboard for each region you want to collect from (for example, a France residential endpoint for fr-localized pages). Add one endpoint per country to your pool file to spread a single crawl across regions.
|
||||
|
||||
### Choosing a product
|
||||
|
||||
- **Residential IPv4 / IPv6** — highest trust; best for consumer sites, geo-restricted content, and regional QA.
|
||||
- **Datacenter IPv6** — fastest and most cost-effective; best for high-volume crawling of tolerant endpoints.
|
||||
|
||||
## Single Proxy
|
||||
|
||||
|
|
@ -20,12 +82,12 @@ webclaw https://example.com \
|
|||
|
||||
## Proxy Pool
|
||||
|
||||
Create `proxies.txt` with one proxy per line:
|
||||
Create `proxies.txt` with one proxy per line in `host:port:user:pass` format (lines starting with `#` are ignored):
|
||||
|
||||
```text
|
||||
http://user:pass@proxy-1.example.com:8080
|
||||
http://user:pass@proxy-2.example.com:8080
|
||||
http://user:pass@proxy-3.example.com:8080
|
||||
proxy-1.example.com:8080:user:pass
|
||||
proxy-2.example.com:8080:user:pass
|
||||
proxy-3.example.com:8080:user:pass
|
||||
```
|
||||
|
||||
Run a crawl with controlled concurrency:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue