apunkt/webclaw
1
0
Fork 0
mirror of https://github.com/0xMassi/webclaw.git synced 2026-06-21 02:28:27 +02:00
Code Issues Projects Releases Packages Wiki Activity

feat: add allow_subdomains and allow_external_links to CrawlConfig

Browse source 
Crawls are same-origin by default. Enable allow_subdomains to follow
sibling/child subdomains (blog.example.com from example.com), or
allow_external_links for full cross-origin crawling.

Root domain extraction uses a heuristic that handles two-part TLDs
(co.uk, com.au). Includes 5 unit tests for root_domain().

Bump to 0.3.12.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Valerio 2026-04-14 19:33:06 +02:00
parent a4c351d5ae
commit 050b2ef463
7 changed files with 109 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

12
Cargo.lock generated
View file

@ -3102,7 +3102,7 @@ dependencies = [
[[package]]
name = "webclaw-cli"
version = "0.3.11"
version = "0.3.12"
dependencies = [
"clap",
"dotenvy",
@ -3122,7 +3122,7 @@ dependencies = [
[[package]]
name = "webclaw-core"
version = "0.3.11"
version = "0.3.12"
dependencies = [
"ego-tree",
"once_cell",
@ -3140,7 +3140,7 @@ dependencies = [
[[package]]
name = "webclaw-fetch"
version = "0.3.11"
version = "0.3.12"
dependencies = [
"bytes",
"calamine",
@ -3162,7 +3162,7 @@ dependencies = [
[[package]]
name = "webclaw-llm"
version = "0.3.11"
version = "0.3.12"
dependencies = [
"async-trait",
"reqwest",
@ -3175,7 +3175,7 @@ dependencies = [
[[package]]
name = "webclaw-mcp"
version = "0.3.11"
version = "0.3.12"
dependencies = [
"dirs",
"dotenvy",
@ -3196,7 +3196,7 @@ dependencies = [
[[package]]
name = "webclaw-pdf"
version = "0.3.11"
version = "0.3.12"
dependencies = [
"pdf-extract",
"thiserror",