mirror of
https://github.com/0xMassi/webclaw.git
synced 2026-04-25 00:06:21 +02:00
32c035c543
Embeds QuickJS (rquickjs) to execute inline <script> tags and extract data hidden in JavaScript variable assignments. Captures window.__* objects like __preloadedData (NYTimes), __PRELOADED_STATE__ (Wired), and self.__next_f (Next.js RSC flight data). Results: - NYTimes: 1,552 → 4,162 words (+168%) - Wired: 1,459 → 9,937 words (+580%) - Zero measurable performance overhead (<15ms per page) - Feature-gated: disable with --no-default-features for WASM Smart text filtering rejects CSS, base64, file paths, code strings. Only readable prose is appended under "## Additional Content". Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
26 lines
626 B
TOML
26 lines
626 B
TOML
[package]
|
|
name = "webclaw-core"
|
|
description = "Pure HTML content extraction engine for LLMs"
|
|
version.workspace = true
|
|
edition.workspace = true
|
|
license.workspace = true
|
|
|
|
[features]
|
|
default = ["quickjs"]
|
|
quickjs = ["rquickjs"]
|
|
|
|
[dependencies]
|
|
serde = { workspace = true }
|
|
serde_json = { workspace = true }
|
|
thiserror = { workspace = true }
|
|
tracing = { workspace = true }
|
|
scraper = "0.22"
|
|
ego-tree = "0.10"
|
|
url = { version = "2", features = ["serde"] }
|
|
regex = "1"
|
|
once_cell = "1"
|
|
similar = "2"
|
|
rquickjs = { version = "0.9", features = ["classes", "properties"], optional = true }
|
|
|
|
[dev-dependencies]
|
|
tokio = { workspace = true }
|