From aab51bea919a1509a71767bd3e274bae0a6006d8 Mon Sep 17 00:00:00 2001 From: Valerio Date: Mon, 18 May 2026 18:56:00 +0200 Subject: [PATCH 1/2] docs: add workflow examples --- README.md | 8 +++ examples/README.md | 8 +++ examples/cloudflare-diagnostics/README.md | 58 ++++++++++++++++++++ examples/firecrawl-compatible-api/README.md | 60 +++++++++++++++++++++ examples/html-to-markdown-rag/README.md | 50 +++++++++++++++++ examples/mcp-web-scraping/README.md | 44 +++++++++++++++ examples/proxy-backed-crawling/README.md | 53 ++++++++++++++++++ 7 files changed, 281 insertions(+) create mode 100644 examples/cloudflare-diagnostics/README.md create mode 100644 examples/firecrawl-compatible-api/README.md create mode 100644 examples/html-to-markdown-rag/README.md create mode 100644 examples/mcp-web-scraping/README.md create mode 100644 examples/proxy-backed-crawling/README.md diff --git a/README.md b/README.md index 06bed01..1ca7380 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,14 @@ webclaw https://example.com \ webclaw https://docs.rust-lang.org --crawl --depth 2 --max-pages 50 ``` +### Workflow examples + +- [HTML to Markdown for RAG](examples/html-to-markdown-rag/) +- [Firecrawl-compatible API](examples/firecrawl-compatible-api/) +- [MCP web scraping](examples/mcp-web-scraping/) +- [Proxy-backed crawling](examples/proxy-backed-crawling/) +- [Cloudflare diagnostics](examples/cloudflare-diagnostics/) + ### Extract brand assets ```bash diff --git a/examples/README.md b/examples/README.md index 142f132..0967d74 100644 --- a/examples/README.md +++ b/examples/README.md @@ -2,6 +2,14 @@ Practical examples showing what webclaw can do. Each example is a self-contained command you can run immediately. +## Workflow Guides + +- [HTML to Markdown for RAG](./html-to-markdown-rag/) turns web pages into markdown or compact LLM text for retrieval pipelines. +- [Firecrawl-Compatible API](./firecrawl-compatible-api/) shows the `/v2` compatibility routes for scrape, crawl, map, and search. +- [MCP Web Scraping](./mcp-web-scraping/) connects webclaw to MCP clients such as Claude Code, Claude Desktop, Cursor, and Codex CLI. +- [Proxy-Backed Crawling](./proxy-backed-crawling/) shows single-proxy and proxy-pool crawling from the CLI. +- [Cloudflare Diagnostics](./cloudflare-diagnostics/) gives a reproducible checklist for blocked or empty protected-site results. + ## Basic Extraction ```bash diff --git a/examples/cloudflare-diagnostics/README.md b/examples/cloudflare-diagnostics/README.md new file mode 100644 index 0000000..e8fd197 --- /dev/null +++ b/examples/cloudflare-diagnostics/README.md @@ -0,0 +1,58 @@ +# Cloudflare Diagnostics + +Use this checklist when a page works in the browser but fails from a scraper, returns a challenge page, or produces empty extracted content. + +## 1. Save the Raw Response + +```bash +webclaw https://protected.example.com --raw-html > raw.html +``` + +Inspect `raw.html` for challenge copy, blocked request text, empty shells, or application HTML that needs JavaScript rendering. + +## 2. Compare Extracted Formats + +```bash +webclaw https://protected.example.com --format markdown > page.md +webclaw https://protected.example.com --format json > page.json +webclaw https://protected.example.com --format llm > page.txt +``` + +If raw HTML has content but markdown is empty, tune extraction with selectors: + +```bash +webclaw https://protected.example.com \ + --include "main, article, [role=main]" \ + --exclude "nav, footer, aside, .cookie-banner" \ + --format markdown +``` + +## 3. Try Another Browser Fingerprint + +```bash +webclaw https://protected.example.com --browser firefox --format markdown +webclaw https://protected.example.com --browser random --format markdown +``` + +## 4. Use Cloud Fallback + +```bash +export WEBCLAW_API_KEY=wc_your_key + +webclaw https://protected.example.com --cloud --format markdown +``` + +Cloud mode can use hosted routing, JS rendering, and protected-site handling that are not part of the fully local open-source path. + +## 5. Keep a Reproducible Report + +When reporting a problem, include: + +- target URL +- command used +- selected format +- whether `--raw-html` returned a challenge or normal page HTML +- whether `--browser firefox` changed the result +- whether cloud mode changed the result + +Remove cookies, tokens, customer data, and private URLs before sharing logs. diff --git a/examples/firecrawl-compatible-api/README.md b/examples/firecrawl-compatible-api/README.md new file mode 100644 index 0000000..e6c3f4b --- /dev/null +++ b/examples/firecrawl-compatible-api/README.md @@ -0,0 +1,60 @@ +# Firecrawl-Compatible API + +webclaw exposes Firecrawl-compatible v2 routes for teams migrating existing scrape, crawl, map, or search calls. + +## Scrape + +```bash +curl https://api.webclaw.io/v2/scrape \ + -H "Authorization: Bearer $WEBCLAW_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "url": "https://example.com", + "formats": ["markdown"] + }' +``` + +## Crawl + +```bash +curl https://api.webclaw.io/v2/crawl \ + -H "Authorization: Bearer $WEBCLAW_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "url": "https://docs.example.com", + "limit": 25, + "maxDepth": 2 + }' +``` + +Poll the returned crawl id: + +```bash +curl https://api.webclaw.io/v2/crawl/$CRAWL_ID \ + -H "Authorization: Bearer $WEBCLAW_API_KEY" +``` + +## Map + +```bash +curl https://api.webclaw.io/v2/map \ + -H "Authorization: Bearer $WEBCLAW_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "url": "https://docs.example.com" + }' +``` + +## Search + +```bash +curl https://api.webclaw.io/v2/search \ + -H "Authorization: Bearer $WEBCLAW_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "query": "site:docs.rs tokio tutorial", + "limit": 5 + }' +``` + +Compatibility routes are meant to reduce migration friction. For new projects, prefer the native `/v1` API because it exposes webclaw-specific options more directly. diff --git a/examples/html-to-markdown-rag/README.md b/examples/html-to-markdown-rag/README.md new file mode 100644 index 0000000..d4c29b3 --- /dev/null +++ b/examples/html-to-markdown-rag/README.md @@ -0,0 +1,50 @@ +# HTML to Markdown for RAG + +Turn web pages into clean markdown or compact LLM text before chunking, embedding, or passing the page to an agent. + +## CLI + +```bash +# Clean markdown with headings, links, and readable structure. +webclaw https://docs.anthropic.com --format markdown > page.md + +# Token-optimized output for direct LLM context. +webclaw https://docs.anthropic.com --format llm > page.txt + +# Keep the main article content and remove common navigation/footer noise. +webclaw https://docs.anthropic.com \ + --only-main-content \ + --format markdown \ + > page.md +``` + +## Batch a URL List + +Create `urls.txt`: + +```text +https://docs.anthropic.com/ +https://docs.anthropic.com/en/docs/claude-code +https://docs.anthropic.com/en/api/messages +``` + +Run: + +```bash +webclaw --urls-file urls.txt --format llm > corpus.txt +``` + +## Hosted API + +```bash +curl https://api.webclaw.io/v1/scrape \ + -H "Authorization: Bearer $WEBCLAW_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "url": "https://docs.anthropic.com", + "formats": ["markdown", "llm"], + "only_main_content": true + }' +``` + +Use `markdown` when humans may inspect the output. Use `llm` when the next step is chunking, embedding, summarization, or prompt context. diff --git a/examples/mcp-web-scraping/README.md b/examples/mcp-web-scraping/README.md new file mode 100644 index 0000000..0663670 --- /dev/null +++ b/examples/mcp-web-scraping/README.md @@ -0,0 +1,44 @@ +# MCP Web Scraping + +Use webclaw as a local MCP server so Claude Code, Claude Desktop, Cursor, Windsurf, OpenCode, Codex CLI, or another MCP client can fetch clean web context. + +## Install + +```bash +npx create-webclaw +``` + +The installer detects supported MCP clients and can write the config for you. + +## Manual Config + +```json +{ + "mcpServers": { + "webclaw": { + "command": "~/.webclaw/webclaw-mcp", + "env": { + "WEBCLAW_API_KEY": "wc_your_key" + } + } + } +} +``` + +`WEBCLAW_API_KEY` is optional for local extraction. Add it when you want cloud fallback for protected sites, JS rendering, hosted search, or hosted research. + +## Example Prompts + +```text +Scrape https://docs.rs/tokio and summarize the parts about task spawning. +``` + +```text +Crawl https://docs.example.com up to depth 2 and return the pages most relevant to authentication. +``` + +```text +Extract the pricing tiers from https://example.com/pricing as JSON with fields name, price, limits, and features. +``` + +The MCP server exposes tools for scrape, crawl, map, batch, extract, summarize, diff, brand, research, search, and vertical extractors. diff --git a/examples/proxy-backed-crawling/README.md b/examples/proxy-backed-crawling/README.md new file mode 100644 index 0000000..fd49be9 --- /dev/null +++ b/examples/proxy-backed-crawling/README.md @@ -0,0 +1,53 @@ +# Proxy-Backed Crawling + +Use proxy rotation when you need to distribute a crawl across a proxy pool. webclaw supports a single proxy or a proxy file. + +## Single Proxy + +```bash +webclaw https://example.com \ + --proxy http://user:pass@proxy.example.com:8080 \ + --format markdown +``` + +SOCKS5 is supported too: + +```bash +webclaw https://example.com \ + --proxy socks5://proxy.example.com:1080 \ + --format markdown +``` + +## Proxy Pool + +Create `proxies.txt` with one proxy per line: + +```text +http://user:pass@proxy-1.example.com:8080 +http://user:pass@proxy-2.example.com:8080 +http://user:pass@proxy-3.example.com:8080 +``` + +Run a crawl with controlled concurrency: + +```bash +webclaw https://docs.example.com \ + --crawl \ + --depth 2 \ + --max-pages 100 \ + --concurrency 10 \ + --delay 200 \ + --proxy-file proxies.txt \ + --format markdown +``` + +## Batch URLs + +```bash +webclaw --urls-file urls.txt \ + --proxy-file proxies.txt \ + --concurrency 10 \ + --format json +``` + +Proxy rotation helps with throughput and IP reputation. It does not replace request fingerprinting, JS rendering, or challenge handling for heavily protected sites. For those, use hosted cloud mode with `WEBCLAW_API_KEY`. From be8bcfebd975dde4053194efaa32f612b149d65c Mon Sep 17 00:00:00 2001 From: Valerio <88933932+0xMassi@users.noreply.github.com> Date: Tue, 19 May 2026 17:03:52 +0200 Subject: [PATCH 2/2] fix: harden resource limits, path safety, and WASM build (#46) Security audit follow-up across the workspace: - webclaw-core: keep the crate WASM-safe. quickjs/rquickjs is now a cfg(not(wasm32)) target dependency and the extraction entry point uses a direct call on wasm instead of spawning a thread, so it builds and runs on wasm32 with or without default features. - webclaw-core: bound the structured-data scrubber recursion (depth cap) so deeply nested attacker JSON-LD / __NEXT_DATA__ cannot exhaust the stack. - webclaw-fetch: stream the response body with a running ceiling so a small highly compressed payload cannot inflate to gigabytes in memory; redact user:pass@ from proxy URLs before they reach error strings. - webclaw-cli: contain output filenames inside the chosen directory (reject .. / absolute, drop traversal path segments), run --webhook URLs through the public-URL SSRF guard, clamp --watch-interval to >=1s, and make research slug truncation char-safe. - webclaw-mcp: char-safe slug truncation (no multibyte slice panic). - setup.sh / deploy/hetzner.sh: replace eval on read input with printf -v, and mask auth key / API token in console output. - CI: enforce the wasm32 build invariant for webclaw-core. Tests added for every behavioral change. Bump to 0.6.3 + CHANGELOG. --- .github/workflows/ci.yml | 15 ++++ CHANGELOG.md | 7 ++ Cargo.lock | 14 +-- Cargo.toml | 2 +- crates/webclaw-cli/src/main.rs | 139 +++++++++++++++++++++++++++-- crates/webclaw-core/Cargo.toml | 5 ++ crates/webclaw-core/src/lib.rs | 56 ++++++++++-- crates/webclaw-core/src/llm/mod.rs | 68 +++++++++++++- crates/webclaw-fetch/src/client.rs | 69 +++++++++++--- crates/webclaw-fetch/src/tls.rs | 61 ++++++++++++- crates/webclaw-mcp/src/server.rs | 33 ++++++- deploy/hetzner.sh | 28 ++++-- setup.sh | 4 +- 13 files changed, 454 insertions(+), 47 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b14bcc..78e5223 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,21 @@ jobs: - run: cargo fmt --check --all - run: cargo clippy --all -- -D warnings + wasm: + name: WASM + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + targets: wasm32-unknown-unknown + - uses: Swatinem/rust-cache@v2 + # webclaw-core must stay WASM-safe (zero network deps, no threads). + # Check both with and without default features so the quickjs gate + # can't regress. + - run: cargo check --target wasm32-unknown-unknown -p webclaw-core + - run: cargo check --target wasm32-unknown-unknown -p webclaw-core --no-default-features + docs: name: Docs runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index e833578..4400ff1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,13 @@ All notable changes to webclaw are documented here. Format follows [Keep a Changelog](https://keepachangelog.com/). +## [0.6.3] — 2026-05-19 + +### Fixed +- Hardened resource and path-safety limits across the CLI, MCP server, and self-hosted API: oversized or highly compressed responses are capped while streaming, deeply nested page data can no longer exhaust memory, output filenames stay inside the chosen directory, webhook URLs are validated like every other fetch, and multibyte search queries no longer crash slug generation. + +--- + ## [0.6.2] — 2026-05-18 ### Fixed diff --git a/Cargo.lock b/Cargo.lock index 273c0c0..04c093c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3219,7 +3219,7 @@ dependencies = [ [[package]] name = "webclaw-cli" -version = "0.6.2" +version = "0.6.3" dependencies = [ "clap", "dotenvy", @@ -3240,7 +3240,7 @@ dependencies = [ [[package]] name = "webclaw-core" -version = "0.6.2" +version = "0.6.3" dependencies = [ "ego-tree", "once_cell", @@ -3258,7 +3258,7 @@ dependencies = [ [[package]] name = "webclaw-fetch" -version = "0.6.2" +version = "0.6.3" dependencies = [ "async-trait", "bytes", @@ -3284,7 +3284,7 @@ dependencies = [ [[package]] name = "webclaw-llm" -version = "0.6.2" +version = "0.6.3" dependencies = [ "async-trait", "reqwest", @@ -3297,7 +3297,7 @@ dependencies = [ [[package]] name = "webclaw-mcp" -version = "0.6.2" +version = "0.6.3" dependencies = [ "dirs", "dotenvy", @@ -3317,7 +3317,7 @@ dependencies = [ [[package]] name = "webclaw-pdf" -version = "0.6.2" +version = "0.6.3" dependencies = [ "pdf-extract", "thiserror", @@ -3326,7 +3326,7 @@ dependencies = [ [[package]] name = "webclaw-server" -version = "0.6.2" +version = "0.6.3" dependencies = [ "anyhow", "axum", diff --git a/Cargo.toml b/Cargo.toml index 5a3bfc6..2c21290 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ resolver = "2" members = ["crates/*"] [workspace.package] -version = "0.6.2" +version = "0.6.3" edition = "2024" license = "AGPL-3.0" repository = "https://github.com/0xMassi/webclaw" diff --git a/crates/webclaw-cli/src/main.rs b/crates/webclaw-cli/src/main.rs index 1c8515f..1348824 100644 --- a/crates/webclaw-cli/src/main.rs +++ b/crates/webclaw-cli/src/main.rs @@ -613,7 +613,15 @@ fn url_to_filename(raw_url: &str, format: &OutputFormat) -> String { Err(_) => (String::new(), String::new(), None), }; - let mut stem = path.trim_matches('/').to_string(); + // Drop empty / "." / ".." path segments so a URL path like + // `/../../etc/passwd` can't climb out of the output directory. + let cleaned_path: String = path + .split('/') + .filter(|seg| !seg.is_empty() && *seg != "." && *seg != "..") + .collect::>() + .join("/"); + + let mut stem = cleaned_path; if stem.is_empty() { // Use hostname for root URLs to avoid collisions in batch mode let clean_host = host.strip_prefix("www.").unwrap_or(&host); @@ -640,13 +648,59 @@ fn url_to_filename(raw_url: &str, format: &OutputFormat) -> String { format!("{sanitized}.{ext}") } +/// Reject a caller-supplied (CSV `url,filename`) name that could escape the +/// output directory: absolute paths, drive prefixes, root, or any `..` +/// component. Returns the validated relative path on success. +fn safe_relative_filename(filename: &str) -> Result { + let candidate = Path::new(filename); + use std::path::Component; + for comp in candidate.components() { + match comp { + Component::Normal(_) | Component::CurDir => {} + Component::ParentDir => { + return Err(format!("refusing path with '..' component: {filename}")); + } + Component::RootDir | Component::Prefix(_) => { + return Err(format!("refusing absolute output path: {filename}")); + } + } + } + if candidate.as_os_str().is_empty() { + return Err("empty output filename".to_string()); + } + Ok(candidate.to_path_buf()) +} + /// Write extraction output to a file inside `dir`, creating parent dirs as needed. +/// +/// `filename` may originate from an attacker-controlled `--urls-file` +/// (`url,filename` CSV). It is validated for traversal, and the canonical +/// destination directory is asserted to stay under the canonical output +/// directory before any write. fn write_to_file(dir: &Path, filename: &str, content: &str) -> Result<(), String> { - let dest = dir.join(filename); + let rel = safe_relative_filename(filename)?; + let dest = dir.join(&rel); + + std::fs::create_dir_all(dir) + .map_err(|e| format!("failed to create directory {}: {e}", dir.display()))?; + let base = dir + .canonicalize() + .map_err(|e| format!("failed to resolve output dir {}: {e}", dir.display()))?; + if let Some(parent) = dest.parent() { std::fs::create_dir_all(parent) .map_err(|e| format!("failed to create directory {}: {e}", parent.display()))?; + let canon_parent = parent + .canonicalize() + .map_err(|e| format!("failed to resolve {}: {e}", parent.display()))?; + if !canon_parent.starts_with(&base) { + return Err(format!( + "refusing to write outside output dir: {}", + dest.display() + )); + } } + std::fs::write(&dest, content) .map_err(|e| format!("failed to write {}: {e}", dest.display()))?; let word_count = content.split_whitespace().count(); @@ -1679,6 +1733,13 @@ fn fire_webhook(url: &str, payload: &serde_json::Value) { serde_json::to_string(payload).unwrap_or_default() }; tokio::spawn(async move { + // SSRF guard: a webhook URL is user-supplied and otherwise bypasses + // the fetch-layer protections, so resolve + reject private/internal + // destinations before sending the payload. + if let Err(e) = webclaw_fetch::url_security::validate_public_http_url(&url).await { + eprintln!("[webhook] refusing unsafe URL: {e}"); + return; + } match reqwest::Client::builder() .timeout(std::time::Duration::from_secs(10)) .build() @@ -1750,7 +1811,9 @@ async fn run_watch_single( ); loop { - tokio::time::sleep(std::time::Duration::from_secs(cli.watch_interval)).await; + // Clamp to >=1s: `--watch-interval 0` would otherwise spin the + // fetch loop with zero delay and hammer the target. + tokio::time::sleep(std::time::Duration::from_secs(cli.watch_interval.max(1))).await; if cancelled.load(Ordering::Relaxed) { eprintln!("[watch] Stopped"); @@ -1842,7 +1905,9 @@ async fn run_watch_multi( let mut check_number = 0u64; loop { - tokio::time::sleep(std::time::Duration::from_secs(cli.watch_interval)).await; + // Clamp to >=1s: `--watch-interval 0` would otherwise spin the + // fetch loop with zero delay and hammer the target. + tokio::time::sleep(std::time::Duration::from_secs(cli.watch_interval.max(1))).await; if cancelled.load(Ordering::Relaxed) { eprintln!("[watch] Stopped"); @@ -2321,7 +2386,9 @@ async fn run_research(cli: &Cli, query: &str) -> Result<(), String> { .collect::>() .join("-") .to_lowercase(); - let slug = if slug.len() > 50 { &slug[..50] } else { &slug }; + // char-safe truncation: byte slicing panics if char 50 + // lands mid-codepoint (multibyte queries). + let slug: String = slug.chars().take(50).collect(); let filename = format!("research-{slug}.json"); let json = serde_json::to_string_pretty(&status_resp).unwrap_or_default(); @@ -2773,4 +2840,66 @@ mod tests { assert_eq!(content, "hello"); let _ = std::fs::remove_dir_all(&dir); } + + #[test] + fn url_to_filename_strips_traversal_segments() { + // `..` / `.` / empty path segments must not survive into the path. + let out = url_to_filename( + "https://example.com/../../etc/passwd", + &OutputFormat::Markdown, + ); + assert!(!out.contains(".."), "traversal leaked: {out}"); + assert_eq!(out, "etc/passwd.md"); + let out2 = url_to_filename("https://example.com/a/./b//c", &OutputFormat::Json); + assert_eq!(out2, "a/b/c.json"); + } + + #[test] + fn safe_relative_filename_rejects_escapes() { + assert!(safe_relative_filename("../escape.md").is_err()); + assert!(safe_relative_filename("a/../../b.md").is_err()); + assert!(safe_relative_filename("/etc/passwd").is_err()); + assert!(safe_relative_filename("").is_err()); + // Normal nested relative names stay allowed. + assert!(safe_relative_filename("nested/deep/file.md").is_ok()); + assert!(safe_relative_filename("./ok.md").is_ok()); + } + + #[test] + fn write_to_file_refuses_traversal_filename() { + let dir = std::env::temp_dir().join("webclaw_test_traversal_dir"); + let _ = std::fs::remove_dir_all(&dir); + // CSV-supplied `url,filename` traversal attempt. + let err = write_to_file(&dir, "../../tmp/webclaw_pwned.md", "x").unwrap_err(); + assert!(err.contains("refusing"), "unexpected error: {err}"); + assert!( + !std::path::Path::new("/tmp/webclaw_pwned.md").exists(), + "traversal write escaped the output dir" + ); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn research_slug_truncation_is_char_safe() { + // Multibyte query: byte-slicing at 50 would panic mid-codepoint. + let query = "日本語".repeat(40); // 120 chars, 3 bytes each + let slug: String = query + .chars() + .map(|c| { + if c.is_alphanumeric() || c == ' ' { + c + } else { + ' ' + } + }) + .collect::() + .split_whitespace() + .collect::>() + .join("-") + .to_lowercase(); + let slug: String = slug.chars().take(50).collect(); + assert!(slug.chars().count() <= 50); + // Round-trips through formatting without panicking. + let _ = format!("research-{slug}.json"); + } } diff --git a/crates/webclaw-core/Cargo.toml b/crates/webclaw-core/Cargo.toml index 497e002..19b2e08 100644 --- a/crates/webclaw-core/Cargo.toml +++ b/crates/webclaw-core/Cargo.toml @@ -20,6 +20,11 @@ url = { version = "2", features = ["serde"] } regex = "1" once_cell = "1" similar = "2" + +# rquickjs links a C library and cannot build for wasm32. Gating it per +# target keeps the `quickjs` feature usable on native while leaving the +# crate WASM-safe even with default features enabled. +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] rquickjs = { version = "0.9", features = ["classes", "properties"], optional = true } [dev-dependencies] diff --git a/crates/webclaw-core/src/lib.rs b/crates/webclaw-core/src/lib.rs index 80dbb5c..a3e0725 100644 --- a/crates/webclaw-core/src/lib.rs +++ b/crates/webclaw-core/src/lib.rs @@ -9,7 +9,7 @@ pub mod diff; pub mod domain; pub mod error; pub mod extractor; -#[cfg(feature = "quickjs")] +#[cfg(all(feature = "quickjs", not(target_arch = "wasm32")))] pub mod js_eval; pub mod llm; pub mod markdown; @@ -46,9 +46,13 @@ pub fn extract(html: &str, url: Option<&str>) -> Result, @@ -70,6 +74,16 @@ pub fn extract_with_options( .unwrap_or(Err(ExtractError::NoContent)) } +/// WASM has no threads; run extraction directly on the caller's stack. +#[cfg(target_arch = "wasm32")] +pub fn extract_with_options( + html: &str, + url: Option<&str>, + options: &ExtractionOptions, +) -> Result { + extract_with_options_inner(html, url, options) +} + fn extract_with_options_inner( html: &str, url: Option<&str>, @@ -187,7 +201,7 @@ fn extract_with_options_inner( // QuickJS: execute inline