apunkt/webclaw
1
0
Fork 0
mirror of https://github.com/0xMassi/webclaw.git synced 2026-07-24 07:31:01 +02:00
Code Issues Projects Releases Packages Wiki Activity

fix: prevent stack overflow on deeply nested HTML pages

Browse source 
Pages like Express.co.uk live blogs nest 200+ DOM levels deep, overflowing
the default 1 MB main-thread stack on Windows during recursive markdown
conversion.

Two-layer fix:

1. markdown.rs: add depth parameter to node_to_md/children_to_md/inline_text
   with MAX_DOM_DEPTH=200 guard — falls back to plain text collection at limit

2. lib.rs: wrap extract_with_options in a worker thread with 8 MB stack so
   html5ever parsing and extraction both have room on deeply nested pages

Tested with Express.co.uk live blog (previously crashed, now extracts 2000+
lines of clean markdown) and drudgereport.com (still works correctly).
This commit is contained in:
devnen 2026-04-03 23:45:19 +02:00
parent 95a6681b02
commit 74bac87435
3 changed files with 129 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

65
crates/webclaw-core/src/lib.rs
View file

@ -45,10 +45,35 @@ pub fn extract(html: &str, url: Option<&str>) -> Result<ExtractionResult, Extrac
/// `html` — raw HTML string to parse /// `html` — raw HTML string to parse
/// `url` — optional source URL, used for resolving relative links and domain detection /// `url` — optional source URL, used for resolving relative links and domain detection
/// `options` — controls include/exclude selectors, main content mode, and raw HTML output /// `options` — controls include/exclude selectors, main content mode, and raw HTML output
///
/// Spawns extraction on a thread with an 8 MB stack to handle deeply nested
/// HTML (e.g., Express.co.uk live blogs) without overflowing the default 1-2 MB
/// main-thread stack on Windows.
pub fn extract_with_options( pub fn extract_with_options(
html: &str, html: &str,
url: Option<&str>, url: Option<&str>,
options: &ExtractionOptions, options: &ExtractionOptions,
) -> Result<ExtractionResult, ExtractError> {
// The default main-thread stack on Windows is 1 MB, which can overflow
// on deeply nested pages. Spawn a worker thread with 8 MB to be safe.
const STACK_SIZE: usize = 8 * 1024 * 1024; // 8 MB
let html = html.to_string();
let url = url.map(|u| u.to_string());
let options = options.clone();
std::thread::Builder::new()
.stack_size(STACK_SIZE)
.spawn(move || extract_with_options_inner(&html, url.as_deref(), &options))
.map_err(|_| ExtractError::NoContent)?
.join()
.unwrap_or(Err(ExtractError::NoContent))
}
fn extract_with_options_inner(
html: &str,
url: Option<&str>,
options: &ExtractionOptions,
) -> Result<ExtractionResult, ExtractError> { ) -> Result<ExtractionResult, ExtractError> {
if html.is_empty() { if html.is_empty() {
return Err(ExtractError::NoContent); return Err(ExtractError::NoContent);
@ -530,4 +555,44 @@ mod tests {
"raw_html should be absent from JSON when None" "raw_html should be absent from JSON when None"
); );
} }
#[test]
fn express_live_blog_no_stack_overflow() {
// Real-world Express.co.uk live blog that previously caused stack overflow
let html = include_str!("../testdata/express_test.html");
let result = extract(
html,
Some("https://www.express.co.uk/news/world/2189934/iran-live-donald-trump-uae-dubai-kuwait-attacks"),
);
assert!(result.is_ok(), "Should not stack overflow on Express.co.uk live blog");
let result = result.unwrap();
assert!(
result.metadata.word_count > 100,
"Should extract meaningful content, got {} words",
result.metadata.word_count
);
}
#[test]
fn deeply_nested_html_no_stack_overflow() {
// Simulate deeply nested HTML like Express.co.uk live blogs
let depth = 500;
let mut html = String::from("<html><body>");
for _ in 0..depth {
html.push_str("<div><span>");
}
html.push_str("<p>Deep content here</p>");
for _ in 0..depth {
html.push_str("</span></div>");
}
html.push_str("</body></html>");
let result = extract(&html, None);
assert!(result.is_ok(), "Should not stack overflow on deeply nested HTML");
let result = result.unwrap();
assert!(
result.content.markdown.contains("Deep content"),
"Should extract content from deep nesting"
);
}
} }

77
crates/webclaw-core/src/markdown.rs
View file

@ -14,6 +14,12 @@ use crate::types::{CodeBlock, Image, Link};
static CODE_SELECTOR: Lazy<Selector> = Lazy::new(|| Selector::parse("code").unwrap()); static CODE_SELECTOR: Lazy<Selector> = Lazy::new(|| Selector::parse("code").unwrap());
/// Maximum recursion depth for DOM traversal.
/// Express.co.uk live blogs and similar pages can nest 1000+ levels deep,
/// overflowing the default ~1 MB stack on Windows. When we hit this limit
/// we fall back to plain-text collection (which uses an iterator, not recursion).
const MAX_DOM_DEPTH: usize = 200;
/// Collected assets found during conversion. /// Collected assets found during conversion.
pub struct ConvertedAssets { pub struct ConvertedAssets {
pub links: Vec<Link>, pub links: Vec<Link>,
@ -34,7 +40,7 @@ pub fn convert(
code_blocks: Vec::new(), code_blocks: Vec::new(),
}; };
let md = node_to_md(element, base_url, &mut assets, 0, exclude); let md = node_to_md(element, base_url, &mut assets, 0, exclude, 0);
let plain = strip_markdown(&md); let plain = strip_markdown(&md);
let md = collapse_whitespace(&md); let md = collapse_whitespace(&md);
let plain = collapse_whitespace(&plain); let plain = collapse_whitespace(&plain);
@ -49,11 +55,17 @@ fn node_to_md(
assets: &mut ConvertedAssets, assets: &mut ConvertedAssets,
list_depth: usize, list_depth: usize,
exclude: &HashSet<NodeId>, exclude: &HashSet<NodeId>,
depth: usize,
) -> String { ) -> String {
if exclude.contains(&element.id()) { if exclude.contains(&element.id()) {
return String::new(); return String::new();
} }
// Guard against deeply nested DOM trees (e.g., Express.co.uk live blogs).
if depth > MAX_DOM_DEPTH {
return collect_text(element);
}
if noise::is_noise(element) || noise::is_noise_descendant(element) { if noise::is_noise(element) || noise::is_noise_descendant(element) {
// Still collect images and links from noise elements — they're useful // Still collect images and links from noise elements — they're useful
// metadata even though we don't include the noise text in markdown. // metadata even though we don't include the noise text in markdown.
@ -67,38 +79,38 @@ fn node_to_md(
// Headings // Headings
"h1" => format!( "h1" => format!(
"\n\n# {}\n\n", "\n\n# {}\n\n",
inline_text(element, base_url, assets, exclude) inline_text(element, base_url, assets, exclude, depth)
), ),
"h2" => format!( "h2" => format!(
"\n\n## {}\n\n", "\n\n## {}\n\n",
inline_text(element, base_url, assets, exclude) inline_text(element, base_url, assets, exclude, depth)
), ),
"h3" => format!( "h3" => format!(
"\n\n### {}\n\n", "\n\n### {}\n\n",
inline_text(element, base_url, assets, exclude) inline_text(element, base_url, assets, exclude, depth)
), ),
"h4" => format!( "h4" => format!(
"\n\n#### {}\n\n", "\n\n#### {}\n\n",
inline_text(element, base_url, assets, exclude) inline_text(element, base_url, assets, exclude, depth)
), ),
"h5" => format!( "h5" => format!(
"\n\n##### {}\n\n", "\n\n##### {}\n\n",
inline_text(element, base_url, assets, exclude) inline_text(element, base_url, assets, exclude, depth)
), ),
"h6" => format!( "h6" => format!(
"\n\n###### {}\n\n", "\n\n###### {}\n\n",
inline_text(element, base_url, assets, exclude) inline_text(element, base_url, assets, exclude, depth)
), ),
// Paragraph // Paragraph
"p" => format!( "p" => format!(
"\n\n{}\n\n", "\n\n{}\n\n",
inline_text(element, base_url, assets, exclude) inline_text(element, base_url, assets, exclude, depth)
), ),
// Links // Links
"a" => { "a" => {
let text = inline_text(element, base_url, assets, exclude); let text = inline_text(element, base_url, assets, exclude, depth);
let href = element let href = element
.value() .value()
.attr("href") .attr("href")
@ -167,18 +179,18 @@ fn node_to_md(
// in <b>), treat as a container instead of inline bold. // in <b>), treat as a container instead of inline bold.
"strong" | "b" => { "strong" | "b" => {
if cell_has_block_content(element) { if cell_has_block_content(element) {
children_to_md(element, base_url, assets, list_depth, exclude) children_to_md(element, base_url, assets, list_depth, exclude, depth)
} else { } else {
format!("**{}**", inline_text(element, base_url, assets, exclude)) format!("**{}**", inline_text(element, base_url, assets, exclude, depth))
} }
} }
// Italic — same block-content check as bold. // Italic — same block-content check as bold.
"em" | "i" => { "em" | "i" => {
if cell_has_block_content(element) { if cell_has_block_content(element) {
children_to_md(element, base_url, assets, list_depth, exclude) children_to_md(element, base_url, assets, list_depth, exclude, depth)
} else { } else {
format!("*{}*", inline_text(element, base_url, assets, exclude)) format!("*{}*", inline_text(element, base_url, assets, exclude, depth))
} }
} }
@ -213,13 +225,13 @@ fn node_to_md(
.attr("class") .attr("class")
.and_then(extract_language_from_class) .and_then(extract_language_from_class)
}); });
(collect_preformatted_text(code_el), lang) (collect_preformatted_text(code_el, depth), lang)
} else { } else {
let lang = element let lang = element
.value() .value()
.attr("class") .attr("class")
.and_then(extract_language_from_class); .and_then(extract_language_from_class);
(collect_preformatted_text(element), lang) (collect_preformatted_text(element, depth), lang)
}; };
let code = code.trim_matches('\n').to_string(); let code = code.trim_matches('\n').to_string();
@ -234,7 +246,7 @@ fn node_to_md(
// Blockquote // Blockquote
"blockquote" => { "blockquote" => {
let inner = children_to_md(element, base_url, assets, list_depth, exclude); let inner = children_to_md(element, base_url, assets, list_depth, exclude, depth);
let quoted = inner let quoted = inner
.trim() .trim()
.lines() .lines()
@ -246,19 +258,19 @@ fn node_to_md(
// Unordered list // Unordered list
"ul" => { "ul" => {
let items = list_items(element, base_url, assets, list_depth, false, exclude); let items = list_items(element, base_url, assets, list_depth, false, exclude, depth);
format!("\n\n{items}\n\n") format!("\n\n{items}\n\n")
} }
// Ordered list // Ordered list
"ol" => { "ol" => {
let items = list_items(element, base_url, assets, list_depth, true, exclude); let items = list_items(element, base_url, assets, list_depth, true, exclude, depth);
format!("\n\n{items}\n\n") format!("\n\n{items}\n\n")
} }
// List item — handled by ul/ol parent, but if encountered standalone: // List item — handled by ul/ol parent, but if encountered standalone:
"li" => { "li" => {
let text = inline_text(element, base_url, assets, exclude); let text = inline_text(element, base_url, assets, exclude, depth);
format!("- {text}\n") format!("- {text}\n")
} }
@ -271,11 +283,11 @@ fn node_to_md(
// Table // Table
"table" => format!( "table" => format!(
"\n\n{}\n\n", "\n\n{}\n\n",
table_to_md(element, base_url, assets, exclude) table_to_md(element, base_url, assets, exclude, depth)
), ),
// Divs and other containers — just recurse // Divs and other containers — just recurse
_ => children_to_md(element, base_url, assets, list_depth, exclude), _ => children_to_md(element, base_url, assets, list_depth, exclude, depth),
} }
} }
@ -286,13 +298,14 @@ fn children_to_md(
assets: &mut ConvertedAssets, assets: &mut ConvertedAssets,
list_depth: usize, list_depth: usize,
exclude: &HashSet<NodeId>, exclude: &HashSet<NodeId>,
depth: usize,
) -> String { ) -> String {
let mut out = String::new(); let mut out = String::new();
for child in element.children() { for child in element.children() {
match child.value() { match child.value() {
Node::Element(_) => { Node::Element(_) => {
if let Some(child_el) = ElementRef::wrap(child) { if let Some(child_el) = ElementRef::wrap(child) {
let chunk = node_to_md(child_el, base_url, assets, list_depth, exclude); let chunk = node_to_md(child_el, base_url, assets, list_depth, exclude, depth + 1);
if !chunk.is_empty() && !out.is_empty() && needs_separator(&out, &chunk) { if !chunk.is_empty() && !out.is_empty() && needs_separator(&out, &chunk) {
out.push(' '); out.push(' ');
} }
@ -315,13 +328,14 @@ fn inline_text(
base_url: Option<&Url>, base_url: Option<&Url>,
assets: &mut ConvertedAssets, assets: &mut ConvertedAssets,
exclude: &HashSet<NodeId>, exclude: &HashSet<NodeId>,
depth: usize,
) -> String { ) -> String {
let mut out = String::new(); let mut out = String::new();
for child in element.children() { for child in element.children() {
match child.value() { match child.value() {
Node::Element(_) => { Node::Element(_) => {
if let Some(child_el) = ElementRef::wrap(child) { if let Some(child_el) = ElementRef::wrap(child) {
let chunk = node_to_md(child_el, base_url, assets, 0, exclude); let chunk = node_to_md(child_el, base_url, assets, 0, exclude, depth + 1);
if !chunk.is_empty() && !out.is_empty() && needs_separator(&out, &chunk) { if !chunk.is_empty() && !out.is_empty() && needs_separator(&out, &chunk) {
out.push(' '); out.push(' ');
} }
@ -356,7 +370,10 @@ fn collect_text(element: ElementRef<'_>) -> String {
/// Every text node is pushed verbatim -- no trimming, no collapsing. /// Every text node is pushed verbatim -- no trimming, no collapsing.
/// Handles `<br>` as newlines and inserts newlines between block-level children /// Handles `<br>` as newlines and inserts newlines between block-level children
/// (e.g., `<div>` lines produced by some syntax highlighters). /// (e.g., `<div>` lines produced by some syntax highlighters).
fn collect_preformatted_text(element: ElementRef<'_>) -> String { fn collect_preformatted_text(element: ElementRef<'_>, depth: usize) -> String {
if depth > MAX_DOM_DEPTH {
return element.text().collect::<String>();
}
let mut out = String::new(); let mut out = String::new();
for child in element.children() { for child in element.children() {
match child.value() { match child.value() {
@ -370,12 +387,12 @@ fn collect_preformatted_text(element: ElementRef<'_>) -> String {
if !out.is_empty() && !out.ends_with('\n') { if !out.is_empty() && !out.ends_with('\n') {
out.push('\n'); out.push('\n');
} }
out.push_str(&collect_preformatted_text(child_el)); out.push_str(&collect_preformatted_text(child_el, depth + 1));
if !out.ends_with('\n') { if !out.ends_with('\n') {
out.push('\n'); out.push('\n');
} }
} else { } else {
out.push_str(&collect_preformatted_text(child_el)); out.push_str(&collect_preformatted_text(child_el, depth + 1));
} }
} }
} }
@ -405,6 +422,7 @@ fn list_items(
depth: usize, depth: usize,
ordered: bool, ordered: bool,
exclude: &HashSet<NodeId>, exclude: &HashSet<NodeId>,
dom_depth: usize,
) -> String { ) -> String {
let indent = " ".repeat(depth); let indent = " ".repeat(depth);
let mut out = String::new(); let mut out = String::new();
@ -443,6 +461,7 @@ fn list_items(
depth + 1, depth + 1,
child_tag == "ol", child_tag == "ol",
exclude, exclude,
dom_depth + 1,
)); ));
} else { } else {
inline_parts.push_str(&node_to_md( inline_parts.push_str(&node_to_md(
@ -451,6 +470,7 @@ fn list_items(
assets, assets,
depth, depth,
exclude, exclude,
dom_depth + 1,
)); ));
} }
} else if let Some(text) = li_child.value().as_text() { } else if let Some(text) = li_child.value().as_text() {
@ -495,6 +515,7 @@ fn table_to_md(
base_url: Option<&Url>, base_url: Option<&Url>,
assets: &mut ConvertedAssets, assets: &mut ConvertedAssets,
exclude: &HashSet<NodeId>, exclude: &HashSet<NodeId>,
depth: usize,
) -> String { ) -> String {
// Collect all <td>/<th> cells grouped by row, and detect layout tables // Collect all <td>/<th> cells grouped by row, and detect layout tables
let mut raw_rows: Vec<Vec<ElementRef<'_>>> = Vec::new(); let mut raw_rows: Vec<Vec<ElementRef<'_>>> = Vec::new();
@ -542,7 +563,7 @@ fn table_to_md(
for row in &raw_rows { for row in &raw_rows {
for cell in row { for cell in row {
let content = let content =
children_to_md(*cell, base_url, assets, 0, exclude); children_to_md(*cell, base_url, assets, 0, exclude, depth);
let content = content.trim(); let content = content.trim();
if !content.is_empty() { if !content.is_empty() {
if !out.is_empty() { if !out.is_empty() {
@ -560,7 +581,7 @@ fn table_to_md(
.iter() .iter()
.map(|row| { .map(|row| {
row.iter() row.iter()
.map(|c| inline_text(*c, base_url, assets, exclude)) .map(|c| inline_text(*c, base_url, assets, exclude, depth))
.collect() .collect()
}) })
.collect(); .collect();

15
crates/webclaw-core/testdata/express_test.html vendored Normal file
View file

File diff suppressed because one or more lines are too long