diff --git a/crates/webclaw-core/src/lib.rs b/crates/webclaw-core/src/lib.rs index 9fc7945..deb6b85 100644 --- a/crates/webclaw-core/src/lib.rs +++ b/crates/webclaw-core/src/lib.rs @@ -179,8 +179,9 @@ pub fn extract_with_options( let domain_type = domain::detect(url, html); let domain_data = Some(DomainData { domain_type }); - // JSON-LD structured data (Schema.org Product, Article, etc.) - let structured_data = structured_data::extract_json_ld(html); + // Structured data (JSON-LD + JS data islands) + let mut structured_data = structured_data::extract_json_ld(html); + structured_data.extend(structured_data::extract_js_objects(html)); Ok(ExtractionResult { metadata: meta, diff --git a/crates/webclaw-core/src/structured_data.rs b/crates/webclaw-core/src/structured_data.rs index 0e9342f..b8992c7 100644 --- a/crates/webclaw-core/src/structured_data.rs +++ b/crates/webclaw-core/src/structured_data.rs @@ -62,6 +62,115 @@ pub fn extract_json_ld(html: &str) -> Vec { results } +/// Extract JSON-like objects from regular ") else { + search_from = content_start; + continue; + }; + + let js_code = &remaining[..close_offset]; + search_from = content_start + close_offset + 9; + + // Try to find the "data" array in SvelteKit initialization + // Uses a regex to handle data: [ or data:[ or data: [ + let data_re = regex::Regex::new(r"data\s*:\s*\[").unwrap(); + if let Some(mat) = data_re.find(js_code) { + let start_idx = mat.start() + (mat.as_str().find('[').unwrap()); + if let Some(data_array) = extract_balanced_bracket(&js_code[start_idx..], '[', ']') { + if let Ok(val) = parse_js_literal(&data_array) { + if let Value::Array(arr) = val { + for item in arr { + if !item.is_null() { + results.push(item); + } + } + } else { + results.push(val); + } + } + } + } + } + + results +} + +/// Extract content between balanced brackets. +fn extract_balanced_bracket(text: &str, open: char, close: char) -> Option { + let mut depth = 0; + let mut start = None; + + for (i, c) in text.char_indices() { + if c == open { + if depth == 0 { + start = Some(i); + } + depth += 1; + } else if c == close { + depth -= 1; + if depth == 0 { + if let Some(s) = start { + return Some(text[s..=i].to_string()); + } + } + } + } + None +} + +/// Attempt to parse a JavaScript object literal as JSON by quoting keys. +/// This is a heuristic parser for data islands. +fn parse_js_literal(js: &str) -> Result { + // 1. Try direct parse (sometimes it IS valid JSON) + if let Ok(v) = serde_json::from_str(js) { + return Ok(v); + } + + // 2. heuristic: wrap keys in quotes and remove JS specific values + let mut cleaned = js.to_string(); + + // Replace unquoted keys: {key: or ,key: -> {"key": or ,"key": + // Handles whitespace around the key and colon + let key_regex = regex::Regex::new(r#"([{,])\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*:"#).unwrap(); + cleaned = key_regex.replace_all(&cleaned, r#"$1"$2":"#).to_string(); + + // Remove "new URL(...)" wrappers - greedy matching for the constructor + let url_regex = regex::Regex::new(r"new\s+URL\s*\([^)]+\)").unwrap(); + cleaned = url_regex.replace_all(&cleaned, "\"redacted_url\"").to_string(); + + // Remove "undefined" + cleaned = cleaned.replace("undefined", "null"); + + serde_json::from_str(&cleaned) +} + #[cfg(test)] mod tests { use super::*;