feat: add support for SvelteKit and JS object literal extraction in structured data

crates/webclaw-core/src/lib.rs

@@ -179,8 +179,9 @@ pub fn extract_with_options(
     let domain_type = domain::detect(url, html);
     let domain_data = Some(DomainData { domain_type });
 
-    // JSON-LD structured data (Schema.org Product, Article, etc.)
-    let structured_data = structured_data::extract_json_ld(html);
+    // Structured data (JSON-LD + JS data islands)
+    let mut structured_data = structured_data::extract_json_ld(html);
+    structured_data.extend(structured_data::extract_js_objects(html));
 
     Ok(ExtractionResult {
         metadata: meta,

crates/webclaw-core/src/structured_data.rs

@@ -62,6 +62,115 @@ pub fn extract_json_ld(html: &str) -> Vec<Value> {
     results
 }
 
+/// Extract JSON-like objects from regular <script> tags.
+///
+/// This handles modern frameworks like SvelteKit that embed data as JS object
+/// literals (where keys aren't quoted) rather than pure JSON.
+pub fn extract_js_objects(html: &str) -> Vec<Value> {
+    let mut results = Vec::new();
+
+    // Look for SvelteKit's kit.start({ ... data: [...] }) pattern
+    // or general large JS object assignments.
+    let mut search_from = 0;
+    while let Some(tag_start) = html[search_from..].find("<script") {
+        let abs_start = search_from + tag_start;
+        let tag_region = &html[abs_start..];
+
+        let Some(tag_end_offset) = tag_region.find('>') else {
+            search_from = abs_start + 7;
+            continue;
+        };
+
+        let opening_tag = &tag_region[..tag_end_offset].to_lowercase();
+        // Skip JSON-LD (already handled) or external scripts
+        if opening_tag.contains("application/ld+json") || opening_tag.contains("src=") {
+            search_from = abs_start + tag_end_offset + 1;
+            continue;
+        }
+
+        let content_start = abs_start + tag_end_offset + 1;
+        let remaining = &html[content_start..];
+        let Some(close_offset) = remaining.to_lowercase().find("</script>") else {
+            search_from = content_start;
+            continue;
+        };
+
+        let js_code = &remaining[..close_offset];
+        search_from = content_start + close_offset + 9;
+
+        // Try to find the "data" array in SvelteKit initialization
+        // Uses a regex to handle data: [ or data:[ or data:  [
+        let data_re = regex::Regex::new(r"data\s*:\s*\[").unwrap();
+        if let Some(mat) = data_re.find(js_code) {
+            let start_idx = mat.start() + (mat.as_str().find('[').unwrap());
+            if let Some(data_array) = extract_balanced_bracket(&js_code[start_idx..], '[', ']') {
+                if let Ok(val) = parse_js_literal(&data_array) {
+                    if let Value::Array(arr) = val {
+                        for item in arr {
+                            if !item.is_null() {
+                                results.push(item);
+                            }
+                        }
+                    } else {
+                        results.push(val);
+                    }
+                }
+            }
+        }
+    }
+
+    results
+}
+
+/// Extract content between balanced brackets.
+fn extract_balanced_bracket(text: &str, open: char, close: char) -> Option<String> {
+    let mut depth = 0;
+    let mut start = None;
+
+    for (i, c) in text.char_indices() {
+        if c == open {
+            if depth == 0 {
+                start = Some(i);
+            }
+            depth += 1;
+        } else if c == close {
+            depth -= 1;
+            if depth == 0 {
+                if let Some(s) = start {
+                    return Some(text[s..=i].to_string());
+                }
+            }
+        }
+    }
+    None
+}
+
+/// Attempt to parse a JavaScript object literal as JSON by quoting keys.
+/// This is a heuristic parser for data islands.
+fn parse_js_literal(js: &str) -> Result<Value, serde_json::Error> {
+    // 1. Try direct parse (sometimes it IS valid JSON)
+    if let Ok(v) = serde_json::from_str(js) {
+        return Ok(v);
+    }
+
+    // 2. heuristic: wrap keys in quotes and remove JS specific values
+    let mut cleaned = js.to_string();
+
+    // Replace unquoted keys: {key: or ,key: -> {"key": or ,"key":
+    // Handles whitespace around the key and colon
+    let key_regex = regex::Regex::new(r#"([{,])\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*:"#).unwrap();
+    cleaned = key_regex.replace_all(&cleaned, r#"$1"$2":"#).to_string();
+
+    // Remove "new URL(...)" wrappers - greedy matching for the constructor
+    let url_regex = regex::Regex::new(r"new\s+URL\s*\([^)]+\)").unwrap();
+    cleaned = url_regex.replace_all(&cleaned, "\"redacted_url\"").to_string();
+
+    // Remove "undefined"
+    cleaned = cleaned.replace("undefined", "null");
+
+    serde_json::from_str(&cleaned)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;