Improve --format llm output quality (#37)

Improve LLM-format output for modern news and documentation pages.

- Filter noisy hydration and low-value page chrome structured data while preserving content-bearing Schema.org records
- Fix element/text spacing without detaching punctuation on docs, forums, and reference pages
- Remove common accessibility link chrome from LLM text and link labels
- Bump workspace version to 0.6.0 and update the changelog

Thanks to Nenad Oric (@devnen) for the original PR and contribution.

crates/webclaw-core/src/llm/body.rs

@@ -29,6 +29,9 @@ pub(crate) fn process_body(markdown: &str) -> ProcessedBody {
     // 0c. Strip leaked JavaScript (framework hydration, self.__wrap_n, etc.)
     let text = cleanup::strip_leaked_js(&text);
 
+    // 0c2. Strip a11y link chrome ("opens new tab", external link hints)
+    let text = cleanup::strip_a11y_link_chrome(&text);
+
     // 0d. Collapse spaced-out text (CSS animation artifacts like "S t a r t")
     // Must run before any dedup -- spaced text confuses word-based dedup.
     let text = cleanup::collapse_spaced_text(&text);

crates/webclaw-core/src/llm/cleanup.rs

@@ -146,6 +146,45 @@ pub(crate) fn strip_leaked_js(input: &str) -> String {
     out
 }
 
+// ---------------------------------------------------------------------------
+// Accessibility link chrome ("opens new tab", "external link")
+// ---------------------------------------------------------------------------
+
+/// Strip screen-reader-only link chrome that bleeds into rendered text.
+///
+/// Sites like Reuters wrap external/new-window links with hidden spans
+/// like `<span class="visually-hidden">, opens new tab</span>`. The noise
+/// filter can't reliably catch these (no consistent class hook across
+/// sites), so they end up duplicated all over the body text. This is a
+/// targeted text-level scrub of the most common phrasings.
+pub(crate) fn strip_a11y_link_chrome(input: &str) -> String {
+    static A11Y_PATTERN: Lazy<Regex> = Lazy::new(|| {
+        Regex::new(
+            r"(?i)(?:\s*,\s*(?:opens (?:in )?(?:a )?new (?:tab|window)|opens external (?:link|website)|external link)\b\.?|\s+\((?:opens (?:in )?(?:a )?new (?:tab|window)|opens external (?:link|website)|external link)\)\.?|\s+external link\b\.?$)",
+        )
+        .unwrap()
+    });
+
+    let mut out = String::with_capacity(input.len());
+    let mut in_code_fence = false;
+    for (i, line) in input.lines().enumerate() {
+        if i > 0 {
+            out.push('\n');
+        }
+        if line.trim().starts_with("```") {
+            in_code_fence = !in_code_fence;
+            out.push_str(line);
+            continue;
+        }
+        if in_code_fence {
+            out.push_str(line);
+            continue;
+        }
+        out.push_str(&A11Y_PATTERN.replace_all(line, ""));
+    }
+    out
+}
+
 // ---------------------------------------------------------------------------
 // Spaced-out text collapsing (CSS animation artifacts)
 // ---------------------------------------------------------------------------
@@ -1356,4 +1395,48 @@ mod tests {
         let input = "```\nImage of something in code\n```";
         assert_eq!(strip_alt_text_noise(input), input);
     }
+
+    #[test]
+    fn a11y_strips_opens_new_tab() {
+        let input = "Download the App, opens new tab and Subscribe, opens new tab.";
+        let out = strip_a11y_link_chrome(input);
+        assert!(!out.to_lowercase().contains("opens new tab"), "leak: {out}");
+        assert!(out.contains("Download the App"));
+        assert!(out.contains("Subscribe"));
+    }
+
+    #[test]
+    fn a11y_strips_external_link_variants() {
+        let cases = [
+            ("Visit our docs, opens external link", "Visit our docs"),
+            ("Click here, opens in a new window.", "Click here"),
+            ("More info external link", "More info"),
+        ];
+        for (input, expected_prefix) in cases {
+            let out = strip_a11y_link_chrome(input);
+            assert!(
+                out.starts_with(expected_prefix),
+                "input={input:?} got={out:?}"
+            );
+            assert!(!out.to_lowercase().contains("opens"), "leak: {out}");
+        }
+    }
+
+    #[test]
+    fn a11y_preserves_code_blocks() {
+        let input = "```\nopens new tab is a function\n```\nDownload, opens new tab";
+        let out = strip_a11y_link_chrome(input);
+        assert!(
+            out.contains("opens new tab is a function"),
+            "code stripped: {out}"
+        );
+        // Outside the fence, the chrome is removed.
+        assert!(!out.to_lowercase().contains("download, opens new tab"));
+    }
+
+    #[test]
+    fn a11y_preserves_external_link_prose() {
+        let input = "Researchers found an external link between the two incidents.";
+        assert_eq!(strip_a11y_link_chrome(input), input);
+    }
 }

crates/webclaw-core/src/llm/links.rs

@@ -88,10 +88,19 @@ fn is_noise_link(text: &str, href: &str) -> bool {
 static MD_MARKERS_RE: Lazy<Regex> =
     Lazy::new(|| Regex::new(r"#{1,6}\s+|\*{1,2}|_{1,2}|`").unwrap());
 
+static A11Y_LABEL_RE: Lazy<Regex> = Lazy::new(|| {
+    Regex::new(
+        r"(?i)(?:\s*,?\s*(?:opens (?:in )?(?:a )?new (?:tab|window)|opens external (?:link|website))\b\.?|\s*,\s*external link\b\.?|\s+external link\b\.?$)",
+    )
+    .unwrap()
+});
+
 /// Clean a link label: strip markdown, dedup repeated phrases, truncate.
 pub(crate) fn clean_link_label(raw: &str) -> String {
     // Strip markdown markers
     let label = MD_MARKERS_RE.replace_all(raw, "").to_string();
+    // Strip a11y link chrome ("opens new tab", etc.)
+    let label = A11Y_LABEL_RE.replace_all(&label, "").to_string();
     let label = label.split_whitespace().collect::<Vec<_>>().join(" ");
 
     // Dedup repeated phrases in label
@@ -181,4 +190,20 @@ mod tests {
         assert!(is_noise_link("user", "https://hn.com/user?id=foo"));
         assert!(!is_noise_link("Rust docs", "https://rust-lang.org"));
     }
+
+    #[test]
+    fn link_label_preserves_external_link_prose() {
+        assert_eq!(
+            clean_link_label("Research found an external link between incidents"),
+            "Research found an external link between incidents"
+        );
+    }
+
+    #[test]
+    fn link_label_strips_terminal_external_link_chrome() {
+        assert_eq!(
+            clean_link_label("Reuters story external link"),
+            "Reuters story"
+        );
+    }
 }

crates/webclaw-core/src/llm/mod.rs

@@ -46,15 +46,73 @@ pub fn to_llm_text(result: &ExtractionResult, url: Option<&str>) -> String {
     }
 
     // -- 4. Structured data (NEXT_DATA, SvelteKit, JSON-LD) --
-    if !result.structured_data.is_empty() {
-        out.push_str("\n\n## Structured Data\n\n```json\n");
-        out.push_str(&serde_json::to_string_pretty(&result.structured_data).unwrap_or_default());
-        out.push_str("\n```");
+    // Only emit useful items: Schema.org records with a meaningful @type,
+    // and only if the total serialized size stays under a budget. Framework
+    // hydration blobs (Next.js pageProps full of ad-targeting flags, build
+    // IDs, schedule paths) explode to hundreds of KB and drown the LLM in
+    // noise — drop them rather than ship them.
+    let useful: Vec<_> = result
+        .structured_data
+        .iter()
+        .filter(|v| is_useful_structured_data(v))
+        .cloned()
+        .collect();
+    if !useful.is_empty() {
+        let serialized = serde_json::to_string_pretty(&useful).unwrap_or_default();
+        const STRUCTURED_DATA_MAX_BYTES: usize = 16 * 1024;
+        if serialized.len() <= STRUCTURED_DATA_MAX_BYTES {
+            out.push_str("\n\n## Structured Data\n\n```json\n");
+            out.push_str(&serialized);
+            out.push_str("\n```");
+        }
     }
 
     out.trim().to_string()
 }
 
+/// Decide whether a structured-data value carries content worth emitting.
+///
+/// Schema.org records with a recognizable content `@type` (Article, NewsArticle,
+/// Product, Recipe, FAQPage, HowTo, Event, Person, Organization, BreadcrumbList,
+/// VideoObject, JobPosting, etc.) are kept. Generic `WebSite` / `WebPage` /
+/// `ItemList` records and Next.js `pageProps`-style blobs without a useful
+/// `@type` are dropped — they're almost always navigation chrome or framework
+/// hydration state.
+fn is_useful_structured_data(v: &serde_json::Value) -> bool {
+    let Some(obj) = v.as_object() else {
+        // SvelteKit can emit compact arrays of page data. Keep those if they
+        // are small enough to be useful, while still dropping giant hydration
+        // arrays under the same budget as untyped objects.
+        if v.is_array() {
+            let serialized = serde_json::to_string(v).unwrap_or_default();
+            return serialized.len() <= 4 * 1024;
+        }
+        return false;
+    };
+    // JSON-LD: @type drives the decision.
+    if let Some(t) = obj.get("@type") {
+        let types: Vec<String> = match t {
+            serde_json::Value::String(s) => vec![s.to_ascii_lowercase()],
+            serde_json::Value::Array(a) => a
+                .iter()
+                .filter_map(|x| x.as_str())
+                .map(str::to_ascii_lowercase)
+                .collect(),
+            _ => Vec::new(),
+        };
+        if types.is_empty() {
+            return false;
+        }
+        // Drop low-info chrome types.
+        const DROP_TYPES: &[&str] = &["website", "webpage", "sitenavigationelement"];
+        return types.iter().any(|t| !DROP_TYPES.iter().any(|d| t == d));
+    }
+    // Next.js pageProps / SvelteKit data without @type: keep only if compact.
+    // Anything over ~4KB is almost certainly hydration state, not content.
+    let serialized = serde_json::to_string(v).unwrap_or_default();
+    serialized.len() <= 4 * 1024
+}
+
 // ---------------------------------------------------------------------------
 // Integration tests that exercise the full pipeline through to_llm_text
 // ---------------------------------------------------------------------------
@@ -700,4 +758,86 @@ mod tests {
         assert!(out.contains("Some content"), "Content before lost: {out}");
         assert!(out.contains("More content"), "Content after lost: {out}");
     }
+
+    // -- Structured-data gating tests --
+
+    fn make_result_with_structured(values: Vec<serde_json::Value>) -> ExtractionResult {
+        let mut r = make_result("# Body");
+        r.structured_data = values;
+        r
+    }
+
+    #[test]
+    fn structured_data_drops_chrome_types() {
+        // WebSite/WebPage records are framework chrome — should be dropped.
+        let r = make_result_with_structured(vec![serde_json::json!({
+            "@type": "WebSite",
+            "name": "Example",
+            "url": "https://example.com"
+        })]);
+        let out = to_llm_text(&r, None);
+        assert!(
+            !out.contains("## Structured Data"),
+            "WebSite chrome leaked into output: {out}"
+        );
+    }
+
+    #[test]
+    fn structured_data_keeps_article_types() {
+        let r = make_result_with_structured(vec![serde_json::json!({
+            "@type": "NewsArticle",
+            "headline": "Big news",
+            "datePublished": "2026-05-10"
+        })]);
+        let out = to_llm_text(&r, None);
+        assert!(
+            out.contains("## Structured Data"),
+            "NewsArticle dropped: {out}"
+        );
+        assert!(out.contains("Big news"));
+    }
+
+    #[test]
+    fn structured_data_drops_oversized_blob() {
+        // 32KB pageProps-style blob with no @type — should be dropped.
+        let big = "x".repeat(32 * 1024);
+        let r = make_result_with_structured(vec![serde_json::json!({
+            "buildId": "abc",
+            "isFallback": false,
+            "noise": big
+        })]);
+        let out = to_llm_text(&r, None);
+        assert!(
+            !out.contains("## Structured Data"),
+            "Oversized untyped blob leaked: len={}",
+            out.len()
+        );
+    }
+
+    #[test]
+    fn structured_data_keeps_compact_untyped() {
+        // Small untyped record (e.g. a parsed pageProps with real content) — keep.
+        let r = make_result_with_structured(vec![serde_json::json!({
+            "title": "Hi",
+            "body": "small enough to keep"
+        })]);
+        let out = to_llm_text(&r, None);
+        assert!(
+            out.contains("## Structured Data"),
+            "Compact untyped dropped: {out}"
+        );
+    }
+
+    #[test]
+    fn structured_data_keeps_compact_untyped_array() {
+        // SvelteKit can emit compact arrays rather than objects.
+        let r = make_result_with_structured(vec![serde_json::json!([
+            { "title": "Hi", "body": "small array item" }
+        ])]);
+        let out = to_llm_text(&r, None);
+        assert!(
+            out.contains("small array item"),
+            "Compact untyped array dropped: {out}"
+        );
+    }
 }