feat(crawler): harden web crawler and agent tooling for real-world research tasks

Crawler engine: escalate thin JS-shell pages past static fetch, repair
currency-lossy extractions, emit categorized link records with anchor
provenance, and decode percent-encoded mailto:/tel: contacts; site crawls
reuse the connector ladder via Scrapling's spider engine with URL pattern
filters. Agent layer: read_run gains char_offset paging, search_run gains
match excerpts, new export_run turns stored runs into CSV workspace docs;
reddit search fair-shares the item budget across queries and dedupes
cross-query hits. Subagent prompts and routing teach crawl-after-search,
full-run coverage before summarizing, and executing own-tool next steps
instead of returning partial.

surfsense_backend/tests/unit/capabilities/test_run_truncation.py

@@ -127,7 +127,9 @@ def _patch_session(monkeypatch, value, calls):
 
 
 def _tools():
-    read_run, search_run = run_reader.build_run_reader_tools(workspace_id=7)
+    read_run, search_run, _export_run = run_reader.build_run_reader_tools(
+        workspace_id=7
+    )
     return read_run, search_run
 
 
@@ -150,6 +152,46 @@ async def test_read_run_paginates(monkeypatch):
     assert "workspace_id" in calls[0]
 
 
+@pytest.mark.asyncio
+async def test_read_run_char_offset_pages_inside_one_huge_item(monkeypatch):
+    """A single item bigger than the cap is fully reachable via char_offset."""
+    huge_line = "A" * RUN_OUTPUT_CHAR_CAP + "MARKER" + "B" * 1000
+    _patch_session(monkeypatch, huge_line, [])
+    read_run, _ = _tools()
+    ref = "run_" + "0" * 8 + "-0000-0000-0000-000000000000"
+
+    first = await read_run.ainvoke({"ref": ref, "offset": 0, "limit": 1})
+    assert "MARKER" not in first  # clipped at the cap
+    assert f"char_offset={RUN_OUTPUT_CHAR_CAP}" in first  # continuation hint
+
+    second = await read_run.ainvoke(
+        {"ref": ref, "offset": 0, "limit": 1, "char_offset": RUN_OUTPUT_CHAR_CAP}
+    )
+    assert "MARKER" in second
+    assert "truncated" not in second  # remainder fits
+
+    past_end = await read_run.ainvoke(
+        {"ref": ref, "offset": 0, "limit": 1, "char_offset": len(huge_line) + 5}
+    )
+    assert "No content at char_offset" in past_end
+
+
+@pytest.mark.asyncio
+async def test_search_run_excerpts_huge_matched_line(monkeypatch):
+    """A match inside a huge line returns a window around it, not the whole line."""
+    huge_line = "x" * 100_000 + "NEEDLE" + "y" * 100_000
+    _patch_session(monkeypatch, huge_line, [])
+    _, search_run = _tools()
+
+    out = await search_run.ainvoke(
+        {"ref": "run_" + "0" * 8 + "-0000-0000-0000-000000000000",
+         "pattern": "NEEDLE"}
+    )
+    assert "NEEDLE" in out
+    assert "match at char 100000" in out
+    assert len(out) < 2000  # excerpt, not the 200k line
+
+
 @pytest.mark.asyncio
 async def test_read_run_rejects_bad_ref(monkeypatch):
     _patch_session(monkeypatch, _BODY, [])
@@ -180,6 +222,97 @@ async def test_search_run_matches(monkeypatch):
     assert "item_1" not in out.split("item_7")[0]
 
 
+# --- export_run ------------------------------------------------------------
+
+
+_CRAWL_BODY = "\n".join(
+    [
+        json.dumps(
+            {
+                "url": "https://x.com/team/",
+                "status": "success",
+                "links": [
+                    {"url": "https://x.com/author/jane/", "text": "Jane Doe",
+                     "context": "Jane Doe General Partner", "kind": "internal"},
+                    {"url": "https://x.com/author/bob/", "text": "Bob Roe",
+                     "context": "Bob Roe Operations", "kind": "internal"},
+                    # Duplicate of Jane (nav + card) — must dedupe.
+                    {"url": "https://x.com/author/jane/", "text": "Jane Doe",
+                     "context": "Jane Doe General Partner", "kind": "internal"},
+                    {"url": "https://x.com/about/", "text": "About", "kind": "internal"},
+                ],
+            }
+        ),
+        json.dumps({"url": "https://x.com/jobs/", "status": "failed", "links": []}),
+        "not json — skipped",
+    ]
+)
+
+
+def test_rows_from_body_links_explode_and_items():
+    links = run_reader._rows_from_body(_CRAWL_BODY, "links")
+    assert len(links) == 4
+    assert links[0]["page"] == "https://x.com/team/"
+    assert links[0]["text"] == "Jane Doe"
+
+    items = run_reader._rows_from_body(_CRAWL_BODY, "items")
+    assert [i["url"] for i in items] == ["https://x.com/team/", "https://x.com/jobs/"]
+
+
+def test_rows_to_csv_dedupes_and_orders_columns():
+    records = run_reader._rows_from_body(_CRAWL_BODY, "links")
+    csv_text, count = run_reader._rows_to_csv(records, ["page", "url", "text"])
+    lines = csv_text.strip().split("\n")
+    assert lines[0] == "page,url,text"
+    assert count == 3  # 4 records - 1 duplicate
+    assert len(lines) == 4  # header + 3 rows
+    assert "Jane Doe" in lines[1]
+
+
+@pytest.mark.asyncio
+async def test_export_run_filters_and_saves(monkeypatch):
+    _patch_session(monkeypatch, _CRAWL_BODY, [])
+    saved: dict = {}
+
+    async def _fake_save(*, virtual_path, content, workspace_id):
+        saved["path"] = virtual_path
+        saved["content"] = content
+        saved["workspace_id"] = workspace_id
+        return 42, "/documents/exports/team.csv"
+
+    monkeypatch.setattr(run_reader, "_save_export_document", _fake_save)
+    _, _, export_run = run_reader.build_run_reader_tools(workspace_id=7)
+
+    out = await export_run.ainvoke(
+        {
+            "ref": "run_" + "0" * 8 + "-0000-0000-0000-000000000000",
+            "path": "exports/team.csv",
+            "rows": "links",
+            "include_pattern": "/author/",
+        }
+    )
+    assert "Exported 2 rows" in out  # Jane + Bob; About filtered; dupe deduped
+    assert "/documents/exports/team.csv" in out
+    assert "document id 42" in out
+    assert saved["workspace_id"] == 7
+    assert "About" not in saved["content"]
+    assert "Bob Roe" in saved["content"]
+
+
+@pytest.mark.asyncio
+async def test_export_run_empty_filter_is_error(monkeypatch):
+    _patch_session(monkeypatch, _CRAWL_BODY, [])
+    _, _, export_run = run_reader.build_run_reader_tools(workspace_id=7)
+    out = await export_run.ainvoke(
+        {
+            "ref": "run_" + "0" * 8 + "-0000-0000-0000-000000000000",
+            "path": "exports/none.csv",
+            "include_pattern": "no-such-thing-anywhere",
+        }
+    )
+    assert out.startswith("Error: no rows to export")
+
+
 @pytest.mark.asyncio
 async def test_search_run_falls_back_on_bad_regex(monkeypatch):
     _patch_session(monkeypatch, _BODY, [])

surfsense_backend/tests/unit/capabilities/web/crawl/test_executor.py

@@ -94,6 +94,55 @@ async def test_failed_page_has_no_markdown_but_keeps_error() -> None:
     assert item.error == "boom"
 
 
+async def test_aggregated_contacts_carry_provenance_and_site_wide_flag() -> None:
+    footer = "https://linkedin.com/company/e"
+    person = "https://linkedin.com/in/jane"
+
+    class _ContactsEngine:
+        async def crawl_url(self, url: str) -> CrawlOutcome:
+            socials = [footer] + ([person] if url.endswith("/about") else [])
+            links = ["https://e.com/about", "https://e.com/blog"] if url == "https://e.com/" else []
+            return CrawlOutcome(
+                status=_SUCCESS,
+                result={
+                    "content": "ok",
+                    "metadata": {},
+                    "links": links,
+                    "contacts": {"emails": [], "phones": [], "socials": socials},
+                },
+            )
+
+    execute = build_crawl_executor(engine=_ContactsEngine())
+    out = await execute(
+        CrawlInput(startUrls=["https://e.com/"], maxCrawlDepth=1, maxCrawlPages=10)
+    )
+
+    by_value = {ref.value: ref for ref in out.contacts.socials}
+    assert by_value[footer].siteWide  # on all 3 pages -> boilerplate
+    assert by_value[footer].pageCount == 3
+    assert not by_value[person].siteWide  # only on /about -> page-local entity
+    assert by_value[person].pages == ["https://e.com/about"]
+
+
+async def test_single_page_crawl_marks_contacts_site_wide() -> None:
+    class _OnePageEngine:
+        async def crawl_url(self, url: str) -> CrawlOutcome:
+            return CrawlOutcome(
+                status=_SUCCESS,
+                result={
+                    "content": "ok",
+                    "metadata": {},
+                    "links": [],
+                    "contacts": {"emails": ["a@e.com"], "phones": [], "socials": []},
+                },
+            )
+
+    execute = build_crawl_executor(engine=_OnePageEngine())
+    out = await execute(CrawlInput(startUrls=["https://e.com/"]))
+
+    assert out.contacts.emails[0].siteWide  # one page: no signal to split on
+
+
 async def test_captcha_telemetry_is_rolled_up_for_billing() -> None:
     class _CaptchaEngine:
         async def crawl_url(self, url: str) -> CrawlOutcome:

surfsense_backend/tests/unit/platforms/reddit/test_search_budget.py

@@ -0,0 +1,69 @@
+"""Offline tests for multi-search budgeting in ``iter_reddit``.
+
+No network: ``_search_flow`` is faked. Asserts the maxItems budget is
+fair-shared across concurrent searches (a noisy query can't starve the rest)
+and that the same post surfacing via several queries is emitted once.
+"""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterator
+
+from app.proprietary.platforms.reddit import scraper
+from app.proprietary.platforms.reddit.schemas import RedditScrapeInput
+
+
+def _fake_search_flow(results_by_query: dict[str, list[str]]):
+    """Fake flow yielding post dicts (id per entry), honoring ``max_items``."""
+    calls: dict[str, int] = {}
+
+    def flow(
+        query: str,
+        *,
+        input_model: RedditScrapeInput,
+        subreddit: str | None = None,
+        max_items: int | None = None,
+    ) -> AsyncIterator[dict]:
+        cap = input_model.maxItems if max_items is None else max_items
+        calls[query] = cap
+
+        async def gen() -> AsyncIterator[dict]:
+            for pid in results_by_query.get(query, [])[:cap]:
+                yield {"dataType": "post", "id": pid, "title": pid}
+
+        return gen()
+
+    return flow, calls
+
+
+async def test_budget_is_fair_shared_across_searches(monkeypatch):
+    # One noisy query with 100 hits must not starve the two precise ones.
+    data = {
+        "noisy": [f"n{i}" for i in range(100)],
+        "precise_a": ["a1", "a2", "a3"],
+        "precise_b": ["b1", "b2"],
+    }
+    flow, calls = _fake_search_flow(data)
+    monkeypatch.setattr(scraper, "_search_flow", flow)
+
+    model = RedditScrapeInput(searches=list(data), maxItems=30)
+    items = await scraper.scrape_reddit(model, limit=30)
+
+    ids = {i["id"] for i in items}
+    # Every precise result made it in; noisy filled only its ceil(30/3)=10 share.
+    assert {"a1", "a2", "a3", "b1", "b2"} <= ids
+    assert sum(1 for i in ids if i.startswith("n")) == 10
+    assert all(cap == 10 for cap in calls.values())
+
+
+async def test_duplicate_posts_across_searches_emit_once(monkeypatch):
+    data = {"q1": ["dup", "x1"], "q2": ["dup", "x2"]}
+    flow, _ = _fake_search_flow(data)
+    monkeypatch.setattr(scraper, "_search_flow", flow)
+
+    model = RedditScrapeInput(searches=["q1", "q2"], maxItems=10)
+    items = await scraper.scrape_reddit(model, limit=10)
+
+    ids = [i["id"] for i in items]
+    assert ids.count("dup") == 1
+    assert {"x1", "x2"} <= set(ids)

surfsense_backend/tests/unit/proprietary/web_crawler/test_connector.py

@@ -86,6 +86,100 @@ async def test_escalates_to_dynamic_on_static_miss(
     assert outcome.tier == "scrapling-dynamic"
 
 
+def test_dropped_currency_amounts_detection() -> None:
+    """Fires only when the DOM has currency figures that the markdown lost."""
+    dropped = connector_module.dropped_currency_amounts
+    html = "<html><body><div>Pro plan <b>$49</b>/mo</div></body></html>"
+    assert dropped(html, "Pro plan without figures")
+    assert not dropped(html, "Pro plan $49/mo")  # markdown kept it
+    assert not dropped("<html><body>no prices here</body></html>", "text")
+    # Country-agnostic: symbol-after-amount and ISO codes count too.
+    assert dropped("<html><body>ab 49€ pro Monat</body></html>", "ab pro Monat")
+    assert dropped("<html><body>USD 2,500 per year</body></html>", "per year")
+    # Script content is not visible: a JSON payload price must not trigger.
+    assert not dropped(
+        "<html><body><script>{'price':'$9'}</script>hi</body></html>", "hi"
+    )
+
+
+def test_build_result_repairs_pricing_card_loss() -> None:
+    """div-grid pricing cards dropped by trafilatura get recovered."""
+    cards = "".join(
+        f"<div class='col'><h3>{name}</h3><div class='price'>${price}</div>"
+        f"<ul><li>feature a</li><li>feature b</li></ul>"
+        f"<a href='/signup'>Choose {name}</a></div>"
+        for name, price in (("Free", 0), ("Pro", 49), ("Enterprise", 199))
+    )
+    html = (
+        "<html><head><title>Pricing</title></head><body>"
+        "<article><h1>Simple pricing</h1><p>"
+        + "Choose the plan that fits your team best. " * 30
+        + "</p></article><section class='grid'>"
+        + cards
+        + "</section></body></html>"
+    )
+    result = WebCrawlerConnector()._build_result(
+        html, "https://x.com/pricing", "t", allow_raw_fallback=False
+    )
+    assert result is not None
+    assert "$49" in result["content"]
+    assert "$199" in result["content"]
+
+
+def test_looks_like_js_shell_thresholds() -> None:
+    """Shell = huge HTML AND near-empty extraction; either alone is healthy."""
+    shell = connector_module.looks_like_js_shell
+    assert shell(4_200_000, 597)  # a16z.com/team
+    assert not shell(200_000, 13_092)  # a16z investment-list: normal page
+    assert not shell(45_000, 1_356)  # small brochure page: small is not thin
+    assert not shell(4_200_000, 50_000)  # huge but content-rich (long article)
+
+
+async def test_thin_static_shell_escalates_to_dynamic(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """A JS-shell static result escalates; the hydrated dynamic result wins."""
+    crawler = WebCrawlerConnector()
+
+    async def _thin_static(_url: str, *_args) -> dict:
+        return _result("scrapling-static") | {"thin_static": True}
+
+    async def _dynamic(_url: str, *_args) -> dict:
+        return _result("scrapling-dynamic")
+
+    monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _thin_static)
+    monkeypatch.setattr(crawler, "_crawl_with_dynamic", _dynamic)
+
+    outcome = await crawler.crawl_url("https://example.com")
+
+    assert outcome.status is CrawlOutcomeStatus.SUCCESS
+    assert outcome.tier == "scrapling-dynamic"
+
+
+async def test_thin_static_is_fallback_when_browser_tiers_fail(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Browser tiers unavailable -> the partial static content still returns."""
+    crawler = WebCrawlerConnector()
+
+    async def _thin_static(_url: str, *_args) -> dict:
+        return _result("scrapling-static") | {"thin_static": True}
+
+    async def _unavailable(_url: str, *_args) -> None:
+        raise NotImplementedError("no subprocess support")
+
+    monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _thin_static)
+    monkeypatch.setattr(crawler, "_crawl_with_dynamic", _unavailable)
+    monkeypatch.setattr(crawler, "_crawl_with_stealthy", _unavailable)
+
+    outcome = await crawler.crawl_url("https://example.com")
+
+    assert outcome.status is CrawlOutcomeStatus.SUCCESS
+    assert outcome.tier == "scrapling-static"
+    assert outcome.result is not None
+    assert "thin_static" not in outcome.result  # internal tag never leaks
+
+
 async def test_all_tiers_empty_is_empty(monkeypatch: pytest.MonkeyPatch) -> None:
     """Every tier fetched but extracted nothing -> EMPTY (not billable)."""
     crawler = WebCrawlerConnector()
@@ -345,6 +439,46 @@ async def test_static_4xx_is_classified(
     assert block_state["block_type"] is BlockType.CLOUDFLARE
 
 
+class _FakeScrollPage:
+    """Playwright-page stand-in: height grows per scroll until a plateau."""
+
+    def __init__(self, heights: list[int]):
+        self._heights = heights
+        self._i = 0
+        self.scrolls = 0
+
+    def evaluate(self, script: str):
+        if "scrollHeight" in script and "scrollTo" not in script:
+            return self._heights[min(self._i, len(self._heights) - 1)]
+        self.scrolls += 1
+        self._i += 1
+        return None
+
+    def wait_for_timeout(self, _ms: int) -> None:
+        pass
+
+
+def test_scroll_to_bottom_stops_when_height_stops_growing() -> None:
+    page = _FakeScrollPage([1000, 2000, 3000, 3000])
+    assert connector_module.scroll_to_bottom(page) is page
+    assert page.scrolls == 3  # scrolled at 1000/2000/3000; 3000-again broke the loop
+
+
+def test_scroll_to_bottom_is_bounded_on_endless_feeds() -> None:
+    page = _FakeScrollPage([i * 1000 for i in range(1, 100)])  # never stabilizes
+    connector_module.scroll_to_bottom(page)
+    assert page.scrolls == connector_module._SCROLL_MAX_ROUNDS
+
+
+def test_scroll_to_bottom_swallows_page_errors() -> None:
+    class _Broken:
+        def evaluate(self, _script: str):
+            raise RuntimeError("target closed")
+
+    page = _Broken()
+    assert connector_module.scroll_to_bottom(page) is page  # never raises
+
+
 def test_build_result_ok_on_real_content() -> None:
     """03e: a normal 200 page with content classifies OK."""
     crawler = WebCrawlerConnector()

surfsense_backend/tests/unit/proprietary/web_crawler/test_url_policy.py

@@ -5,10 +5,9 @@ from __future__ import annotations
 import pytest
 
 from app.proprietary.web_crawler.url_policy import (
-    canonicalize_url,
+    extract_link_records,
     extract_links,
     host_of,
-    same_site,
 )
 
 pytestmark = pytest.mark.unit
@@ -60,30 +59,67 @@ def test_extract_links_on_empty_or_blank_html_is_empty() -> None:
     assert extract_links(None, "https://e.com") == []
 
 
-def test_canonicalize_lowercases_host_sorts_query_and_drops_fragment() -> None:
-    assert (
-        canonicalize_url("https://E.com/a?b=2&a=1#frag") == "https://e.com/a?a=1&b=2"
-    )
+def test_extract_link_records_classifies_kinds_and_keeps_anchor_text() -> None:
+    html = """
+    <a href="/about">  About\n  us </a>
+    <a href="https://other.com/x">External</a>
+    <a href="https://www.linkedin.com/in/jane">Jane Doe</a>
+    <a href="mailto:a@b.com?subject=hi">Mail</a>
+    <a href="tel:+1-555-0100">Call</a>
+    """
+    records = {r["url"]: r for r in extract_link_records(html, "https://example.com/")}
+    assert records["https://example.com/about"]["kind"] == "internal"
+    assert records["https://example.com/about"]["text"] == "About us"  # collapsed ws
+    assert records["https://other.com/x"]["kind"] == "external"
+    assert records["https://www.linkedin.com/in/jane"] == {
+        "url": "https://www.linkedin.com/in/jane",
+        "text": "Jane Doe",
+        "context": "",
+        "rel": "",
+        "kind": "social",
+    }
+    assert records["a@b.com"]["kind"] == "email"  # mailto query stripped
+    assert records["+1-555-0100"]["kind"] == "tel"
 
 
-def test_canonicalize_collapses_fragment_and_empty_query_to_one_key() -> None:
-    # The three forms must dedupe to the same visited-set key.
-    canonical = canonicalize_url("https://e.com/a")
-    assert canonicalize_url("https://e.com/a#frag") == canonical
-    assert canonicalize_url("https://e.com/a?") == canonical
+def test_percent_encoded_tel_and_mailto_are_decoded() -> None:
+    """Seen live: <a href="tel:+1%20408-629-1770"> must not leak %20."""
+    html = """
+    <a href="tel:+1%20408-629-1770">Call</a>
+    <a href="mailto:hello%40acme.io">Email</a>
+    """
+    records = {r["kind"]: r for r in extract_link_records(html, "https://example.com/")}
+    assert records["tel"]["url"] == "+1 408-629-1770"
+    assert records["email"]["url"] == "hello@acme.io"
 
 
-def test_canonicalize_keeps_nondefault_port() -> None:
-    assert canonicalize_url("https://e.com:8443/x") == "https://e.com:8443/x"
+def test_icon_only_social_link_gets_ancestor_context() -> None:
+    html = """
+    <div class="team-card">
+      <h3>Jane Doe</h3><p>General Partner</p>
+      <a href="https://linkedin.com/in/jane"><svg></svg></a>
+    </div>
+    """
+    (record,) = extract_link_records(html, "https://example.com/")
+    assert record["text"] == ""
+    assert record["context"] == "Jane Doe General Partner"
+
+
+def test_icon_social_link_prefers_aria_label_over_context() -> None:
+    html = '<div>Footer<a href="https://x.com/acme" aria-label="Acme on X"><svg></svg></a></div>'
+    (record,) = extract_link_records(html, "https://example.com/")
+    assert record["text"] == "Acme on X"
+    assert record["context"] == ""
+
+
+def test_extract_link_records_dedupes_keeping_first_nonempty_text() -> None:
+    html = '<a href="/p"><img src="logo.png"/></a><a href="/p">Pricing</a>'
+    records = extract_link_records(html, "https://example.com/")
+    assert records == [
+        {"url": "https://example.com/p", "text": "Pricing", "rel": "", "kind": "internal"}
+    ]
 
 
 def test_host_of_strips_www_and_lowercases() -> None:
     assert host_of("https://www.Example.com/x") == "example.com"
     assert host_of("https://Example.com/x") == "example.com"
-
-
-def test_same_site_matches_on_normalized_host() -> None:
-    allowed = {"example.com"}
-    assert same_site("https://www.example.com/a", allowed) is True
-    assert same_site("https://example.com/b", allowed) is True
-    assert same_site("https://other.com/c", allowed) is False

surfsense_backend/tests/unit/utils/crawl/test_contacts.py

@@ -0,0 +1,103 @@
+"""``extract_contacts`` behavior: harvest emails/phones/socials from raw HTML."""
+
+from __future__ import annotations
+
+import pytest
+
+from app.utils.crawl import extract_contacts
+
+pytestmark = pytest.mark.unit
+
+
+def test_harvests_mailto_tel_and_socials() -> None:
+    html = """
+    <html><body>
+      <footer>
+        <a href="mailto:hello@acme.io?subject=hi">Email us</a>
+        <a href="tel:+1-555-0100">Call</a>
+        <a href="https://www.linkedin.com/company/acme">LinkedIn</a>
+        <a href="https://x.com/acme">X</a>
+        <a href="https://github.com/acme/repo#readme">GitHub</a>
+        <a href="https://acme.io/about">About</a>
+      </footer>
+    </body></html>
+    """
+    c = extract_contacts(html)
+    assert c.emails == ["hello@acme.io"]  # mailto query stripped
+    assert c.phones == ["+1-555-0100"]
+    assert c.socials == [
+        "https://www.linkedin.com/company/acme",
+        "https://x.com/acme",
+        "https://github.com/acme/repo",  # fragment stripped
+    ]
+    # Same-site, non-social link is not a contact signal.
+    assert "https://acme.io/about" not in c.socials
+
+
+def test_plaintext_email_without_mailto_is_found() -> None:
+    html = "<html><body><p>Reach us at hello@cochat.ai for support.</p></body></html>"
+    assert extract_contacts(html).emails == ["hello@cochat.ai"]
+
+
+def test_filters_noise_emails_and_asset_false_positives() -> None:
+    html = """
+    <html><body>
+      <img src="logo@2x.png">
+      <script src="react@18.2.0.js"></script>
+      <p>ops@sentry.io</p>
+      <a href="mailto:real@company.com">x</a>
+    </body></html>
+    """
+    assert extract_contacts(html).emails == ["real@company.com"]
+
+
+def test_filters_template_placeholders() -> None:
+    html = """
+    <html><body>
+      <p>youremail@business.com your.email@acme.io john-doe@acme.io</p>
+      <a href="mailto:hello@acme.io">real</a>
+      <a href="https://github.com/username">gh template</a>
+      <a href="https://twitter.com/your-handle">tw template</a>
+      <a href="https://www.linkedin.com/in/jane-doe/">real person</a>
+    </body></html>
+    """
+    c = extract_contacts(html)
+    assert c.emails == ["hello@acme.io"]  # your.email + john-doe normalized away
+    assert c.socials == ["https://www.linkedin.com/in/jane-doe/"]
+
+
+def test_regional_social_hosts_are_harvested() -> None:
+    """WhatsApp/Line/VK/Weibo etc. are the business contact channel outside the US."""
+    html = """
+    <a href="https://wa.me/5511999999999">WhatsApp</a>
+    <a href="https://line.me/R/ti/p/@acme">Line</a>
+    <a href="https://vk.com/acme">VK</a>
+    <a href="https://weibo.com/acme">Weibo</a>
+    <a href="https://www.xing.com/pages/acme">Xing</a>
+    """
+    assert len(extract_contacts(html).socials) == 5
+
+
+def test_percent_encoded_hrefs_are_decoded() -> None:
+    """Sites URL-encode tel/mailto hrefs (seen live: tel:+1%20408-629-1770)."""
+    html = """
+    <a href="tel:+1%20408-629-1770">Call</a>
+    <a href="mailto:hello%40acme.io">Email</a>
+    """
+    c = extract_contacts(html)
+    assert c.phones == ["+1 408-629-1770"]
+    assert "hello@acme.io" in c.emails
+
+
+def test_dedupes_case_insensitively_preserving_order() -> None:
+    html = """
+    <a href="mailto:Hello@Acme.io">a</a>
+    <a href="mailto:hello@acme.io">b</a>
+    """
+    assert extract_contacts(html).emails == ["Hello@Acme.io"]
+
+
+def test_empty_or_unparseable_html_is_empty() -> None:
+    assert extract_contacts("").is_empty
+    assert extract_contacts(None).is_empty
+    assert extract_contacts("   ").is_empty