mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-08 22:22:17 +02:00
59 lines
2 KiB
Python
59 lines
2 KiB
Python
# SurfSense proprietary crawler engine.
|
|
#
|
|
# Part of the ``app.proprietary`` package; licensed separately from the
|
|
# Apache-2.0 project root (see ``app/proprietary/LICENSE``).
|
|
"""URL helpers for the site crawler: link extraction, canonical key, same-site scope.
|
|
|
|
Pure functions (no I/O) so the crawl frontier stays deterministic and testable.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from urllib.parse import urldefrag, urljoin, urlsplit
|
|
|
|
from lxml import html as lxml_html
|
|
from lxml.etree import ParserError
|
|
from w3lib.url import canonicalize_url as _w3lib_canonicalize_url
|
|
|
|
|
|
def extract_links(page_html: str | None, base_url: str) -> list[str]:
|
|
"""Absolute, http(s), fragment-free, de-duplicated ``<a href>`` targets.
|
|
|
|
Relative hrefs resolve against ``base_url``; the page's own URL is dropped.
|
|
First-seen order is preserved to keep the frontier stable.
|
|
"""
|
|
if not page_html or not page_html.strip():
|
|
return []
|
|
try:
|
|
root = lxml_html.fromstring(page_html)
|
|
except (ParserError, ValueError):
|
|
return []
|
|
|
|
self_url, _ = urldefrag(base_url)
|
|
seen: set[str] = set()
|
|
links: list[str] = []
|
|
for href in root.xpath("//a/@href"):
|
|
target, _ = urldefrag(urljoin(base_url, href.strip()))
|
|
if urlsplit(target).scheme not in ("http", "https"):
|
|
continue
|
|
if target == self_url or target in seen:
|
|
continue
|
|
seen.add(target)
|
|
links.append(target)
|
|
return links
|
|
|
|
|
|
def canonicalize_url(url: str) -> str:
|
|
"""Stable visited-set key: sorts query, normalizes encoding, drops fragment."""
|
|
return _w3lib_canonicalize_url(url, keep_fragments=False)
|
|
|
|
|
|
def host_of(url: str) -> str:
|
|
"""Lowercased host with a leading ``www.`` removed, for same-site matching."""
|
|
host = (urlsplit(url).hostname or "").lower()
|
|
return host[4:] if host.startswith("www.") else host
|
|
|
|
|
|
def same_site(url: str, allowed_hosts: set[str]) -> bool:
|
|
"""Whether ``url``'s host (``www.``-normalized) is in ``allowed_hosts``."""
|
|
return host_of(url) in allowed_hosts
|