From 0d83916cc5dec911133efded40f9a47fcb8b80e2 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Fri, 3 Jul 2026 20:51:45 -0700 Subject: [PATCH] feat(native-connector): added google search results scraper --- .../platforms/google_search/README.md | 208 ++++++ .../platforms/google_search/__init__.py | 12 + .../platforms/google_search/fetch.py | 347 ++++++++++ .../platforms/google_search/parsers.py | 629 ++++++++++++++++++ .../platforms/google_search/query_builder.py | 167 +++++ .../platforms/google_search/schemas.py | 252 +++++++ .../platforms/google_search/scraper.py | 185 ++++++ .../scripts/e2e_google_search.py | 159 +++++ .../unit/platforms/google_search/__init__.py | 0 .../platforms/google_search/test_skeleton.py | 529 +++++++++++++++ 10 files changed, 2488 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/google_search/README.md create mode 100644 surfsense_backend/app/proprietary/platforms/google_search/__init__.py create mode 100644 surfsense_backend/app/proprietary/platforms/google_search/fetch.py create mode 100644 surfsense_backend/app/proprietary/platforms/google_search/parsers.py create mode 100644 surfsense_backend/app/proprietary/platforms/google_search/query_builder.py create mode 100644 surfsense_backend/app/proprietary/platforms/google_search/schemas.py create mode 100644 surfsense_backend/app/proprietary/platforms/google_search/scraper.py create mode 100644 surfsense_backend/scripts/e2e_google_search.py create mode 100644 surfsense_backend/tests/unit/platforms/google_search/__init__.py create mode 100644 surfsense_backend/tests/unit/platforms/google_search/test_skeleton.py diff --git a/surfsense_backend/app/proprietary/platforms/google_search/README.md b/surfsense_backend/app/proprietary/platforms/google_search/README.md new file mode 100644 index 000000000..e1c1861bf --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/google_search/README.md @@ -0,0 +1,208 @@ +# Google Search Results Scraper + +A platform-native Google SERP scraper intended as a **drop-in clone of the +Apify "Google Search Results Scraper" actor** — same input surface, same +output item shape (one item per SERP page). Built on the same layout and +progressive-implementation approach as the sibling `../youtube` and +`../google_maps` scrapers. + +**Current status: organic + paid SERP scraping works end-to-end.** The full +Apify input surface is accepted and validated, the output models mirror the +actor's example JSON, query composition is implemented, and `_serp_page_flow` +fetches and parses live SERPs: **organic results (with `siteLinks`), text ads +(`paidResults`), product/shopping ads (`paidProducts`), related queries, +suggested results, People Also Ask questions *and answers* (with source +url/title), the inline AI Overview (content + cited sources), and +`resultsTotal`**. `mobileResults` renders with a phone UA and parses Google's +mobile lightweight layout; `includeUnfilteredResults` (`filter=0`) is verified +live; **Google AI Mode** (`aiModeSearch.enableAiMode`) emits a conversational +answer + cited sources as its own item; `includeIcons` puts the base64 favicon +on organic/paid results; `saveHtml` attaches the raw page. The only remaining +piece is the HTTP route. + +### How fetching works (and why it's slow) + +Google's web `/search` is hostile: most residential IPs get a 429 "unusual +traffic" wall, and the IPs that pass serve a JavaScript shell whose organic +results only materialize after the page's JS runs. So `fetch.py` needs both a +**non-blocked IP** and a **real browser render**, on the same IP: + +1. Reuse the last known-good sticky IP if it still passes a cheap re-precheck; + otherwise race prechecks on several fresh sticky IPs at once (DataImpulse + maps ports → sessions) and take the first that passes, +2. render on that IP using a **shared long-lived browser** (launched once per + process, per layout; each fetch only opens a fresh context carrying its + vetted proxy) — during which the render clicks the AI Overview's "Show + more" clamp and the initially-served People-Also-Ask questions open (all + clicks fired first, then one shared wait) so their content lands in the DOM; +3. retry on fresh IPs until one returns the results container. + +A warm fetch (browser up, IP cached) runs ~8 s; the first fetch of a process +also pays the ~5 s Chromium launch and a vetting round. Requires the browser +tier (patchright Chromium via Scrapling's `AsyncStealthySession`) and a +residential proxy — set `PROXY_PROVIDER=custom` + `CUSTOM_PROXY_URL` (see +`.env`). Long-running callers can `await fetch.close_sessions()` on shutdown; +scripts that exit anyway can skip it. + +## Scope + +Included (this actor's own features): + +- Organic results, paid results (ads), product ads +- Related queries, People Also Ask, suggested results +- AI Overview extraction (`aiOverview.scrapeFullAiOverview`) +- Google AI Mode (`aiModeSearch.enableAiMode`) — google.com's dedicated AI + search interface, distinct from inline AI Overviews +- Full localization: country (`gl`), search language (`lr`), interface + language (`hl`), exact location (`uule`) +- Advanced search filters composed into query operators (site, related, + intitle/intext/inurl, filetype, before/after/qdr date ranges, exact match) +- Inline HTML capture (`saveHtml`), icons (`includeIcons`). The actor's + key-value-store snapshot (`saveHtmlToKeyValueStore` → `htmlSnapshotUrl`) is + Apify storage plumbing and is skipped (accepted but ignored). + +Excluded on purpose (Apify implements these by piping into *other* actors / +third-party data brokers): `perplexitySearch`, `chatGptSearch`, +`copilotSearch`, `geminiSearch`, `linkProspecting`, and business leads +enrichment (`maximumLeadsEnrichmentRecords`, `leadsEnrichmentDepartments`, +`verifyLeadsEnrichmentEmails`). A verbatim Apify payload containing them still +validates (`extra="allow"`) but they are ignored. + +## Quick start + +```python +from app.proprietary.platforms.google_search import ( + GoogleSearchScrapeInput, scrape_serps, +) + +# One output item per SERP page; queries mixes terms and Google Search URLs. +items = await scrape_serps( + GoogleSearchScrapeInput( + queries="best SEO tools\nhttps://www.google.com/search?q=apify", + maxPagesPerQuery=2, + countryCode="us", + site="example.com", + aiModeSearch={"enableAiMode": True}, + ) +) +``` + +`iter_serps()` is the streaming twin. (No HTTP route yet — module only, per +the progressive rollout.) + +## Module map + +| File | Responsibility | +| ------------------ | ------------------------------------------------------------------------------------------------------------------------- | +| `__init__.py` | Public exports (entry points + schemas). | +| `schemas.py` | Pydantic input/output models mirroring the Apify camelCase spec. `extra="allow"` on outputs keeps the contract open. | +| `scraper.py` | Orchestrator. `iter_serps` dispatches each `queries` line to `_term_flow` / `_url_flow` (+ `_ai_mode_flow` per term). | +| `query_builder.py` | Pure: classify `queries` lines, fold advanced filters into search operators, resolve relative dates, build the URL. | +| `fetch.py` | Proxy-vetted two-phase fetch: cheap precheck GET + headless render on a shared sticky IP, retrying across IPs. | +| `parsers.py` | Rendered SERP HTML → organic / text ads / product ads / related / People-Also-Ask / `resultsTotal` (degrades per-field). | + +## Input semantics (matching Apify) + +- `queries` (required) is a **newline-separated string**; each line is either + a plain search term (advanced Google operators allowed) or a full Google + Search URL (scraped as-is; its own URL parameters win). +- `maxPagesPerQuery` unset means 1 page (~10 results per page). +- `forceExactMatch` wraps the whole term in quotes. +- `site:` takes precedence over `related:` — when both are set, + `relatedToSite` is ignored. +- `wordsInTitle`/`wordsInText`/`wordsInUrl` emit one `intitle:`/`intext:`/ + `inurl:` per word (never the `allin*:` forms — they conflict with other + operators). +- `fileTypes` are OR-joined (`filetype:pdf OR filetype:doc`). +- `beforeDate`/`afterDate` accept absolute (`2024-05-03`, UTC) or relative + (`3 months`) dates → `before:`/`after:` operators. `quickDateRange` + (`d10`/`w2`/`m6`/`y1`) → `tbs=qdr:`. Avoid combining the two. +- `includeUnfilteredResults` → `filter=0`. +- Localization: `countryCode` → `gl=`, `searchLanguage` → `lr=lang_*`, + `languageCode` → `hl=`, `locationUule` → `uule=`. Google retired country + ccTLDs (google.es et al. redirect to google.com since 2025), so the country + is carried by `gl` and the domain is always `google.com`. +- `saveHtmlToKeyValueStore` defaults **true** (matching the actor); + `saveHtml` defaults false. + +## Output shape (`SerpItem`, one per SERP page) + +- `searchQuery` — provenance: `term`, `url`, `device` (DESKTOP/MOBILE), + `page`, `type`, `domain`, `countryCode`, `languageCode`, `locationUule` +- `resultsTotal` +- `organicResults[]` — `title`, `url`, `displayedUrl`, `description`, + `emphasizedKeywords`, `siteLinks`, `productInfo`, `icon`, `type`, + `position` +- `paidResults[]`, `paidProducts[]` +- `relatedQueries[]`, `peopleAlsoAsk[]` +- `suggestedResults[]` — the related queries re-emitted in result shape + (`title`, google-search `url`, `type: "organic"`, 1-based `position`), + matching how the actor synthesizes them +- `aiOverview` — `{content, sources[{title, url, description, imageUrl}]}` + when an AI Overview appears (always fully expanded) +- `aiModeResult` — `{engine, provider, text, sources[], query, kvsHtmlUrl, + url}` when the AI Mode add-on is enabled +- `html` / `htmlSnapshotUrl` — HTML capture add-ons + +All list fields default to `[]`, unsourced scalars to `None` — parity is +additive, consumers never break on missing keys. + +## Testing + +Offline unit tests (no network — query building, schema, and SERP parsing +against a synthetic fixture): + +```bash +cd surfsense_backend +.venv/Scripts/python.exe -m pytest tests/unit/platforms/google_search/ +``` + +Live end-to-end (needs the proxy + browser tier configured): + +```bash +.venv/Scripts/python.exe scripts/e2e_google_search.py +``` + +## Implementation TODO (progressive, like YouTube/Maps) + +- **Done:** `_serp_page_flow` organic / text-ad (`paidResults`) / product-ad + (`paidProducts`) / related / PAA / `resultsTotal` parsing over a proxy-vetted + browser render. +- **Done:** `focusOnPaidAds` — re-renders on fresh IPs (up to 3 tries) until + ads surface, since Google serves ads non-deterministically; falls back to the + richest ad-less render. +- **Done:** People-Also-Ask answers — the render clicks the first ~4 questions + open (`fetch._expand_paa`); the parser handles both answer shapes + (featured-snippet `.hgKElc` with a source link, and AI-generated `.n6owBd` + paragraphs with inline source chips stripped). Expansion appends extra + collapsed questions, which emit question-only. +- **Done:** `siteLinks` on organic results (the expanded sitelinks table of + brand queries' top result) and `suggestedResults` (related queries re-shaped + with `type`/`position`, per the actor's output). +- **Done:** inline AI Overview (`#m-x-content`) — generated prose (paragraphs + + bullets, inline source chips stripped) plus cited sources (title, url, + snippet, thumbnail). The render always clicks "Show more", so the full + overview is scraped whether or not `scrapeFullAiOverview` is set (a superset + of the actor's gated behavior). +- **Done:** `mobileResults` — renders with a Chrome-on-Android UA + phone + viewport. Google serves its *lightweight mobile layout* (a different DOM: + `Gx5Zad` blocks, `/url?q=` redirect anchors, PAA answers and the full AI + Overview pre-loaded — no clicks needed); `parse_serp` auto-detects the + layout and dispatches to the `_mobile_*` extractors. Mobile pages carry no + `resultsTotal`, marked ads, or sitelinks, so those emit `None`/`[]`. +- **Done:** `includeUnfilteredResults` (`filter=0`) verified live end-to-end. +- **Done:** `_ai_mode_flow` — renders `google.com/search?udm=50`; the + conversational answer streams into `[data-subtree='aimc']`, which is built + from the same DOM blocks as the AI Overview, so the prose/source extractors + are shared. Emits one extra item per term with `aiModeResult` + (`engine/provider/text/sources/query/url`). +- **Done:** `includeIcons` — the rendered desktop SERP inlines every favicon + as a `data:image/...;base64,` URI (`img.XNo5Ab`), which is exactly the + actor's output shape, so it's a straight attribute read on organic + paid + results. The mobile lightweight layout carries no favicons. +- **Skipped on purpose:** key-value-store HTML snapshots + (`saveHtmlToKeyValueStore` → `htmlSnapshotUrl`) — that's Apify storage + plumbing (persist the raw page for debugging/auditing), not extraction; we + have no KVS equivalent and `saveHtml` already returns the raw HTML inline + when callers want it. +- HTTP route + registration once the flows are live. diff --git a/surfsense_backend/app/proprietary/platforms/google_search/__init__.py b/surfsense_backend/app/proprietary/platforms/google_search/__init__.py new file mode 100644 index 000000000..1c105f291 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/google_search/__init__.py @@ -0,0 +1,12 @@ +"""Platform-native Google Search results scraper (Apify Google Search Results +Scraper-compatible).""" + +from .schemas import GoogleSearchScrapeInput, SerpItem +from .scraper import iter_serps, scrape_serps + +__all__ = [ + "GoogleSearchScrapeInput", + "SerpItem", + "iter_serps", + "scrape_serps", +] diff --git a/surfsense_backend/app/proprietary/platforms/google_search/fetch.py b/surfsense_backend/app/proprietary/platforms/google_search/fetch.py new file mode 100644 index 000000000..5cb6141bc --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/google_search/fetch.py @@ -0,0 +1,347 @@ +"""Proxy-aware fetch seam for the Google Search scraper. + +Google's web ``/search`` endpoint is far more hostile than Maps: most +residential IPs get a 429 "unusual traffic" wall, and the ones that pass serve +a JavaScript shell whose organic results only materialize after the page's JS +runs. So a plain GET is never enough — we need a *non-blocked* IP **and** a real +browser render, both on the same IP. + +Strategy (measured against DataImpulse residential IPs, ~50% pass rate): + +1. **Reuse the last good IP.** A sticky IP that just served a real SERP is the + best predictor of the next success, so it's cached module-wide and only a + <1 s re-precheck stands between it and the render. +2. **Vet fresh IPs cheaply and in parallel.** A curl_cffi GET costs ~90 KB and + tells us in <1 s whether an IP is walled; several candidates race at once + and the first pass wins. Rotating gateways (``:823``) hand a fresh IP per + request, which makes a *browser* session look like a botnet — so we pin a + per-attempt **sticky** port (one IP for the whole precheck+render) via + :func:`_sticky_variant`. +3. **Render on the vetted IP.** Only then do we spend the headless render, + reusing the same sticky IP. The browser itself is launched **once** and kept + alive module-wide (:func:`_get_session`); each fetch only opens a fresh + context on the vetted proxy, cutting ~5 s of launch cost per page. +4. **Retry across IPs** until one yields real results or we exhaust the budget. + +``ponytail:`` sticky-port rewriting is DataImpulse-specific (its gateway maps +ports→sessions); other providers just reuse their single URL. The upgrade path +if we add another sticky vendor is to extend ``_STICKY_HOSTS``. +""" + +from __future__ import annotations + +import asyncio +import logging +import random +import time +from urllib.parse import urlsplit, urlunsplit + +from scrapling.fetchers import AsyncFetcher + +from app.utils.proxy import get_proxy_url + +try: # browser tier is optional (needs `scrapling[fetchers]` browsers installed) + from scrapling.fetchers import AsyncStealthySession, ProxyRotator +except Exception: # pragma: no cover - import guard + AsyncStealthySession = None # type: ignore[assignment] + ProxyRotator = None # type: ignore[assignment] + +logger = logging.getLogger(__name__) + +# Consent cookies to dodge the EU interstitial (mirrors the Maps/YouTube seams). +CONSENT_COOKIES = {"CONSENT": "PENDING+987", "SOCS": "CAESHAgBEhIaAB"} +_HEADERS = {"Accept-Language": "en-US,en;q=0.9"} + +# Gateways whose sticky sessions are selected by destination port. A random port +# in this range pins one residential IP for the duration of a browser session. +_STICKY_HOSTS = {"gw.dataimpulse.com"} +_STICKY_PORT_RANGE = (10000, 20000) + +# How many distinct IPs to try before giving up on a page. Prechecks are cheap +# and now run in parallel, so the budget is generous — it only bites during a +# rate-limited window where most IPs are walled. +_MAX_IP_ATTEMPTS = 24 +# How many fresh sticky IPs to precheck concurrently per vetting round. At +# ~50% pass rate, 4 in parallel almost always yields a winner in one ~1 s +# round instead of a serial walk. +_VET_CONCURRENCY = 4 +# When a whole vetting round comes back walled, Google is rate-limiting this +# egress; a short pause before the next round lets it cool instead of burning +# the budget in a couple of seconds. +_WALLED_ROUND_BACKOFF_S = 3.0 + +# The sticky IP that most recently served a real SERP; the strongest hint for +# the next fetch. Re-vetted (cheap) before reuse, dropped on failure. +# ponytail: a single slot, not a pool — one render runs at a time today. +_last_good_proxy: str | None = None + +# A usable precheck responds in <1 s; anything slower is a dead/slow sticky IP +# (seen hanging ~60 s). Abandon it on this deadline so a slow IP costs no more +# than a walled one, keeping per-page time predictable. +_PRECHECK_TIMEOUT_S = 5.0 + +# A walled response is small and says so; the anti-bot page never carries the +# results container. Precheck (small shell page) keys off the text markers; +# the rendered page is judged structurally (presence of the results container), +# because a *good* rendered SERP embeds "/sorry/" etc. inside its own scripts. +_BLOCK_MARKERS = ("unusual traffic", "detected unusual", "/sorry/") +# Desktop markers + the mobile lightweight layout's result-block class. +_RESULTS_MARKERS = ('id="rso"', 'id="search"', 'class="tF2Cxc', "Gx5Zad") + + +def _sticky_variant(proxy_url: str | None) -> str | None: + """Pin a random sticky port for gateways that key sessions by port. + + For a rotating gateway this converts ``…@gw.dataimpulse.com:823`` (new IP + per request) into ``…@gw.dataimpulse.com:<10000-20000>`` (one IP held for + the whole browser session). Non-sticky providers are returned unchanged. + """ + if not proxy_url: + return None + parts = urlsplit(proxy_url) + if parts.hostname not in _STICKY_HOSTS: + return proxy_url + port = random.randint(*_STICKY_PORT_RANGE) + userinfo = "" + if parts.username: + userinfo = parts.username + if parts.password: + userinfo += f":{parts.password}" + userinfo += "@" + netloc = f"{userinfo}{parts.hostname}:{port}" + return urlunsplit((parts.scheme, netloc, parts.path, parts.query, parts.fragment)) + + +def _is_walled(html: str | None) -> bool: + """True for the small anti-bot interstitial (precheck GET use).""" + low = (html or "").lower() + return any(m in low for m in _BLOCK_MARKERS) + + +def _has_results(html: str | None) -> bool: + """True when the rendered DOM carries the organic results container. + + A fully-rendered SERP is ~1 MB and legitimately mentions ``/sorry/`` etc. + inside its own scripts, so the render is judged by *structure* (the results + container is present) rather than by text markers. + """ + return any(m in (html or "") for m in _RESULTS_MARKERS) + + +async def _precheck(url: str, proxy: str | None) -> bool: + """Cheap GET to decide if this IP is walled (True = looks usable).""" + try: + r = await AsyncFetcher.get( + url, + cookies=CONSENT_COOKIES, + proxy=proxy, + stealthy_headers=True, + timeout=_PRECHECK_TIMEOUT_S, + ) + except Exception as e: + # A timeout (slow/dead IP) lands here too; treat it like a walled IP. + logger.debug("[google_search] precheck error: %s", e) + return False + return r.status == 200 and not _is_walled(r.html_content) + + +async def _vet_fresh_ip(url: str, base: str) -> str | None: + """Race prechecks on :data:`_VET_CONCURRENCY` fresh sticky IPs. + + The first IP to pass wins and the rest are cancelled, so one round costs + about one precheck (~1 s) instead of a serial walk over walled IPs. + """ + + async def vet(proxy: str | None) -> str | None: + return proxy if await _precheck(url, proxy) else None + + tasks = [ + asyncio.create_task(vet(_sticky_variant(base))) + for _ in range(_VET_CONCURRENCY) + ] + winner: str | None = None + for fut in asyncio.as_completed(tasks): + winner = await fut + if winner: + break + for task in tasks: + task.cancel() + return winner + + +# People-also-ask answers only load when a question is expanded (clicked). +# We expand just the initially-served questions (~4); each expansion appends +# more questions we deliberately leave collapsed, or the loop never ends. +# All clicks are fired first and the XHRs load concurrently behind one shared +# wait, instead of a serial click→wait per question. +_PAA_EXPAND_LIMIT = 4 +_PAA_ANSWER_WAIT_MS = 1500 +# The AI Overview's "Show more" clamp, when present. +_AIO_SHOW_MORE_SEL = "[aria-label='Show more AI Overview']" + + +async def _expand_blocks(page): + """Click open the lazy SERP blocks so their content renders into the DOM. + + Runs inside the browser render (Playwright async API — the persistent + session's ``page_action`` must be a coroutine). Two expansions: + + * AI Overview "Show more" (``ponytail:`` clicked unconditionally, so the + full overview is always scraped and ``scrapeFullAiOverview`` needs no + plumbing down here — a superset of the actor's gated behavior). + * The initially-served People-Also-Ask questions (answers load on click). + + Both are free on pages without the block, and best-effort: a failed click + just leaves that section collapsed rather than failing the render. + """ + clicked = 0 + try: + more = await page.query_selector(_AIO_SHOW_MORE_SEL) + if more: + await more.click(timeout=1500) + clicked += 1 + except Exception: # clamp absent/detached; the collapsed text still parses + pass + try: + pairs = await page.query_selector_all("div.related-question-pair") + for pair in pairs[:_PAA_EXPAND_LIMIT]: + try: + await pair.click(timeout=1500) + clicked += 1 + except Exception: # stale handle/overlay; skip pair + continue + except Exception as e: # never fail the render over PAA + logger.debug("[google_search] PAA expansion skipped: %s", e) + if clicked: + # One shared wait while all the answer XHRs land in parallel. + await page.wait_for_timeout(_PAA_ANSWER_WAIT_MS) + return page + + +# Firefox-on-Android UA to make Google serve its mobile lightweight layout. +# ponytail: the engine underneath is patchright's *Chromium*, so this UA lies +# about the engine — but empirically it's what gets the mobile layout served +# without tripping Google's wall. A Chrome-on-Android UA (the "coherent" +# choice) gets 429-walled on every IP, so don't switch it back without a live +# mobile e2e proving the layout still loads. +_MOBILE_UA = "Mozilla/5.0 (Android 14; Mobile; rv:132.0) Gecko/132.0 Firefox/132.0" +_MOBILE_VIEWPORT = {"width": 412, "height": 915} + + +# One live browser per layout (desktop / mobile — the UA and viewport are +# session-level context options). Launching Chromium costs ~5 s, so it's paid +# once and every fetch just opens a fresh context on its vetted sticky proxy. +_sessions: dict[bool, AsyncStealthySession] = {} +_session_lock = asyncio.Lock() + + +async def _get_session(mobile: bool) -> AsyncStealthySession: + """The shared live browser session for this layout, launching it if needed.""" + async with _session_lock: + session = _sessions.get(mobile) + if session is not None: + return session + kwargs: dict = { + "headless": True, + "network_idle": True, + "google_search": True, + "page_action": _expand_blocks, + "retries": 1, # our own IP loop is the retry policy + } + base = get_proxy_url() + if base: + # Rotator mode makes the session launch a plain browser so each + # fetch can carry its own vetted sticky proxy; the rotator itself + # is never consulted because every fetch passes an explicit proxy. + kwargs["proxy_rotator"] = ProxyRotator([base]) + if mobile: + kwargs["useragent"] = _MOBILE_UA + kwargs["additional_args"] = {"viewport": _MOBILE_VIEWPORT} + session = AsyncStealthySession(**kwargs) + await session.start() + _sessions[mobile] = session + return session + + +async def _drop_session(mobile: bool) -> None: + """Close and forget a session whose browser is presumed broken.""" + async with _session_lock: + session = _sessions.pop(mobile, None) + if session is not None: + try: + await session.close() + except Exception: # already dead; nothing to salvage + pass + + +async def close_sessions() -> None: + """Shut down the shared browsers (for tests/scripts wanting a clean exit).""" + for mobile in (False, True): + await _drop_session(mobile) + + +async def _render(url: str, proxy: str | None, mobile: bool = False): + """Headless render of a SERP on the shared browser (fresh proxy context).""" + session = await _get_session(mobile) + return await session.fetch(url, proxy=proxy) + + +async def fetch_serp_html(url: str, *, mobile: bool = False) -> str | None: + """Return fully-rendered SERP HTML for ``url``, or ``None`` if unobtainable. + + Reuses the last known-good sticky IP when it still passes the cheap + precheck; otherwise races prechecks on fresh sticky IPs and renders on the + first that passes. Retries until a render returns real results or the IP + budget runs out. Requires the browser tier — without it we cannot get + JS-built results. ``mobile`` renders with a phone UA/viewport (the + ``mobileResults`` input). + """ + global _last_good_proxy + if AsyncStealthySession is None: + logger.error("[google_search] browser tier unavailable; cannot render SERPs") + return None + + base = get_proxy_url() + ips_tried = 0 + while ips_tried < _MAX_IP_ATTEMPTS: + if base: + if _last_good_proxy and await _precheck(url, _last_good_proxy): + proxy = _last_good_proxy + else: + _last_good_proxy = None + proxy = await _vet_fresh_ip(url, base) + ips_tried += _VET_CONCURRENCY + if proxy is None: + logger.debug("[google_search] vetting round: all IPs walled") + await asyncio.sleep(_WALLED_ROUND_BACKOFF_S) + continue + else: + proxy = None + ips_tried += 1 + started = time.perf_counter() + try: + page = await _render(url, proxy, mobile=mobile) + except Exception as e: + # Renders on a walled IP still return HTML; an exception means the + # browser side is broken, so relaunch it rather than limp along. + logger.warning("[google_search] render failed: %s", e) + _last_good_proxy = None + await _drop_session(mobile) + continue + fetch_ms = (time.perf_counter() - started) * 1000 + html = page.html_content or "" + good = page.status == 200 and _has_results(html) + logger.info( + "[google_search][perf] status=%s bytes=%d has_results=%s fetch_ms=%.0f reused_ip=%s", + page.status, + len(html), + good, + fetch_ms, + proxy == _last_good_proxy, + ) + if good: + _last_good_proxy = proxy + return html + _last_good_proxy = None + logger.warning("[google_search] exhausted %d IPs for %s", _MAX_IP_ATTEMPTS, url) + return None diff --git a/surfsense_backend/app/proprietary/platforms/google_search/parsers.py b/surfsense_backend/app/proprietary/platforms/google_search/parsers.py new file mode 100644 index 000000000..e74fb4549 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/google_search/parsers.py @@ -0,0 +1,629 @@ +"""Parse a rendered Google Search results page into Apify-shaped models. + +Two layouts are handled: the desktop layout below, and the **mobile +lightweight layout** Google serves to phone UAs (``mobileResults``), which +uses a completely different DOM — see the ``_mobile_*`` extractors and the +dispatch in :func:`parse_serp`. + +Selectors are the current desktop layout's (verified live, Jul 2026): + +* organic result container ...... ``div.tF2Cxc`` +* title ......................... ``h3`` +* link .......................... first ```` in the block +* displayed (green) URL ......... ``cite`` (first line, when it's a URL) +* source/site name .............. ``.VuuXrf`` +* description ................... ``.VwiC3b`` +* inline date ................... ``span.YrbPuc``/``.LEwnzc`` +* emphasized keywords ........... ``em`` +* sitelinks (expanded) .......... ``td.cIkxbf`` cells in the result's card +* related searches .............. ``a.ngTNl`` (bottom block) +* people-also-ask ............... ``div.related-question-pair[data-q]`` +* PAA snippet answer ............ ``.hgKElc`` (+ source ``a`` with ``h3``) +* PAA AI answer ................. ``.n6owBd`` paragraphs (source chips + ``span.WBgIic`` stripped) +* AI Overview ................... ``#m-x-content`` widget; prose ``.n6owBd`` + + ``li.Z1qcYe``, sources ``li.h7wxwc`` +* result count .................. ``#result-stats`` + +``ponytail:`` these class names are Google's obfuscated build hashes and will +drift; each extractor degrades to ``None``/``[]`` rather than raising, so a +layout change loses a field, never the whole page. When a selector goes stale +the fix is to re-capture a live SERP and update the constant here. +""" + +from __future__ import annotations + +import re +from urllib.parse import parse_qs, urlsplit + +from scrapling.parser import Adaptor + +from .schemas import ( + AiModeResult, + AiOverviewResult, + AiSource, + OrganicResult, + PaidProduct, + PaidResult, + PeopleAlsoAskItem, + RelatedQuery, + SerpItem, + SiteLink, + SuggestedResult, +) + +_GOOGLE = "https://www.google.com" +_RESULT_COUNT_RE = re.compile(r"[\d,]+") +# Leading inline date Google prepends to a snippet, e.g. "Jul 2, 2025 · rest…". +_DATE_PREFIX_RE = re.compile(r"^[A-Z][a-z]{2}\s+\d{1,2},\s+\d{4}\s*[·\u00b7]?\s*") +_PRICE_RE = re.compile(r"\$[\d,]+(?:\.\d{2})?") + + +def _one(node, selector: str): + """First element matching ``selector`` under ``node``, or ``None``. + + Scrapling's ``Adaptor``/``Selector`` expose ``css`` (list) but no + ``css_first``, so this is the shared "first match" accessor. + """ + found = node.css(selector) + return found[0] if found else None + + +def _text(node) -> str | None: + """First non-empty text of a node, collapsed to single spaces.""" + if node is None: + return None + raw = node.get_all_text(strip=True) + if not raw: + return None + return re.sub(r"\s+", " ", raw) + + +def _abs_url(href: str | None) -> str | None: + if not href: + return None + if href.startswith("/"): + return _GOOGLE + href + return href + + +def parse_results_total(doc: Adaptor) -> int | None: + """The integer from ``#result-stats`` ("About 123 results" -> 123). + + The timing suffix "(0.53 seconds)" is cut so its digits never match. Some + SERPs (brand queries) render a visible "About 0 results" node while the + real count sits in a second, hidden ``#result-stats`` — so scan all nodes + and prefer the first non-zero count. + """ + totals: list[int] = [] + for node in doc.css("#result-stats"): + stats = _text(node) + match = _RESULT_COUNT_RE.search(stats.split("(", 1)[0]) if stats else None + if match: + totals.append(int(match.group().replace(",", ""))) + if not totals: + return None + return next((t for t in totals if t), totals[0]) + + +def _first_link(block) -> str | None: + for a in block.css("a"): + href = a.attrib.get("href") + if href and href.startswith("http"): + return href + return None + + +def _displayed_url(block) -> str | None: + cite = _one(block, "cite") + if cite is None: + return None + # cite is breadcrumb text ("https://site.com > Blog"); keep the URL head. + head = cite.get_all_text(strip=True).split("\n", 1)[0].strip() + return head if head.startswith("http") else None + + +def _inline_date(block) -> str | None: + node = _one(block, "span.YrbPuc") or _one(block, ".LEwnzc span") + date = _text(node) + # The date span carries a trailing separator ("Jul 2, 2025 · "); drop it. + return re.sub(r"\s*[·\u00b7\-]\s*$", "", date).strip() or None if date else None + + +def _description(block, date: str | None) -> str | None: + desc = _text(_one(block, ".VwiC3b")) + if not desc: + return None + # Google prepends the date to the snippet; drop it so description is clean. + if date and desc.startswith(date): + desc = desc[len(date) :] + desc = _DATE_PREFIX_RE.sub("", desc) + # Strip a separator left behind between the date and the snippet body. + desc = re.sub(r"^\s*[·\u00b7\-]\s*", "", desc) + return desc.strip() or None + + +def _site_links(block) -> list[SiteLink]: + """Expanded sitelinks of one organic result (brand queries' top result). + + The sitelinks table is a *sibling* of the ``tF2Cxc`` block inside the + result's card, so we climb to the widest ancestor that still contains only + this one result and read its ``td.cIkxbf`` cells (title ``h3``/link/ + ``.zz3gNc`` description). ``ponytail:`` only the expanded table variant is + handled; the compact inline-links variant (rare, class-drifty) parses as + no sitelinks rather than wrong ones. + """ + card = None + ancestor = block.parent + for _ in range(4): + if ancestor is None or len(ancestor.css("div.tF2Cxc")) != 1: + break + card = ancestor + ancestor = ancestor.parent + if card is None: + return [] + links: list[SiteLink] = [] + for cell in card.css("td.cIkxbf"): + title = _text(_one(cell, "h3")) + url = _first_link(cell) + if title and url: + links.append( + SiteLink(title=title, url=url, description=_text(_one(cell, ".zz3gNc"))) + ) + return links + + +def _icon(block) -> str | None: + """Favicon of a result block, as the base64 data URI the render inlines. + + The rendered desktop SERP swaps every favicon ``img.XNo5Ab`` src to a + ``data:image/...;base64,`` URI, which is exactly the shape the actor + emits for ``includeIcons``; non-data srcs (unloaded lazy images) are + skipped rather than fetched. + """ + for img in block.css("img.XNo5Ab"): + src = img.attrib.get("src") or "" + if src.startswith("data:image"): + return src + return None + + +def parse_organic(doc: Adaptor, *, include_icons: bool = False) -> list[OrganicResult]: + """Every ``div.tF2Cxc`` organic block, in page order (1-based positions).""" + results: list[OrganicResult] = [] + for i, block in enumerate(doc.css("div.tF2Cxc"), start=1): + title = _text(_one(block, "h3")) + url = _first_link(block) + if not title or not url: + continue + date = _inline_date(block) + emphasized = [] + for em in block.css("em::text"): + word = str(em).strip() + if word and word not in emphasized: + emphasized.append(word) + results.append( + OrganicResult( + title=title, + url=url, + displayedUrl=_displayed_url(block), + description=_description(block, date), + date=date, + emphasizedKeywords=emphasized, + siteLinks=_site_links(block), + icon=_icon(block) if include_icons else None, + position=i, + ) + ) + return results + + +def parse_paid_results(doc: Adaptor, *, include_icons: bool = False) -> list[PaidResult]: + """Text ads (``div[data-text-ad]``), covering the top and bottom ad blocks. + + Fields mirror an organic result: the heading is the title, the ad's anchor + is the (clean) landing URL, ``.x2VHCd`` is the green displayed URL, and the + non-heading ``.Va3FIb`` block is the description. ``adPosition`` comes from + Google's own ``data-ta-slot-pos``. + """ + ads: list[PaidResult] = [] + for block in doc.css("div[data-text-ad]"): + heading = _one(block, "div[role='heading']") + title = _text(heading) + anchor = _one(block, "a.sVXRqc") or _one(block, "a[href^='http']") + url = anchor.attrib.get("href") if anchor is not None else None + if not title or not url: + continue + # The description shares the .Va3FIb class with the heading; pick the + # longest .Va3FIb whose text isn't the title itself. + description = None + for cand in block.css(".Va3FIb"): + text = _text(cand) + if text and text != title and (description is None or len(text) > len(description)): + description = text + slot = block.attrib.get("data-ta-slot-pos") + ads.append( + PaidResult( + title=title, + url=url, + displayedUrl=_text(_one(block, ".x2VHCd")), + description=description, + icon=_icon(block) if include_icons else None, + adPosition=int(slot) if slot and slot.isdigit() else None, + ) + ) + return ads + + +def parse_paid_products(doc: Adaptor) -> list[PaidProduct]: + """Shopping / product ads (``div.pla-unit``). + + Title is the product name (``.bXPcId``); the merchant domain is the + ``data-dtld`` attribute; the clickable card's anchor is the destination; + prices are the current (``.VbBaOe``) and struck-through original + (``.tWaJ3e``) amounts, with a ``$`` regex fallback. + """ + products: list[PaidProduct] = [] + for pla in doc.css("div.pla-unit"): + title = _text(_one(pla, ".bXPcId")) + anchor = _one(pla, "a.pla-unit-single-clickable-target") + url = anchor.attrib.get("href") if anchor is not None else None + if not title or not url: + continue + prices: list[str] = [] + for sel in (".VbBaOe", ".tWaJ3e"): + price = _text(_one(pla, sel)) + if price and price not in prices: + prices.append(price) + if not prices: + prices = _PRICE_RE.findall(_text(pla) or "") + products.append( + PaidProduct( + title=title, + url=url, + displayedUrl=pla.attrib.get("data-dtld"), + description=_text(_one(pla, ".CsnLnf")), + prices=prices, + ) + ) + return products + + +def parse_related_queries(doc: Adaptor) -> list[RelatedQuery]: + """Bottom "related searches" block (``a.ngTNl``).""" + out: list[RelatedQuery] = [] + for a in doc.css("a.ngTNl"): + title = _text(a) + href = _abs_url(a.attrib.get("href")) + if title and href: + out.append(RelatedQuery(title=title, url=href)) + return out + + +def _ai_generated_text(root) -> str | None: + """Prose of an AI-generated block: paragraphs + bullets, in page order. + + Both the SERP AI Overview and PAA AI answers are built from ``.n6owBd`` + paragraphs and ``li.Z1qcYe`` bullets, with inline source chips — + ``span.WBgIic`` "YouTube +2" pills — mixed into the text; the chips are + stripped out. Google renders some blocks twice (collapsed + expanded), so + repeated fragments are dropped. + """ + parts: list[str] = [] + for block in root.css(".n6owBd, li.Z1qcYe"): + text = _text(block) + if not text: + continue + for chip in block.css("span.WBgIic"): + chip_text = _text(chip) + if chip_text: + text = text.replace(chip_text, " ") + text = re.sub(r"\s+", " ", text).strip() + if text and text not in parts: + parts.append(text) + return " ".join(parts) or None + + +def _paa_answer(pair) -> str | None: + """Answer text of an *expanded* PAA pair, or ``None`` if not loaded. + + Two shapes exist: a classic featured-snippet answer (``.hgKElc``) and an + AI-generated one (see :func:`_ai_generated_text`). + """ + snippet = _text(_one(pair, ".hgKElc")) + if snippet: + return snippet + return _ai_generated_text(pair) + + +def _paa_source(pair) -> tuple[str | None, str | None]: + """(url, title) of a snippet answer's source link; (None, None) otherwise. + + Snippet answers cite one page via an anchor wrapping an ``h3``; AI answers + cite many pages inline and carry no single source, matching the actor's + null url/title there. Google's ``#:~:text=`` highlight fragment is an + artifact of the expansion click, not part of the source URL. + """ + for a in pair.css("a[href^='http']"): + href = a.attrib.get("href") or "" + title = _text(_one(a, "h3")) + if href and title and "google.com" not in href: + return href.split("#:~:", 1)[0], title + return None, None + + +def parse_ai_overview(doc: Adaptor) -> AiOverviewResult | None: + """The inline AI Overview widget (``#m-x-content``), or ``None``. + + ``content`` is the generated prose (paragraphs + bullets, source chips + stripped); ``sources`` come from :func:`_ai_sources`. A widget that only + says "not available" parses to ``None``. + """ + box = _one(doc, "#m-x-content") + if box is None: + return None + # Expanded PAA questions embed the same widget; that's the pair's answer, + # not the page's AI Overview. + ancestor = box.parent + while ancestor is not None: + if "related-question-pair" in (ancestor.attrib.get("class") or ""): + return None + ancestor = ancestor.parent + content = _ai_generated_text(box) + if not content: + return None + return AiOverviewResult(content=content, sources=_ai_sources(box)) + + +def _ai_sources(root) -> list[AiSource]: + """Cited sources of an AI answer (AI Overview and AI Mode share the DOM). + + ``li.h7wxwc`` list items: anchor ``a.NDNGvf`` carries the URL and a + ". Opens in new tab." aria-label; ``.vhJ6Pe`` is the snippet and + the thumbnail URL sits in the lazy image's ``data-src``. Google renders + the list twice (collapsed rail + expanded sheet), so dedupe by URL. + """ + sources: list[AiSource] = [] + seen: set[str] = set() + for li in root.css("li.h7wxwc"): + anchor = _one(li, "a.NDNGvf") or _one(li, "a[href^='http']") + if anchor is None: + continue + url = anchor.attrib.get("href") + if not url or url in seen: + continue + seen.add(url) + title = (anchor.attrib.get("aria-label") or "").removesuffix( + ". Opens in new tab." + ).strip() or None + image = _one(li, "img[data-src]") + sources.append( + AiSource( + title=title, + url=url, + description=_text(_one(li, ".vhJ6Pe")), + imageUrl=image.attrib.get("data-src") if image is not None else None, + ) + ) + return sources + + +def parse_ai_mode(html: str, *, query: str, url: str) -> AiModeResult | None: + """Parse a Google AI Mode page (``/search?udm=50``) into an AiModeResult. + + The conversational answer lives in the ``[data-subtree='aimc']`` + container, built from the same blocks as the AI Overview (``.n6owBd`` + paragraphs + ``li.Z1qcYe`` bullets, sources in ``li.h7wxwc``), so the + extractors are shared. Returns ``None`` when the answer container is + missing or empty (answer failed to stream before network-idle). + """ + doc = Adaptor(html) + box = _one(doc, "[data-subtree='aimc']") + if box is None: + return None + text = _ai_generated_text(box) + if not text: + return None + return AiModeResult(text=text, sources=_ai_sources(box), query=query, url=url) + + +def parse_people_also_ask(doc: Adaptor) -> list[PeopleAlsoAskItem]: + """People-also-ask pairs (``div.related-question-pair[data-q]``). + + The fetch layer clicks the initially-served questions open (see + ``fetch._expand_paa``), so expanded pairs carry answers here. Expansion + appends extra collapsed questions; those emit with ``answer=None``. + """ + out: list[PeopleAlsoAskItem] = [] + seen: set[str] = set() + for pair in doc.css("div.related-question-pair"): + question = pair.attrib.get("data-q") or _text(_one(pair, "span")) + if not question or question in seen: + continue + seen.add(question) + url, title = _paa_source(pair) + out.append( + PeopleAlsoAskItem( + question=question, + answer=_paa_answer(pair), + url=url, + title=title, + date=_inline_date(pair), + ) + ) + return out + + +# --------------------------------------------------------------------------- +# Mobile lightweight layout (phone UAs). Verified live, Jul 2026: +# +# * result/section block ....... ``div.Gx5Zad`` (organic ones contain ``h3``) +# * anchor title ............... ``.UFvD1`` +# * displayed breadcrumb ....... ``.AKfAgb`` +# * description ................ ``.H66NU`` ("Jun 14, 2026 · snippet…") +# * PAA question ............... ``.bN5znb`` inside the "People also ask" +# block; answers are pre-rendered in the collapsed accordions (no clicks) +# * related searches ........... ``a.HA0EX[href^='/search']`` +# * AI Overview ................ block headed "AI Overview"; full text is +# pre-rendered behind the Show more clamp +# +# Result links are Google redirects (``/url?q=<target>&sa=…``). There is no +# ``#result-stats`` and no marked ad/sitelink blocks in this layout. +# --------------------------------------------------------------------------- + +_MOBILE_AIO_CHROME = ( + "AI Overview", + "Can't generate an AI overview right now. Try again later.", + "Show more", + "Show less", + "Learn more", +) + + +def _mobile_target(anchor) -> str | None: + """Landing URL of a mobile redirect anchor (``/url?q=<target>&…``).""" + href = anchor.attrib.get("href") or "" + if href.startswith("/url?"): + return (parse_qs(urlsplit(href).query).get("q") or [None])[0] + return href if href.startswith("http") else None + + +def _mobile_section(doc: Adaptor, header: str): + """The ``Gx5Zad`` block whose text starts with ``header``, or ``None``.""" + for block in doc.css("div.Gx5Zad"): + text = _text(block) or "" + if text.startswith(header): + return block + return None + + +def _mobile_organic(doc: Adaptor) -> list[OrganicResult]: + """Blocks carrying an ``h3`` title (PAA/AIO embeds carry none). + + ``ponytail:`` emphasizedKeywords and siteLinks aren't distinguishable in + this layout and emit empty; upgrade path is a fresh capture if the actor's + mobile output proves richer. + """ + results: list[OrganicResult] = [] + for block in doc.css("div.Gx5Zad"): + title = _text(_one(block, "h3")) + anchor = _one(block, "a[href^='/url?']") + url = _mobile_target(anchor) if anchor is not None else None + if not title or not url: + continue + raw_desc = _text(_one(block, ".H66NU")) + date_match = _DATE_PREFIX_RE.match(raw_desc or "") + date = re.sub(r"[\s·]+$", "", date_match.group()) if date_match else None + results.append( + OrganicResult( + title=title, + url=url, + displayedUrl=_text(_one(block, ".AKfAgb")), + description=_DATE_PREFIX_RE.sub("", raw_desc or "") or None, + date=date or None, + position=len(results) + 1, + ) + ) + return results + + +def _mobile_related(doc: Adaptor) -> list[RelatedQuery]: + out: list[RelatedQuery] = [] + for a in doc.css("a.HA0EX[href^='/search']"): + title = _text(a) + href = _abs_url(a.attrib.get("href")) + if title and href: + out.append(RelatedQuery(title=title, url=href)) + return out + + +def _mobile_paa(doc: Adaptor) -> list[PeopleAlsoAskItem]: + """Accordion entries of the "People also ask" block (answers pre-loaded).""" + section = _mobile_section(doc, "People also ask") + if section is None: + return [] + out: list[PeopleAlsoAskItem] = [] + for accordion in section.css(".Z99dvb"): + question = _text(_one(accordion, ".bN5znb")) + if not question: + continue + answer = _text(_one(accordion, ".hgMFsd")) + anchor = _one(accordion, "a[href^='/url?']") + url = _mobile_target(anchor) if anchor is not None else None + title = _text(_one(anchor, ".UFvD1")) if anchor is not None else None + out.append( + PeopleAlsoAskItem(question=question, answer=answer, url=url, title=title) + ) + return out + + +def _mobile_ai_overview(doc: Adaptor) -> AiOverviewResult | None: + """The "AI Overview" block; its full text sits behind a CSS-only clamp. + + The prose is interleaved with widget chrome (header, error stub, the + Show more/less toggle), so the block text is taken whole and the known + chrome strings are stripped out. + + ponytail: source-link titles stay inline in ``content`` (they're + interleaved with the prose in this layout, with no clean container to + split on); the upgrade path is per-child-div walking of the expansion. + """ + section = _mobile_section(doc, "AI Overview") + if section is None: + return None + content = _text(section) or "" + for chrome in _MOBILE_AIO_CHROME: + content = content.replace(chrome, " ") + content = re.sub(r"\s+", " ", content).strip() + if not content: + return None + sources: list[AiSource] = [] + seen: set[str] = set() + for anchor in section.css("a[href^='/url?']"): + url = _mobile_target(anchor) + title = _text(_one(anchor, ".UFvD1")) + # google.com targets are widget chrome ("Learn more"), not citations. + if url and url not in seen and "google.com" not in url: + seen.add(url) + sources.append(AiSource(title=title, url=url)) + return AiOverviewResult(content=content, sources=sources) + + +def parse_serp(html: str, *, include_icons: bool = False) -> SerpItem: + """Parse a full rendered SERP page into a :class:`SerpItem`. + + Provenance (``searchQuery``) is stamped by the caller; this fills the + result blocks. Missing sections yield empty lists, never errors. The + mobile lightweight layout (no ``#rso``, ``Gx5Zad`` blocks) dispatches to + the ``_mobile_*`` extractors (which carry no favicon imgs, so + ``include_icons`` is a desktop-only concern). + """ + doc = Adaptor(html) + if _one(doc, "#rso") is None and doc.css("div.Gx5Zad"): + related = _mobile_related(doc) + return SerpItem( + organicResults=_mobile_organic(doc), + relatedQueries=related, + peopleAlsoAsk=_mobile_paa(doc), + aiOverview=_mobile_ai_overview(doc), + suggestedResults=[ + SuggestedResult(title=r.title, url=r.url, position=i) + for i, r in enumerate(related, start=1) + ], + ) + related = parse_related_queries(doc) + return SerpItem( + resultsTotal=parse_results_total(doc), + organicResults=parse_organic(doc, include_icons=include_icons), + paidResults=parse_paid_results(doc, include_icons=include_icons), + paidProducts=parse_paid_products(doc), + relatedQueries=related, + peopleAlsoAsk=parse_people_also_ask(doc), + aiOverview=parse_ai_overview(doc), + # The actor synthesizes suggestedResults from the related-searches + # block, re-shaped as typed/positioned result entries. + suggestedResults=[ + SuggestedResult(title=r.title, url=r.url, position=i) + for i, r in enumerate(related, start=1) + ], + ) diff --git a/surfsense_backend/app/proprietary/platforms/google_search/query_builder.py b/surfsense_backend/app/proprietary/platforms/google_search/query_builder.py new file mode 100644 index 000000000..85869652f --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/google_search/query_builder.py @@ -0,0 +1,167 @@ +"""Pure query/URL composition for the Google Search scraper. + +The ``queries`` input is a newline-separated string mixing plain search terms +and full Google Search URLs (both accepted verbatim by the Apify actor). This +module classifies each entry, folds the advanced-filter fields into search +operators (``site:``, ``intitle:``, ``filetype:``, ``before:``/``after:``, …), +and builds the final search URL — all pure string work, no network, so it is +the part of the skeleton that is implemented and unit-tested up front (like +``url_resolver`` in the Maps scraper). +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from urllib.parse import parse_qs, quote_plus, urlparse + +from .schemas import GoogleSearchScrapeInput + +# Google redirected every country ccTLD (google.es, google.co.uk, …) to +# google.com in 2025; localization is controlled by the ``gl`` URL parameter. +# ponytail: we always hit google.com + gl=<countryCode> instead of keeping a +# ~240-entry ccTLD table; the searchQuery.domain output field still reports +# google.com, which is what the redirect would land on anyway. +_GOOGLE_DOMAIN = "google.com" + +_RESULTS_PER_PAGE = 10 + +# Relative date like "8 days", "3 months" (Apify's beforeDate/afterDate). +_RELATIVE_DATE_RE = re.compile(r"^\s*(\d+)\s*(day|week|month|year)s?\s*$", re.I) +_ABSOLUTE_DATE_RE = re.compile(r"^\s*\d{4}-\d{2}-\d{2}\s*$") + +# ponytail: calendar-exact month/year arithmetic buys nothing for a search +# date *filter*; 30/365-day approximations are within Google's own precision. +_UNIT_DAYS = {"day": 1, "week": 7, "month": 30, "year": 365} + + +@dataclass(frozen=True) +class QueryEntry: + """One line of the ``queries`` input, classified.""" + + kind: str # "term" | "url" + value: str # the search term, or the full Google Search URL + + +def parse_queries(queries: str) -> list[QueryEntry]: + """Split the newline-separated ``queries`` input into classified entries.""" + entries: list[QueryEntry] = [] + for raw in queries.splitlines(): + line = raw.strip() + if not line: + continue + if _is_search_url(line): + entries.append(QueryEntry("url", line)) + else: + entries.append(QueryEntry("term", line)) + return entries + + +def _is_search_url(line: str) -> bool: + if not line.lower().startswith(("http://", "https://")): + return False + parsed = urlparse(line) + host = parsed.hostname or "" + return "google." in host and parsed.path.startswith("/search") + + +def term_from_url(url: str) -> str | None: + """The ``q`` parameter of a Google Search URL (for provenance stamping).""" + return parse_qs(urlparse(url).query).get("q", [None])[0] + + +def resolve_date(value: str, *, now: datetime | None = None) -> str | None: + """Normalize an Apify date input to ``YYYY-MM-DD``. + + Accepts an absolute date (kept as-is) or a relative one like ``"3 months"`` + (resolved from now into the past, in UTC per the Apify spec). Returns + ``None`` for unparseable input rather than guessing. + """ + if _ABSOLUTE_DATE_RE.match(value): + return value.strip() + match = _RELATIVE_DATE_RE.match(value) + if not match: + return None + count, unit = int(match.group(1)), match.group(2).lower() + moment = (now or datetime.now(UTC)) - timedelta(days=count * _UNIT_DAYS[unit]) + return moment.strftime("%Y-%m-%d") + + +def augment_query(term: str, input_model: GoogleSearchScrapeInput) -> str: + """Fold the advanced-filter fields into the search term as operators. + + Mirrors Apify's documented behavior: ``forceExactMatch`` wraps the whole + term in quotes; ``site:`` takes precedence over ``related:``; word filters + use one ``intitle:``/``intext:``/``inurl:`` per word (never the + ``allin*:`` forms); multiple ``fileTypes`` are OR-joined; ``beforeDate``/ + ``afterDate`` become ``before:``/``after:`` operators. + """ + parts: list[str] = [] + parts.append(f'"{term}"' if input_model.forceExactMatch else term) + + if input_model.site: + parts.append(f"site:{input_model.site}") + elif input_model.relatedToSite: + parts.append(f"related:{input_model.relatedToSite}") + + for op, words in ( + ("intitle", input_model.wordsInTitle), + ("intext", input_model.wordsInText), + ("inurl", input_model.wordsInUrl), + ): + for word in words: + value = f'"{word}"' if " " in word else word + parts.append(f"{op}:{value}") + + if input_model.fileTypes: + parts.append(" OR ".join(f"filetype:{ft}" for ft in input_model.fileTypes)) + + if input_model.beforeDate: + resolved = resolve_date(input_model.beforeDate) + if resolved: + parts.append(f"before:{resolved}") + if input_model.afterDate: + resolved = resolve_date(input_model.afterDate) + if resolved: + parts.append(f"after:{resolved}") + + return " ".join(parts) + + +def build_search_url( + term: str, input_model: GoogleSearchScrapeInput, *, page: int = 1 +) -> str: + """The full ``google.com/search`` URL for one query page (1-based).""" + params: list[tuple[str, str]] = [("q", augment_query(term, input_model))] + if page > 1: + params.append(("start", str((page - 1) * _RESULTS_PER_PAGE))) + if input_model.countryCode: + params.append(("gl", input_model.countryCode.lower())) + if input_model.searchLanguage: + params.append(("lr", f"lang_{input_model.searchLanguage}")) + if input_model.languageCode: + params.append(("hl", input_model.languageCode)) + if input_model.locationUule: + params.append(("uule", input_model.locationUule)) + if input_model.quickDateRange: + params.append(("tbs", f"qdr:{input_model.quickDateRange}")) + if input_model.includeUnfilteredResults: + params.append(("filter", "0")) + query_string = "&".join(f"{k}={quote_plus(v)}" for k, v in params) + return f"https://www.{_GOOGLE_DOMAIN}/search?{query_string}" + + +def build_ai_mode_url(term: str, input_model: GoogleSearchScrapeInput) -> str: + """The Google AI Mode URL (``udm=50``) for one query. + + AI Mode takes the plain conversational query — search operators and + result-shaping parameters don't apply — plus localization. + """ + params: list[tuple[str, str]] = [("q", term), ("udm", "50")] + if input_model.countryCode: + params.append(("gl", input_model.countryCode.lower())) + if input_model.languageCode: + params.append(("hl", input_model.languageCode)) + query_string = "&".join(f"{k}={quote_plus(v)}" for k, v in params) + return f"https://www.{_GOOGLE_DOMAIN}/search?{query_string}" diff --git a/surfsense_backend/app/proprietary/platforms/google_search/schemas.py b/surfsense_backend/app/proprietary/platforms/google_search/schemas.py new file mode 100644 index 000000000..be8b5ecc2 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/google_search/schemas.py @@ -0,0 +1,252 @@ +# ruff: noqa: N815 - field names intentionally mirror the Apify camelCase spec +"""Apify-compatible input/output models for the Google Search results scraper. + +The models mirror the public Apify "Google Search Results Scraper" actor spec +so the endpoint can be a drop-in. The skeleton accepts the full input surface; +output fields the implementation does not source yet are emitted as +``None``/``[]``/``{}`` so parity is additive. + +Excluded on purpose (Apify implements them by piping into *other* actors / +third-party data brokers, out of scope here): ``perplexitySearch``, +``chatGptSearch``, ``copilotSearch``, ``geminiSearch``, ``linkProspecting``, +and the business-leads enrichment trio (``maximumLeadsEnrichmentRecords``, +``leadsEnrichmentDepartments``, ``verifyLeadsEnrichmentEmails``). They are +still *accepted* via ``extra="allow"`` — a verbatim Apify payload validates — +but they are ignored, not modeled. + +Outputs use ``extra="allow"`` on purpose: it lets us grow the output shape +without breaking existing consumers, exactly like the YouTube/Maps models. +""" + +from __future__ import annotations + +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field + +Device = Literal["DESKTOP", "MOBILE"] + + +class AiOverviewAddon(BaseModel): + """``aiOverview`` add-on toggle object (Apify nests it).""" + + model_config = ConfigDict(extra="allow") + + scrapeFullAiOverview: bool = False + + +class AiModeAddon(BaseModel): + """``aiModeSearch`` add-on toggle object (Google AI Mode on google.com).""" + + model_config = ConfigDict(extra="allow") + + enableAiMode: bool = False + + +class GoogleSearchScrapeInput(BaseModel): + """Full Apify "Google Search Results Scraper" input surface (minus the + other-actor add-ons; see module docstring). + + Semantics follow Apify: ``queries`` is a newline-separated string mixing + plain search terms and full Google Search URLs; ``maxPagesPerQuery=None`` + means one page; add-on toggles default off; ``saveHtmlToKeyValueStore`` + defaults **on** (matching the actor). + """ + + model_config = ConfigDict(extra="allow") + + # Discovery + queries: str + maxPagesPerQuery: int | None = Field(default=None, ge=1) + + # AI add-ons ($) + aiOverview: AiOverviewAddon = Field(default_factory=AiOverviewAddon) + aiModeSearch: AiModeAddon = Field(default_factory=AiModeAddon) + + # Paid results add-on ($) + focusOnPaidAds: bool = False + + # Localization + countryCode: str | None = None + searchLanguage: str = "" + languageCode: str = "" + locationUule: str | None = None + + # Advanced search filters (composed into the query string) + forceExactMatch: bool = False + site: str | None = None + relatedToSite: str | None = None + wordsInTitle: list[str] = Field(default_factory=list, max_length=32) + wordsInText: list[str] = Field(default_factory=list, max_length=32) + wordsInUrl: list[str] = Field(default_factory=list, max_length=32) + quickDateRange: str | None = None + beforeDate: str | None = None + afterDate: str | None = None + fileTypes: list[str] = Field(default_factory=list, max_length=10) + + # Result shaping + mobileResults: bool = False + includeUnfilteredResults: bool = False + saveHtml: bool = False + saveHtmlToKeyValueStore: bool = True + includeIcons: bool = False + + +class SearchQuery(BaseModel): + """Provenance block stamped on every SERP item (``searchQuery``).""" + + model_config = ConfigDict(extra="allow") + + term: str | None = None + url: str | None = None + device: Device = "DESKTOP" + page: int | None = None + type: str = "SEARCH" + domain: str | None = None + countryCode: str | None = None + languageCode: str | None = None + locationUule: str | None = None + + +class RelatedQuery(BaseModel): + model_config = ConfigDict(extra="allow") + + title: str | None = None + url: str | None = None + + +class SiteLink(BaseModel): + model_config = ConfigDict(extra="allow") + + title: str | None = None + url: str | None = None + description: str | None = None + + +class OrganicResult(BaseModel): + model_config = ConfigDict(extra="allow") + + title: str | None = None + url: str | None = None + displayedUrl: str | None = None + description: str | None = None + date: str | None = None + emphasizedKeywords: list[str] = Field(default_factory=list) + siteLinks: list[SiteLink] = Field(default_factory=list) + productInfo: dict[str, Any] = Field(default_factory=dict) + icon: str | None = None # Base64 image data, only when includeIcons + type: str = "organic" + position: int | None = None + + +class PaidResult(BaseModel): + model_config = ConfigDict(extra="allow") + + title: str | None = None + url: str | None = None + displayedUrl: str | None = None + description: str | None = None + emphasizedKeywords: list[str] = Field(default_factory=list) + siteLinks: list[SiteLink] = Field(default_factory=list) + icon: str | None = None + type: str = "paid" + adPosition: int | None = None + + +class PaidProduct(BaseModel): + model_config = ConfigDict(extra="allow") + + title: str | None = None + url: str | None = None + displayedUrl: str | None = None + description: str | None = None + prices: list[str] = Field(default_factory=list) + + +class PeopleAlsoAskItem(BaseModel): + model_config = ConfigDict(extra="allow") + + question: str | None = None + answer: str | None = None + url: str | None = None + title: str | None = None + date: str | None = None + + +class SuggestedResult(BaseModel): + """A relatedQueries entry re-emitted in result shape (Apify synthesizes + suggestedResults from the related-searches block, 1-based positions).""" + + model_config = ConfigDict(extra="allow") + + title: str | None = None + url: str | None = None + type: str = "organic" + position: int | None = None + + +class AiSource(BaseModel): + """A page cited by an AI answer (AI Overview / AI Mode).""" + + model_config = ConfigDict(extra="allow") + + title: str | None = None + url: str | None = None + description: str | None = None + imageUrl: str | None = None + + +class AiOverviewResult(BaseModel): + """The AI Overview block that appears inline on some SERPs.""" + + model_config = ConfigDict(extra="allow") + + content: str | None = None + sources: list[AiSource] = Field(default_factory=list) + + +class AiModeResult(BaseModel): + """One Google AI Mode answer (the ``aiModeResult`` add-on output).""" + + model_config = ConfigDict(extra="allow") + + engine: str = "AI Mode" + provider: str = "Google" + text: str | None = None + sources: list[AiSource] = Field(default_factory=list) + query: str | None = None + kvsHtmlUrl: str | None = None + url: str | None = None + + +class SerpItem(BaseModel): + """Apify "Google Search Results Scraper" output item (one per SERP page). + + Mirrors the actor's example JSON. Unsourced fields default to + ``None``/``[]``; ``extra="allow"`` keeps the contract open. + """ + + model_config = ConfigDict(extra="allow") + + searchQuery: SearchQuery = Field(default_factory=SearchQuery) + resultsTotal: int | None = None + + organicResults: list[OrganicResult] = Field(default_factory=list) + paidResults: list[PaidResult] = Field(default_factory=list) + paidProducts: list[PaidProduct] = Field(default_factory=list) + relatedQueries: list[RelatedQuery] = Field(default_factory=list) + peopleAlsoAsk: list[PeopleAlsoAskItem] = Field(default_factory=list) + suggestedResults: list[SuggestedResult] = Field(default_factory=list) + + # AI add-ons (populated only when the respective add-on is enabled / + # the block appears on the page) + aiOverview: AiOverviewResult | None = None + aiModeResult: AiModeResult | None = None + + # HTML capture (saveHtml / saveHtmlToKeyValueStore) + html: str | None = None + htmlSnapshotUrl: str | None = None + + def to_output(self) -> dict[str, Any]: + """Serialize to the flat dict shape Apify emits (keeps extras).""" + return self.model_dump(exclude_none=False) diff --git a/surfsense_backend/app/proprietary/platforms/google_search/scraper.py b/surfsense_backend/app/proprietary/platforms/google_search/scraper.py new file mode 100644 index 000000000..be900bdb5 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/google_search/scraper.py @@ -0,0 +1,185 @@ +"""Orchestrator for the Google Search results scraper (Apify-compatible). + +Skeleton mirroring the YouTube/Maps scraper layout: the core is the async +generator :func:`iter_serps` (one item per SERP page), :func:`scrape_serps` is +a thin collector with a caller-supplied ``limit`` guard. Each ``queries`` line +dispatches to a per-kind flow (search term / direct Google Search URL) which +is currently a no-op — each will be implemented progressively, exactly like +the YouTube and Maps flows were. +""" + +from __future__ import annotations + +import logging +from collections.abc import AsyncIterator +from typing import Any + +from .fetch import fetch_serp_html +from .parsers import parse_ai_mode, parse_serp +from .query_builder import ( + build_ai_mode_url, + build_search_url, + parse_queries, + term_from_url, +) +from .schemas import GoogleSearchScrapeInput, SearchQuery, SerpItem + +logger = logging.getLogger(__name__) + +__all__ = ["iter_serps", "scrape_serps"] + +# ``focusOnPaidAds``: Google serves ads non-deterministically, so a single +# render of a commercial query can come back with zero ads. When the add-on is +# on we re-render (fresh IP each time) until ads appear, capped here. +# ponytail: caps at 3 tries — each is a full ~10 s render, and beyond a few +# tries an ad-less result is genuinely ad-less, not just unlucky. +_PAID_ADS_MAX_TRIES = 3 + + +def _search_query_stamp( + term: str | None, url: str, page: int, input_model: GoogleSearchScrapeInput +) -> SearchQuery: + """The ``searchQuery`` provenance block Apify stamps on every item.""" + return SearchQuery( + term=term, + url=url, + device="MOBILE" if input_model.mobileResults else "DESKTOP", + page=page, + domain="google.com", + countryCode=(input_model.countryCode or "US").upper(), + languageCode=input_model.languageCode or None, + locationUule=input_model.locationUule, + ) + + +async def _serp_page_flow( + url: str, input_model: GoogleSearchScrapeInput +) -> SerpItem | None: + """Fetch and parse one SERP page into a :class:`SerpItem`. + + Renders ``url`` through the proxy and parses organic/paid/related/PAA blocks. + Returns ``None`` when the page could not be fetched (all IPs walled), so the + caller stops paging. + + With ``focusOnPaidAds`` we re-render up to :data:`_PAID_ADS_MAX_TRIES` times + until ads appear, returning the first ad-bearing SERP. If none surface, we + return the richest ad-less render seen (a render occasionally comes back + with the results container but no parsable organic blocks, so "last" is not + a safe fallback). + """ + tries = _PAID_ADS_MAX_TRIES if input_model.focusOnPaidAds else 1 + best: SerpItem | None = None + for attempt in range(1, tries + 1): + html = await fetch_serp_html(url, mobile=input_model.mobileResults) + if html is None: + logger.warning("[google_search] no SERP HTML for %s", url) + break + item = parse_serp(html, include_icons=input_model.includeIcons) + if input_model.saveHtml: + item.html = html + if not input_model.focusOnPaidAds or item.paidResults or item.paidProducts: + return item + # No ads yet; keep the render with the most organic results as fallback. + if best is None or len(item.organicResults) > len(best.organicResults): + best = item + logger.info( + "[google_search] focusOnPaidAds: no ads on try %d/%d, re-rendering", + attempt, + tries, + ) + return best + + +async def _term_flow( + term: str, input_model: GoogleSearchScrapeInput +) -> AsyncIterator[dict[str, Any]]: + """Search-term discovery: one item per result page, up to + ``maxPagesPerQuery``, stopping early when a page has no next page.""" + pages = input_model.maxPagesPerQuery or 1 + for page in range(1, pages + 1): + url = build_search_url(term, input_model, page=page) + item = await _serp_page_flow(url, input_model) + if item is None: + return + item.searchQuery = _search_query_stamp(term, url, page, input_model) + yield item.to_output() + # An empty organic page means we've run past the last result page. + if not item.organicResults: + return + + +async def _url_flow( + url: str, input_model: GoogleSearchScrapeInput +) -> AsyncIterator[dict[str, Any]]: + """Direct Google Search URL: scraped as-is (the URL's own parameters win + over the localization inputs). ``maxPagesPerQuery`` paging (rewriting the + ``start`` parameter) lands with the fetch implementation.""" + term = term_from_url(url) + item = await _serp_page_flow(url, input_model) + if item is None: + return + item.searchQuery = _search_query_stamp(term, url, 1, input_model) + yield item.to_output() + + +async def _ai_mode_flow( + term: str, input_model: GoogleSearchScrapeInput +) -> AsyncIterator[dict[str, Any]]: + """Google AI Mode add-on: one conversational AI answer (+ cited sources) + per query, emitted as its own item under ``aiModeResult``. + + Renders ``google.com/search?udm=50`` (the answer streams into + ``[data-subtree='aimc']`` before network-idle). A page whose answer + failed to generate parses to ``None`` and emits nothing. + """ + url = build_ai_mode_url(term, input_model) + html = await fetch_serp_html(url, mobile=input_model.mobileResults) + if html is None: + logger.warning("[google_search] no AI Mode HTML for %r", term) + return + result = parse_ai_mode(html, query=term, url=url) + if result is None: + logger.info("[google_search] AI Mode answer missing for %r", term) + return + item = SerpItem(aiModeResult=result) + if input_model.saveHtml: + item.html = html + item.searchQuery = _search_query_stamp(term, url, 1, input_model) + yield item.to_output() + + +async def iter_serps( + input_model: GoogleSearchScrapeInput, +) -> AsyncIterator[dict[str, Any]]: + """Yield Apify-shaped SERP items for every line of ``queries``. + + Plain terms are searched (with the advanced filters folded in as search + operators); full Google Search URLs are scraped as-is. When the AI Mode + add-on is enabled, each term additionally yields an AI Mode item. + """ + for entry in parse_queries(input_model.queries): + if entry.kind == "url": + async for item in _url_flow(entry.value, input_model): + yield item + continue + async for item in _term_flow(entry.value, input_model): + yield item + if input_model.aiModeSearch.enableAiMode: + async for item in _ai_mode_flow(entry.value, input_model): + yield item + + +async def scrape_serps( + input_model: GoogleSearchScrapeInput, *, limit: int | None = None +) -> list[dict[str, Any]]: + """Collect :func:`iter_serps` into a list, honoring an optional ``limit``. + + ``limit`` is a request-time policy guard (used by the route), NOT a + ceiling in the streaming core. + """ + results: list[dict[str, Any]] = [] + async for item in iter_serps(input_model): + results.append(item) + if limit is not None and len(results) >= limit: + break + return results diff --git a/surfsense_backend/scripts/e2e_google_search.py b/surfsense_backend/scripts/e2e_google_search.py new file mode 100644 index 000000000..acf062819 --- /dev/null +++ b/surfsense_backend/scripts/e2e_google_search.py @@ -0,0 +1,159 @@ +"""Live end-to-end checks for the Google Search scraper (needs proxy + browser). + + .venv/Scripts/python.exe scripts/e2e_google_search.py + +Covers: a plain query, a site: filter, text ads, product ads, the +focusOnPaidAds retry (commercial = ads found; non-commercial = retries capped, +organic still returned), People-Also-Ask answer expansion, sitelinks, the AI +Overview, the mobile layout, filter=0, base64 icons, and Google AI Mode. + +Pass case names as args to run a subset, e.g.: + + .venv/Scripts/python.exe scripts/e2e_google_search.py paa +""" + +import asyncio +import logging +import sys +import time +from pathlib import Path + +from dotenv import load_dotenv + +if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(encoding="utf-8") + +_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(_ROOT)) +load_dotenv(_ROOT / ".env") + +logging.basicConfig(level=logging.WARNING) +logging.getLogger("app.proprietary.platforms.google_search.scraper").setLevel(logging.INFO) + +from app.proprietary.platforms.google_search import ( # noqa: E402 + GoogleSearchScrapeInput, + scrape_serps, +) +from app.proprietary.platforms.google_search.fetch import close_sessions # noqa: E402 + + +async def run_ai_mode(label: str, *, queries: str) -> None: + print(f"\n=== {label} ===") + t0 = time.perf_counter() + inp = GoogleSearchScrapeInput( + queries=queries, countryCode="us", languageCode="en", + aiModeSearch={"enableAiMode": True}, + ) + items = await scrape_serps(inp, limit=2) + ai_items = [i for i in items if i["aiModeResult"]] + assert ai_items, f"{label}: no aiModeResult item emitted" + res = ai_items[0]["aiModeResult"] + print(f" text={len(res['text'])} chars, sources={len(res['sources'])} " + f"({time.perf_counter()-t0:.0f}s)") + print(f" {res['text'][:130]!r}") + for s in res["sources"][:3]: + print(f" src: {(s['title'] or '')[:60]!r}") + assert res["text"] and len(res["text"]) > 100, f"{label}: answer too short" + assert res["sources"], f"{label}: no cited sources" + assert "udm=50" in ai_items[0]["searchQuery"]["url"] + + +async def run( + label: str, *, expect_ads=False, expect_products=False, expect_paa_answers=False, + expect_sitelinks=False, expect_aio=False, expect_device=None, + expect_icons=False, **kwargs +) -> None: + print(f"\n=== {label} ===") + t0 = time.perf_counter() + inp = GoogleSearchScrapeInput(countryCode="us", languageCode="en", **kwargs) + items = await scrape_serps(inp, limit=1) + assert items, f"{label}: no SERP item" + it = items[0] + paa_answered = [p for p in it["peopleAlsoAsk"] if p["answer"]] + sitelinked = [o for o in it["organicResults"] if o["siteLinks"]] + print(f" term={it['searchQuery']['term']!r} resultsTotal={it['resultsTotal']}") + print(f" organic={len(it['organicResults'])} paidResults={len(it['paidResults'])} " + f"paidProducts={len(it['paidProducts'])} related={len(it['relatedQueries'])} " + f"suggested={len(it['suggestedResults'])} " + f"paa={len(it['peopleAlsoAsk'])} (answered={len(paa_answered)}) " + f"({time.perf_counter()-t0:.0f}s)") + for o in sitelinked[:2]: + print(f" [sitelinks on #{o['position']}] " + + ", ".join(s["title"] for s in o["siteLinks"][:5])) + aio = it["aiOverview"] + if aio: + print(f" [aiOverview] content={len(aio['content'])} chars, " + f"sources={len(aio['sources'])}") + print(f" {aio['content'][:110]!r}") + for s in aio["sources"][:3]: + print(f" src: {(s['title'] or '')[:55]!r}") + for a in it["paidResults"][:3]: + print(f" [ad {a['adPosition']}] {a['title'][:44]!r} {(a['url'] or '')[:45]}") + for p in it["paidProducts"][:3]: + print(f" [pla] {p['title'][:40]!r} {p['prices']} {p['displayedUrl']}") + for p in paa_answered[:3]: + print(f" [paa] {p['question'][:48]!r}") + print(f" A: {p['answer'][:90]!r}") + print(f" src: {p['url'] or '-'} | {(p['title'] or '-')[:45]}") + assert it["organicResults"], f"{label}: no organic results" + if expect_ads: + assert it["paidResults"], f"{label}: expected text ads, got none" + if expect_products: + assert it["paidProducts"], f"{label}: expected product ads, got none" + if expect_paa_answers: + assert paa_answered, f"{label}: expected PAA answers, got none" + if expect_sitelinks: + assert sitelinked, f"{label}: expected sitelinks, got none" + assert it["suggestedResults"], f"{label}: expected suggestedResults" + if expect_aio: + assert aio and aio["content"], f"{label}: expected an AI Overview" + assert aio["sources"], f"{label}: expected AI Overview sources" + if expect_device: + assert it["searchQuery"]["device"] == expect_device, ( + f"{label}: device={it['searchQuery']['device']}" + ) + if expect_icons: + iconed = [o for o in it["organicResults"] + if (o["icon"] or "").startswith("data:image")] + print(f" [icons] {len(iconed)}/{len(it['organicResults'])} organic " + f"carry a base64 favicon") + assert iconed, f"{label}: expected base64 icons on organic results" + + +_CASES = { + "plain": lambda: run("plain query", queries="python asyncio tutorial"), + "site": lambda: run("site: filter", queries="machine learning", site="arxiv.org"), + "ads": lambda: run("text ads", queries="car insurance quotes", expect_ads=True), + "products": lambda: run("product ads", queries="buy running shoes", expect_products=True), + "focus": lambda: run("focusOnPaidAds (commercial)", queries="car insurance quotes", + focusOnPaidAds=True, expect_ads=True), + "focus-neg": lambda: run("focusOnPaidAds (non-commercial, retries capped)", + queries="python asyncio tutorial", focusOnPaidAds=True), + "paa": lambda: run("people also ask", queries="what is seo", expect_paa_answers=True), + "sitelinks": lambda: run("sitelinks + suggested (brand query)", queries="amazon", + expect_sitelinks=True), + "aio": lambda: run("AI Overview", queries="benefits of green tea", expect_aio=True), + "mobile": lambda: run("mobile layout (mobileResults)", queries="best seo tools", + mobileResults=True, expect_device="MOBILE"), + "unfiltered": lambda: run("includeUnfilteredResults (filter=0)", + queries="python asyncio tutorial", + includeUnfilteredResults=True), + "icons": lambda: run("includeIcons (base64 favicons)", queries="github", + includeIcons=True, expect_icons=True), + "aimode": lambda: run_ai_mode("Google AI Mode (udm=50)", + queries="what is quantum computing"), +} + + +async def main() -> None: + names = sys.argv[1:] or list(_CASES) + try: + for name in names: + await _CASES[name]() + finally: + await close_sessions() + print("\nALL E2E OK") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/surfsense_backend/tests/unit/platforms/google_search/__init__.py b/surfsense_backend/tests/unit/platforms/google_search/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/platforms/google_search/test_skeleton.py b/surfsense_backend/tests/unit/platforms/google_search/test_skeleton.py new file mode 100644 index 000000000..2025089c5 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/google_search/test_skeleton.py @@ -0,0 +1,529 @@ +"""Offline checks for the Google Search results scraper. + +Covers the pure parts (no network): queries classification, search-operator +folding, URL building, Apify-spec schema defaults/serialization, and parsing a +rendered SERP into result blocks (against a compact synthetic fixture). The +live fetch / AI Mode flows are exercised by the e2e script, not here. +""" + +from datetime import UTC, datetime + +from app.proprietary.platforms.google_search import ( + GoogleSearchScrapeInput, + SerpItem, +) +from app.proprietary.platforms.google_search.parsers import parse_serp +from app.proprietary.platforms.google_search.query_builder import ( + augment_query, + build_search_url, + parse_queries, + resolve_date, + term_from_url, +) + + +def test_parse_queries_classifies_terms_and_urls(): + entries = parse_queries( + "best SEO tools\n" + "\n" # blank lines are skipped + " https://www.google.com/search?q=apify+web+scraping \n" + "javascript OR python site:stackoverflow.com\n" + "https://example.com/search?q=not-google\n" + ) + assert [(e.kind, e.value) for e in entries] == [ + ("term", "best SEO tools"), + ("url", "https://www.google.com/search?q=apify+web+scraping"), + ("term", "javascript OR python site:stackoverflow.com"), + # non-Google URLs are treated as literal search terms, not scrape URLs + ("term", "https://example.com/search?q=not-google"), + ] + + +def test_term_from_url(): + assert term_from_url("https://www.google.com/search?q=apify+scraping") == ( + "apify scraping" + ) + assert term_from_url("https://www.google.com/search") is None + + +def test_augment_query_folds_all_filters(): + inp = GoogleSearchScrapeInput( + queries="x", + forceExactMatch=True, + site="allrecipes.com", + relatedToSite="ignored.com", # site: wins + wordsInTitle=["easy apple", "pie"], + wordsInText=["cinnamon"], + wordsInUrl=["recipe"], + fileTypes=["pdf", "doc"], + beforeDate="2024-12-31", + afterDate="2024-01-01", + ) + assert augment_query("apple pie", inp) == ( + '"apple pie" site:allrecipes.com intitle:"easy apple" intitle:pie ' + "intext:cinnamon inurl:recipe filetype:pdf OR filetype:doc " + "before:2024-12-31 after:2024-01-01" + ) + + +def test_augment_query_related_used_when_no_site(): + inp = GoogleSearchScrapeInput(queries="x", relatedToSite="example.com") + assert augment_query("q", inp) == "q related:example.com" + + +def test_resolve_date_absolute_and_relative(): + assert resolve_date("2024-05-03") == "2024-05-03" + now = datetime(2026, 7, 3, tzinfo=UTC) + assert resolve_date("8 days", now=now) == "2026-06-25" + assert resolve_date("3 months", now=now) == "2026-04-04" + assert resolve_date("1 year", now=now) == "2025-07-03" + assert resolve_date("someday") is None + + +def test_build_search_url_localization_and_paging(): + inp = GoogleSearchScrapeInput( + queries="x", + countryCode="ES", + searchLanguage="de", + languageCode="en", + locationUule="w+CAIQICIhVW5pdGVkIFN0YXRlcyx1c2E=", + quickDateRange="m6", + includeUnfilteredResults=True, + ) + url = build_search_url("hotels in Seattle", inp, page=2) + assert url.startswith("https://www.google.com/search?q=hotels+in+Seattle") + assert "start=10" in url + assert "gl=es" in url + assert "lr=lang_de" in url + assert "hl=en" in url + assert "uule=w%2BCAIQICIhVW5pdGVkIFN0YXRlcyx1c2E%3D" in url + assert "tbs=qdr%3Am6" in url + assert "filter=0" in url + + plain = build_search_url("q", GoogleSearchScrapeInput(queries="x")) + assert "start=" not in plain # page 1 carries no offset + + +def test_scrape_input_defaults_match_apify_spec(): + inp = GoogleSearchScrapeInput(queries="best SEO tools") + assert inp.maxPagesPerQuery is None # unset = 1 page + assert inp.aiOverview.scrapeFullAiOverview is False + assert inp.aiModeSearch.enableAiMode is False + assert inp.focusOnPaidAds is False + assert inp.forceExactMatch is False + assert inp.mobileResults is False + assert inp.saveHtml is False + assert inp.saveHtmlToKeyValueStore is True # actor default is ON + assert inp.includeIcons is False + # Excluded other-actor add-ons are still accepted (extra="allow") so a + # verbatim Apify payload validates; they are ignored, not modeled. + GoogleSearchScrapeInput( + queries="q", + perplexitySearch={"enablePerplexity": True}, + chatGptSearch={"enableChatGpt": True}, + maximumLeadsEnrichmentRecords=5, + ) + + +def test_output_item_serializes_full_shape(): + item = SerpItem(resultsTotal=42).to_output() + assert item["resultsTotal"] == 42 + assert item["organicResults"] == [] + assert item["paidResults"] == [] + assert item["relatedQueries"] == [] + assert item["peopleAlsoAsk"] == [] + assert item["aiModeResult"] is None # unsourced fields still emitted + assert item["searchQuery"]["device"] == "DESKTOP" + assert item["searchQuery"]["type"] == "SEARCH" + + +# Compact stand-in for a rendered SERP: the selectors parse_serp relies on, +# without the ~1 MB of a live capture. If Google's layout drifts, the fix is in +# parsers.py's selector constants; this fixture pins the expected extraction. +_SERP_FIXTURE = """ +<html><body> + <div id="result-stats">About 1,230 results (0.42 seconds)</div> + <div id="rso"> + <div class="card"> + <div class="tF2Cxc"> + <img class="XNo5Ab" src="data:image/png;base64,iVBORfake"> + <a href="https://example.com/guide"> + <h3>The Example Guide</h3> + <cite>https://example.com<span> > Blog</span></cite> + <span class="VuuXrf">Example</span> + </a> + <div class="VwiC3b"><span class="YrbPuc">Jul 2, 2025 · </span>Learn <em>apple pie</em> the easy way.</div> + </div> + <table><tbody><tr> + <td class="cIkxbf"> + <h3><a href="https://example.com/recipes">Recipes</a></h3> + <div class="zz3gNc">All our recipes ...</div> + </td> + <td class="cIkxbf"> + <h3><a href="https://example.com/about">About</a></h3> + </td> + </tr></tbody></table> + </div> + <div class="tF2Cxc"> + <a href="https://second.example/post"><h3>Second Result</h3> + <cite>second.example</cite></a> + <div class="VwiC3b">No date here, just a snippet.</div> + </div> + </div> + <div id="tads"> + <div data-text-ad="1" data-ta-slot-pos="1"> + <a class="sVXRqc" href="https://shop.example/lp"> + <div role="heading" class="Va3FIb"><span>Buy Apple Pie Online</span></div> + </a> + <span class="x2VHCd">https://shop.example</span> + <div class="Va3FIb">Fresh pies delivered daily. Order now and save 20%.</div> + </div> + </div> + <div class="pla-unit" data-dtld="pieshop.example"> + <a class="pla-unit-single-clickable-target" href="https://pieshop.example/p/123"></a> + <div class="bXPcId">Homemade Apple Pie 9-inch</div> + <div class="CsnLnf">Pie Shop</div> + <span class="VbBaOe">$24.99</span> + <span class="tWaJ3e">$30</span> + </div> + <div class="related-question-pair" data-q="What is apple pie?"> + <span class="hgKElc">A pie with an apple filling.</span> + <a href="https://pies.example/apple#:~:text=A%20pie"><h3>Apple pie - Pies</h3></a> + </div> + <div class="related-question-pair" data-q="How to bake?"> + <div class="n6owBd">Preheat the oven.<span class="WBgIic">Wiki +2</span></div> + <div class="n6owBd">Bake until golden.</div> + </div> + <div class="related-question-pair" data-q="Why bake?"></div> + <div id="m-x-content"> + <div class="n6owBd">Apple pie is a classic dessert.<span class="WBgIic">Wiki +3</span></div> + <ul><li class="Z1qcYe">Best served warm.</li></ul> + <ul> + <li class="h7wxwc"> + <a class="NDNGvf" aria-label="Pie History - Pies.example. Opens in new tab." + href="https://pies.example/history"></a> + <span class="vhJ6Pe">A short history of pie.</span> + <img data-src="https://thumbs.example/pie.jpg" src="data:image/gif;base64,x"> + </li> + <li class="h7wxwc"> + <a class="NDNGvf" aria-label="Pie History - Pies.example. Opens in new tab." + href="https://pies.example/history"></a> + </li> + </ul> + </div> + <div id="botstuff"> + <a class="ngTNl" href="/search?q=easy+apple+pie">easy apple pie</a> + <a class="ngTNl" href="/search?q=apple+pie+recipe">apple pie recipe</a> + <a class="fl" href="/search?q=x&start=10">2</a> + </div> +</body></html> +""" + + +def test_parse_serp_extracts_all_blocks(): + item = parse_serp(_SERP_FIXTURE) + + assert item.resultsTotal == 1230 + + assert len(item.organicResults) == 2 + first = item.organicResults[0] + assert first.position == 1 + assert first.title == "The Example Guide" + assert first.url == "https://example.com/guide" + assert first.displayedUrl == "https://example.com" + assert first.date == "Jul 2, 2025" + assert first.emphasizedKeywords == ["apple pie"] + # The leading date is stripped from the snippet. + assert first.description == "Learn apple pie the easy way." + # Sitelinks come from the sibling table inside this result's card. + assert [(s.title, s.url) for s in first.siteLinks] == [ + ("Recipes", "https://example.com/recipes"), + ("About", "https://example.com/about"), + ] + assert first.siteLinks[0].description == "All our recipes ..." + assert first.siteLinks[1].description is None + + second = item.organicResults[1] + assert second.date is None + assert second.displayedUrl is None # cite without an http head + assert second.siteLinks == [] # no card of its own + + # Icons are opt-in: absent by default, the inlined data URI when asked. + assert first.icon is None + with_icons = parse_serp(_SERP_FIXTURE, include_icons=True) + assert with_icons.organicResults[0].icon == "data:image/png;base64,iVBORfake" + assert with_icons.organicResults[1].icon is None # block carries no favicon + + # Text ad: heading is the title, the anchor is the clean landing URL, and + # the non-heading .Va3FIb is the description (not the title echo). + assert len(item.paidResults) == 1 + ad = item.paidResults[0] + assert ad.title == "Buy Apple Pie Online" + assert ad.url == "https://shop.example/lp" + assert ad.displayedUrl == "https://shop.example" + assert ad.description == "Fresh pies delivered daily. Order now and save 20%." + assert ad.adPosition == 1 + + # Product ad: title, merchant, domain, and both prices. + assert len(item.paidProducts) == 1 + prod = item.paidProducts[0] + assert prod.title == "Homemade Apple Pie 9-inch" + assert prod.url == "https://pieshop.example/p/123" + assert prod.displayedUrl == "pieshop.example" + assert prod.description == "Pie Shop" + assert prod.prices == ["$24.99", "$30"] + + # Related searches exclude the numeric pagination anchor (a.fl). + assert [r.title for r in item.relatedQueries] == ["easy apple pie", "apple pie recipe"] + assert item.relatedQueries[0].url == "https://www.google.com/search?q=easy+apple+pie" + + # suggestedResults are the related queries re-shaped with type/position. + assert [(s.position, s.title, s.type) for s in item.suggestedResults] == [ + (1, "easy apple pie", "organic"), + (2, "apple pie recipe", "organic"), + ] + assert item.suggestedResults[0].url == item.relatedQueries[0].url + + assert [p.question for p in item.peopleAlsoAsk] == [ + "What is apple pie?", + "How to bake?", + "Why bake?", + ] + # Snippet-style answer: text + single source link (highlight fragment cut). + snippet = item.peopleAlsoAsk[0] + assert snippet.answer == "A pie with an apple filling." + assert snippet.url == "https://pies.example/apple" + assert snippet.title == "Apple pie - Pies" + # AI-style answer: paragraphs joined, inline source chips stripped. + ai = item.peopleAlsoAsk[1] + assert ai.answer == "Preheat the oven. Bake until golden." + assert ai.url is None and ai.title is None + # Collapsed (never-expanded) question stays question-only. + assert item.peopleAlsoAsk[2].answer is None + + # AI Overview: prose (chips stripped) + bullet, sources deduped by URL. + aio = item.aiOverview + assert aio is not None + assert aio.content == "Apple pie is a classic dessert. Best served warm." + assert len(aio.sources) == 1 + src = aio.sources[0] + assert src.title == "Pie History - Pies.example" + assert src.url == "https://pies.example/history" + assert src.description == "A short history of pie." + assert src.imageUrl == "https://thumbs.example/pie.jpg" + + +def test_parse_serp_empty_page_is_safe(): + item = parse_serp("<html><body>nothing here</body></html>") + assert item.resultsTotal is None + assert item.organicResults == [] + assert item.paidResults == [] + assert item.paidProducts == [] + assert item.relatedQueries == [] + assert item.peopleAlsoAsk == [] + assert item.aiOverview is None + + +def test_ai_overview_inside_paa_pair_is_not_page_overview(): + # An expanded PAA question embeds the same widget; it must stay the pair's + # answer, not leak into the page-level aiOverview. + html = """ + <html><body> + <div class="related-question-pair" data-q="What is pie?"> + <div id="m-x-content"><div class="n6owBd">Pie is dessert.</div></div> + </div> + </body></html> + """ + item = parse_serp(html) + assert item.aiOverview is None + assert item.peopleAlsoAsk[0].answer == "Pie is dessert." + + +# Mobile lightweight layout (phone UA render): Gx5Zad blocks, /url? redirect +# anchors, pre-loaded PAA accordions, clamped AI Overview. Mirrors a Jul 2026 +# live capture, compacted. +_MOBILE_FIXTURE = """ +<html><body><div id="main"> + <div class="Gx5Zad"> + <div class="ilUpNd"><span class="vA9HTb">AI Overview</span></div> + <div class="frRrnc"><span>Pie is a baked dish.</span></div> + <div class="Z99dvb"> + <div class="duf-h"><div class="A104Bf BMhZCf">Show more</div> + <div class="A104Bf hf22jd">Show less</div></div> + <div id="t1">Best served warm. + <a href="/url?q=https://pies.example/history&sa=U"> + <div class="UFvD1">Pie History</div></a> + <a href="https://support.google.com/websearch?p=ai_overviews">Learn more</a> + </div> + </div> + </div> + <div class="Gx5Zad"> + <a href="/url?q=https://pies.example/apple&sa=U"> + <h3><div class="UFvD1">Apple Pie Recipe</div></h3> + <div class="AKfAgb">pies.example › apple</div> + </a> + <div class="H66NU"><span class="UK5aid">Jun 14, 2026</span><span> · </span> + The best apple pie recipe.</div> + </div> + <div class="Gx5Zad"> + <div class="ilUpNd"><span class="vA9HTb">People also ask</span></div> + <div class="Z99dvb"> + <div class="duf-h"><div class="bN5znb">What is pie?</div></div> + <div id="t2"><div class="hgMFsd">A pie is a baked dish.</div> + <a href="/url?q=https://pies.example/what&sa=U"> + <div class="UFvD1">What is pie - Pies</div></a> + </div> + </div> + </div> + <div class="Gx5Zad"> + <div class="ilUpNd">People also search for</div> + <a class="HA0EX" href="/search?q=easy+pie"><div>easy pie</div></a> + </div> +</div></body></html> +""" + + +def test_parse_serp_mobile_layout(): + item = parse_serp(_MOBILE_FIXTURE) + + assert len(item.organicResults) == 1 + org = item.organicResults[0] + assert org.title == "Apple Pie Recipe" + assert org.url == "https://pies.example/apple" # redirect unwrapped + assert org.displayedUrl == "pies.example › apple" # noqa: RUF001 - Google's breadcrumb char + assert org.date == "Jun 14, 2026" + assert org.description == "The best apple pie recipe." + assert org.position == 1 + + assert [r.title for r in item.relatedQueries] == ["easy pie"] + assert item.relatedQueries[0].url.startswith("https://www.google.com/search") + assert item.suggestedResults[0].title == "easy pie" + + paa = item.peopleAlsoAsk[0] + assert paa.question == "What is pie?" + assert paa.answer == "A pie is a baked dish." + assert paa.url == "https://pies.example/what" + assert paa.title == "What is pie - Pies" + + aio = item.aiOverview + assert aio is not None + # Prose + expansion joined, Show more/less chrome stripped. + assert aio.content == "Pie is a baked dish. Best served warm. Pie History" + assert [s.url for s in aio.sources] == ["https://pies.example/history"] + assert aio.sources[0].title == "Pie History" + + +# Google AI Mode page (udm=50): the conversational answer streams into the +# [data-subtree='aimc'] container, built from the same blocks as the AI +# Overview (n6owBd paragraphs, Z1qcYe bullets, h7wxwc sources). +_AI_MODE_FIXTURE = """ +<html><body><div id="search"> + <div data-subtree="aimc"> + <div class="n6owBd">Quantum computing uses qubits.<span class="WBgIic">IBM +2</span></div> + <ul><li class="Z1qcYe">Superposition: both at once.</li></ul> + <ul> + <li class="h7wxwc"> + <a class="NDNGvf" aria-label="What Is Quantum Computing? | IBM. Opens in new tab." + href="https://www.ibm.com/think/topics/quantum-computing"></a> + <span class="vhJ6Pe">Quantum computing, defined.</span> + </li> + </ul> + </div> +</div></body></html> +""" + + +def test_parse_ai_mode(): + from app.proprietary.platforms.google_search.parsers import parse_ai_mode + + result = parse_ai_mode( + _AI_MODE_FIXTURE, query="what is quantum computing", url="https://g/x" + ) + assert result is not None + assert result.engine == "AI Mode" and result.provider == "Google" + assert result.query == "what is quantum computing" + assert result.url == "https://g/x" + # Prose + bullets joined, source chips stripped. + assert result.text == "Quantum computing uses qubits. Superposition: both at once." + assert len(result.sources) == 1 + src = result.sources[0] + assert src.title == "What Is Quantum Computing? | IBM" + assert src.url == "https://www.ibm.com/think/topics/quantum-computing" + assert src.description == "Quantum computing, defined." + + # A page without the answer container (e.g. generation failed) is None. + assert parse_ai_mode("<html><body></body></html>", query="q", url="u") is None + + +async def test_ai_mode_flow_emits_item(monkeypatch): + from app.proprietary.platforms.google_search import scraper + + async def fake_fetch(url, *, mobile=False): + # SERP flow gets a plain SERP; the AI Mode flow's udm=50 URL gets + # the AI Mode page. + return _AI_MODE_FIXTURE if "udm=50" in url else _NO_ADS_FIXTURE + + monkeypatch.setattr(scraper, "fetch_serp_html", fake_fetch) + + items = await scraper.scrape_serps( + GoogleSearchScrapeInput( + queries="what is quantum computing", + aiModeSearch={"enableAiMode": True}, + ) + ) + assert len(items) == 2 # SERP item + AI Mode item + ai_item = items[1] + assert ai_item["aiModeResult"]["text"].startswith("Quantum computing") + assert ai_item["aiModeResult"]["query"] == "what is quantum computing" + assert "udm=50" in ai_item["searchQuery"]["url"] + assert ai_item["organicResults"] == [] + + +# An organic-only page (no ad blocks) for the focusOnPaidAds retry test. +_NO_ADS_FIXTURE = """ +<html><body><div id="rso"> + <div class="tF2Cxc"><a href="https://x.example/a"><h3>Only Organic</h3></a></div> +</div></body></html> +""" + + +async def test_focus_on_paid_ads_retries_until_ads(monkeypatch): + from app.proprietary.platforms.google_search import scraper + + # First two renders have no ads, the third does; focusOnPaidAds should keep + # re-rendering and return the ad-bearing page. + pages = iter([_NO_ADS_FIXTURE, _NO_ADS_FIXTURE, _SERP_FIXTURE]) + calls = 0 + + async def fake_fetch(_url, *, mobile=False): + nonlocal calls + calls += 1 + return next(pages) + + monkeypatch.setattr(scraper, "fetch_serp_html", fake_fetch) + + items = await scraper.scrape_serps( + GoogleSearchScrapeInput(queries="car insurance", focusOnPaidAds=True), limit=1 + ) + assert calls == 3 # retried past the two ad-less renders + assert items[0]["paidResults"], "should return the ad-bearing SERP" + + +async def test_no_focus_takes_first_render(monkeypatch): + from app.proprietary.platforms.google_search import scraper + + calls = 0 + + async def fake_fetch(_url, *, mobile=False): + nonlocal calls + calls += 1 + return _NO_ADS_FIXTURE + + monkeypatch.setattr(scraper, "fetch_serp_html", fake_fetch) + + items = await scraper.scrape_serps( + GoogleSearchScrapeInput(queries="anything"), limit=1 + ) + assert calls == 1 # no retry without focusOnPaidAds + assert items[0]["paidResults"] == [] + assert items[0]["organicResults"]