From 5dac302938bc48b35ed38d36472b4894766e3200 Mon Sep 17 00:00:00 2001 From: feder-cr <85809106+feder-cr@users.noreply.github.com> Date: Tue, 9 Jun 2026 15:40:02 +0200 Subject: [PATCH] test: activate the full e2e (browser-driving) suite + add `fetch --force` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 138 @pytest.mark.e2e tests were doubly inactive: deselected by addopts AND skipped without a cached binary — and 3 of the 6 per-file firefox_binary fixtures silently ignored INVPW_BINARY_PATH, so they'd test whatever was cached even when you pointed the suite elsewhere (a false-confidence trap). - Centralize firefox_binary into conftest.py (env INVPW_BINARY_PATH → cache → skip); delete the 6 duplicates. Unify test_webrtc_realness onto the same env. - scripts/run_e2e.py: one command that runs ALL e2e against a chosen binary, with reruns so an under-load interaction flake (dblclick/hover pass 3/3 in isolation) self-heals while a real break fails every attempt. The webrtc e2e fake a TCP-only SOCKS locally, so the suite is offline. This is the MANDATORY pre-release browser gate (local — hosted runners are too interaction-flaky). - Running the suite against firefox-9 surfaced a real gap: `invisible_playwright fetch --force` was unrecognized (the subparser took no args) though the e2e test + docstring expect it. Implement it: drop the cached version dir, refetch. - Add pytest-rerunfailures + playwright to the dev extras. Baseline against firefox-9: 136 passed, 1 skipped (linux_only on win host), 1 was the --force gap now fixed. --- pyproject.toml | 2 +- scripts/run_e2e.py | 67 +++++++++++++++++++++++++++ src/invisible_playwright/cli.py | 14 +++++- tests/conftest.py | 37 +++++++++++++++ tests/test_cross_origin_iframe.py | 17 ------- tests/test_e2e.py | 24 ---------- tests/test_fingerprint_consistency.py | 26 ----------- tests/test_fingerprint_surface.py | 28 ----------- tests/test_mouse.py | 13 ------ tests/test_service_worker.py | 23 --------- tests/test_webrtc_realness.py | 4 +- 11 files changed, 120 insertions(+), 135 deletions(-) create mode 100644 scripts/run_e2e.py diff --git a/pyproject.toml b/pyproject.toml index d08f552..4bf9262 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ dependencies = [ ] [project.optional-dependencies] -dev = ["pytest>=7", "pytest-mock>=3", "responses>=0.24", "build>=1"] +dev = ["pytest>=7", "pytest-mock>=3", "responses>=0.24", "build>=1", "pytest-rerunfailures>=14", "playwright>=1.40"] [tool.pytest.ini_options] markers = [ diff --git a/scripts/run_e2e.py b/scripts/run_e2e.py new file mode 100644 index 0000000..d1bfac3 --- /dev/null +++ b/scripts/run_e2e.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Run the FULL e2e suite (every test that opens the browser) against a binary. + +The 138 ``@pytest.mark.e2e`` tests are excluded from the default `pytest` run +(`addopts = -m 'not slow and not e2e'`) because they need a real Firefox binary +and a display, and they skip themselves when no binary is available. That makes +them easy to forget — and "we can't afford for something to not work". This is +the gate that runs them all, deliberately, against a chosen binary. + +It is the MANDATORY pre-release e2e gate: run it green against the freshly-built +release binary BEFORE un-drafting a firefox-N (alongside the fppro + WebRTC +realness gates). It is NOT in the public CI drive-gate — the hosted runners are +content-process unstable under a heavy headless interaction sequence (see +70-known-bugs / 60-ci-release-pipeline); this runs locally on reliable hardware. + +Flake-resilience: under full-suite load a couple of interaction tests (dblclick, +hover/mouseenter) can flake even though they pass 3/3 in isolation, so failures +are reran up to twice on the known transient signatures. A genuinely broken +binary fails all attempts. The webrtc e2e fake a TCP-only SOCKS locally (no +proxy/secrets), so the whole suite is offline. + +Usage: + python scripts/run_e2e.py + python scripts/run_e2e.py # uses $INVPW_BINARY_PATH +""" +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +_RERUN_SIGNATURES = "Timeout|context was destroyed|was detached|not visible|because of a navigation|TargetClosed" + + +def main() -> int: + binary = sys.argv[1] if len(sys.argv) > 1 else os.environ.get("INVPW_BINARY_PATH") + if not binary: + print("usage: run_e2e.py (or set INVPW_BINARY_PATH)", file=sys.stderr) + return 2 + if not Path(binary).exists(): + print(f"ERROR: binary not found: {binary}", file=sys.stderr) + return 2 + + env = dict(os.environ) + # One setting drives the whole suite: conftest's firefox_binary fixture and + # the webrtc e2e both resolve from these. + env["INVPW_BINARY_PATH"] = binary + env["STEALTHFOX_E2E_BINARY"] = binary + + repo = Path(__file__).resolve().parent.parent + cmd = [ + sys.executable, "-m", "pytest", + "-m", "e2e", + "-o", "addopts=", # override the default 'not e2e' deselection + "--reruns", "2", "--reruns-delay", "1", + "--only-rerun", _RERUN_SIGNATURES, + "-p", "no:cacheprovider", + "-q", "--tb=short", + ] + sys.argv[2:] + print(f"[run_e2e] binary={binary}") + print(f"[run_e2e] {' '.join(cmd)}") + return subprocess.run(cmd, cwd=repo, env=env).returncode + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/invisible_playwright/cli.py b/src/invisible_playwright/cli.py index e6057cf..eb12067 100644 --- a/src/invisible_playwright/cli.py +++ b/src/invisible_playwright/cli.py @@ -10,7 +10,15 @@ from .constants import BINARY_VERSION, FIREFOX_UPSTREAM_VERSION from .download import cache_root, ensure_binary -def _cmd_fetch(_args: argparse.Namespace) -> int: +def _cmd_fetch(args: argparse.Namespace) -> int: + # --force: re-download even if already cached (drop the cached version dir, + # then let ensure_binary fetch it fresh). Useful to recover a corrupted cache + # or re-pull after a re-published release. + if getattr(args, "force", False): + from .download import cache_dir_for_version + d = cache_dir_for_version() + if d.exists(): + shutil.rmtree(d, ignore_errors=True) path = ensure_binary() print(path) return 0 @@ -52,7 +60,9 @@ def build_parser() -> argparse.ArgumentParser: ) sub = p.add_subparsers(dest="cmd") - sub.add_parser("fetch", help="download the patched Firefox binary") + fetch_p = sub.add_parser("fetch", help="download the patched Firefox binary") + fetch_p.add_argument("--force", action="store_true", + help="re-download even if already cached") sub.add_parser("path", help="print the absolute path to the cached binary") sub.add_parser("version", help="print wrapper and binary versions") sub.add_parser("clear-cache", help="remove all cached binaries") diff --git a/tests/conftest.py b/tests/conftest.py index 429aa6d..900732b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,8 +1,12 @@ +import os import random +import sys +from pathlib import Path import pytest from invisible_playwright._fpforge import generate_profile +from invisible_playwright.constants import BINARY_ENTRY_REL @pytest.fixture @@ -15,3 +19,36 @@ def deterministic_rng(): def sample_profile(): """A Profile generated from seed=42 for reuse across tests.""" return generate_profile(seed=42) + + +@pytest.fixture(scope="session") +def firefox_binary(): + """Locate the patched Firefox binary for E2E tests, or skip cleanly. + + Single source of truth for every E2E test (previously each test file had its + own copy — and three of them silently ignored INVPW_BINARY_PATH, so they kept + testing whatever was in the cache even when you pointed the suite at a + specific build: a false-confidence trap). Lookup order: + + 1. ``INVPW_BINARY_PATH`` env var — point the whole suite at a local build + or a freshly-extracted release (this is how the full-suite gate runs). + 2. Cached binary under ``cache_dir_for_version()`` (post ``fetch``). + 3. Skip — we never trigger an implicit multi-hundred-MB network download + inside a test run. + """ + env_path = os.environ.get("INVPW_BINARY_PATH") + if env_path: + if Path(env_path).exists(): + return env_path + pytest.skip(f"INVPW_BINARY_PATH={env_path!r} does not exist") + + if sys.platform not in BINARY_ENTRY_REL: + pytest.skip(f"unsupported platform: {sys.platform}") + from invisible_playwright.download import cache_dir_for_version + entry = cache_dir_for_version() / BINARY_ENTRY_REL[sys.platform] + if not entry.exists(): + pytest.skip( + "patched Firefox binary not cached and INVPW_BINARY_PATH unset; " + "set INVPW_BINARY_PATH= or run `invisible-playwright fetch`" + ) + return str(entry) diff --git a/tests/test_cross_origin_iframe.py b/tests/test_cross_origin_iframe.py index 8be39ac..26df483 100644 --- a/tests/test_cross_origin_iframe.py +++ b/tests/test_cross_origin_iframe.py @@ -31,7 +31,6 @@ random free ports. from __future__ import annotations import socket -import sys import threading from http.server import BaseHTTPRequestHandler, HTTPServer @@ -165,22 +164,6 @@ def cross_origin_harness(): sb.shutdown() -@pytest.fixture(scope="session") -def firefox_binary(): - """Locate the cached patched Firefox binary or skip.""" - from invisible_playwright.constants import BINARY_ENTRY_REL - if sys.platform not in BINARY_ENTRY_REL: - pytest.skip(f"unsupported platform: {sys.platform}") - from invisible_playwright.download import cache_dir_for_version - entry = cache_dir_for_version() / BINARY_ENTRY_REL[sys.platform] - if not entry.exists(): - pytest.skip( - "patched Firefox binary not cached; run `invisible-playwright fetch` " - "to enable E2E tests" - ) - return str(entry) - - @pytest.mark.e2e def test_cross_origin_iframe_url_appears_in_page_frames(firefox_binary, cross_origin_harness): """``page.frames`` must list the cross-origin iframe with its real URL. diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 35fad98..d2e59f2 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -8,33 +8,9 @@ handling) do not need a binary and always run. """ from __future__ import annotations -import sys - import pytest from invisible_playwright import InvisiblePlaywright -from invisible_playwright.constants import BINARY_ENTRY_REL - - -@pytest.fixture(scope="session") -def firefox_binary(): - """Locate the patched Firefox binary or skip the calling test. - - We do NOT trigger a network download here: ``ensure_binary`` would - pull a multi-hundred-megabyte archive from a private release, - which is not appropriate inside a unit/E2E test run. Instead we - look for an already-cached binary; if missing we skip. - """ - if sys.platform not in BINARY_ENTRY_REL: - pytest.skip(f"unsupported platform: {sys.platform}") - from invisible_playwright.download import cache_dir_for_version - entry = cache_dir_for_version() / BINARY_ENTRY_REL[sys.platform] - if not entry.exists(): - pytest.skip( - "patched Firefox binary not cached; run `invisible-playwright fetch` " - "to enable E2E tests" - ) - return str(entry) # ──────────────────────────────────────────────────────────────────── diff --git a/tests/test_fingerprint_consistency.py b/tests/test_fingerprint_consistency.py index 0a53d27..9912299 100644 --- a/tests/test_fingerprint_consistency.py +++ b/tests/test_fingerprint_consistency.py @@ -25,12 +25,9 @@ Run only this file: """ from __future__ import annotations -import sys - import pytest from invisible_playwright import InvisiblePlaywright -from invisible_playwright.constants import BINARY_ENTRY_REL PIN = { @@ -45,29 +42,6 @@ PIN = { } -@pytest.fixture(scope="session") -def firefox_binary(): - """See test_fingerprint_surface.firefox_binary for the lookup chain.""" - import os - env_path = os.environ.get("INVPW_BINARY_PATH") - if env_path: - from pathlib import Path - if Path(env_path).exists(): - return env_path - pytest.skip(f"INVPW_BINARY_PATH={env_path!r} does not exist") - if sys.platform not in BINARY_ENTRY_REL: - pytest.skip(f"unsupported platform: {sys.platform}") - from invisible_playwright.download import cache_dir_for_version - entry = cache_dir_for_version() / BINARY_ENTRY_REL[sys.platform] - if not entry.exists(): - pytest.skip( - "patched Firefox not cached; run " - "`python -m invisible_playwright fetch` first, or set " - "INVPW_BINARY_PATH to a local build" - ) - return str(entry) - - @pytest.fixture(scope="module") def page(firefox_binary): with InvisiblePlaywright( diff --git a/tests/test_fingerprint_surface.py b/tests/test_fingerprint_surface.py index cd30076..bd263cf 100644 --- a/tests/test_fingerprint_surface.py +++ b/tests/test_fingerprint_surface.py @@ -27,12 +27,10 @@ Run only this file: from __future__ import annotations import re -import sys import pytest from invisible_playwright import InvisiblePlaywright -from invisible_playwright.constants import BINARY_ENTRY_REL # ──────────────────────────────────────────────────────────────────── @@ -53,32 +51,6 @@ PIN = { } -@pytest.fixture(scope="session") -def firefox_binary(): - """Locate the patched Firefox binary. Three lookup paths: - 1. ``INVPW_BINARY_PATH`` env var (for dev iteration against a local build) - 2. Cached binary under ``cache_dir_for_version()`` (post-fetch) - 3. Skip cleanly (no implicit network download).""" - import os - env_path = os.environ.get("INVPW_BINARY_PATH") - if env_path: - from pathlib import Path - if Path(env_path).exists(): - return env_path - pytest.skip(f"INVPW_BINARY_PATH={env_path!r} does not exist") - if sys.platform not in BINARY_ENTRY_REL: - pytest.skip(f"unsupported platform: {sys.platform}") - from invisible_playwright.download import cache_dir_for_version - entry = cache_dir_for_version() / BINARY_ENTRY_REL[sys.platform] - if not entry.exists(): - pytest.skip( - "patched Firefox not cached; run " - "`python -m invisible_playwright fetch` first, or set " - "INVPW_BINARY_PATH to a local build" - ) - return str(entry) - - @pytest.fixture(scope="module") def page(firefox_binary): """One headless browser shared across the whole module. diff --git a/tests/test_mouse.py b/tests/test_mouse.py index ad0f00e..5ff3478 100644 --- a/tests/test_mouse.py +++ b/tests/test_mouse.py @@ -16,24 +16,11 @@ and covers each patched call site: """ from __future__ import annotations -import sys import urllib.parse import pytest from invisible_playwright import InvisiblePlaywright -from invisible_playwright.constants import BINARY_ENTRY_REL - - -@pytest.fixture(scope="session") -def firefox_binary(): - if sys.platform not in BINARY_ENTRY_REL: - pytest.skip(f"unsupported platform: {sys.platform}") - from invisible_playwright.download import cache_dir_for_version - entry = cache_dir_for_version() / BINARY_ENTRY_REL[sys.platform] - if not entry.exists(): - pytest.skip("patched Firefox binary not cached; run `invisible-playwright fetch`") - return str(entry) def _data_url(html: str) -> str: diff --git a/tests/test_service_worker.py b/tests/test_service_worker.py index 00fd1a6..d077c99 100644 --- a/tests/test_service_worker.py +++ b/tests/test_service_worker.py @@ -22,35 +22,12 @@ For dev iteration: from __future__ import annotations import http.server -import os import socketserver -import sys import threading import pytest from invisible_playwright import InvisiblePlaywright -from invisible_playwright.constants import BINARY_ENTRY_REL - - -@pytest.fixture(scope="session") -def firefox_binary(): - env_path = os.environ.get("INVPW_BINARY_PATH") - if env_path: - from pathlib import Path - if Path(env_path).exists(): - return env_path - pytest.skip(f"INVPW_BINARY_PATH={env_path!r} does not exist") - if sys.platform not in BINARY_ENTRY_REL: - pytest.skip(f"unsupported platform: {sys.platform}") - from invisible_playwright.download import cache_dir_for_version - entry = cache_dir_for_version() / BINARY_ENTRY_REL[sys.platform] - if not entry.exists(): - pytest.skip( - "patched Firefox not cached; run `python -m invisible_playwright fetch` " - "or set INVPW_BINARY_PATH" - ) - return str(entry) # --------------------------------------------------------------------------- diff --git a/tests/test_webrtc_realness.py b/tests/test_webrtc_realness.py index fec01c0..6c126bf 100644 --- a/tests/test_webrtc_realness.py +++ b/tests/test_webrtc_realness.py @@ -342,7 +342,9 @@ _FAKE_EGRESS = "203.0.113.7" # RFC 5737 TEST-NET-3 def _e2e_binary(): - cand = os.environ.get("STEALTHFOX_E2E_BINARY") + # Honor both env vars so the whole e2e suite targets one binary from a single + # setting (INVPW_BINARY_PATH is what conftest's firefox_binary uses). + cand = os.environ.get("STEALTHFOX_E2E_BINARY") or os.environ.get("INVPW_BINARY_PATH") if cand and os.path.exists(cand): return cand built = r"C:\ff\source\obj-x86_64-pc-windows-msvc\dist\bin\firefox.exe"