test: activate the full e2e (browser-driving) suite + add fetch --force

The 138 @pytest.mark.e2e tests were doubly inactive: deselected by addopts AND
skipped without a cached binary — and 3 of the 6 per-file firefox_binary
fixtures silently ignored INVPW_BINARY_PATH, so they'd test whatever was cached
even when you pointed the suite elsewhere (a false-confidence trap).

- Centralize firefox_binary into conftest.py (env INVPW_BINARY_PATH → cache →
  skip); delete the 6 duplicates. Unify test_webrtc_realness onto the same env.
- scripts/run_e2e.py: one command that runs ALL e2e against a chosen binary,
  with reruns so an under-load interaction flake (dblclick/hover pass 3/3 in
  isolation) self-heals while a real break fails every attempt. The webrtc e2e
  fake a TCP-only SOCKS locally, so the suite is offline. This is the MANDATORY
  pre-release browser gate (local — hosted runners are too interaction-flaky).
- Running the suite against firefox-9 surfaced a real gap: `invisible_playwright
  fetch --force` was unrecognized (the subparser took no args) though the e2e
  test + docstring expect it. Implement it: drop the cached version dir, refetch.
- Add pytest-rerunfailures + playwright to the dev extras.

Baseline against firefox-9: 136 passed, 1 skipped (linux_only on win host),
1 was the --force gap now fixed.

pyproject.toml

@@ -30,7 +30,7 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
-dev = ["pytest>=7", "pytest-mock>=3", "responses>=0.24", "build>=1"]
+dev = ["pytest>=7", "pytest-mock>=3", "responses>=0.24", "build>=1", "pytest-rerunfailures>=14", "playwright>=1.40"]
 
 [tool.pytest.ini_options]
 markers = [

scripts/run_e2e.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+"""Run the FULL e2e suite (every test that opens the browser) against a binary.
+
+The 138 ``@pytest.mark.e2e`` tests are excluded from the default `pytest` run
+(`addopts = -m 'not slow and not e2e'`) because they need a real Firefox binary
+and a display, and they skip themselves when no binary is available. That makes
+them easy to forget — and "we can't afford for something to not work". This is
+the gate that runs them all, deliberately, against a chosen binary.
+
+It is the MANDATORY pre-release e2e gate: run it green against the freshly-built
+release binary BEFORE un-drafting a firefox-N (alongside the fppro + WebRTC
+realness gates). It is NOT in the public CI drive-gate — the hosted runners are
+content-process unstable under a heavy headless interaction sequence (see
+70-known-bugs / 60-ci-release-pipeline); this runs locally on reliable hardware.
+
+Flake-resilience: under full-suite load a couple of interaction tests (dblclick,
+hover/mouseenter) can flake even though they pass 3/3 in isolation, so failures
+are reran up to twice on the known transient signatures. A genuinely broken
+binary fails all attempts. The webrtc e2e fake a TCP-only SOCKS locally (no
+proxy/secrets), so the whole suite is offline.
+
+Usage:
+    python scripts/run_e2e.py <firefox-binary>
+    python scripts/run_e2e.py            # uses $INVPW_BINARY_PATH
+"""
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+_RERUN_SIGNATURES = "Timeout|context was destroyed|was detached|not visible|because of a navigation|TargetClosed"
+
+
+def main() -> int:
+    binary = sys.argv[1] if len(sys.argv) > 1 else os.environ.get("INVPW_BINARY_PATH")
+    if not binary:
+        print("usage: run_e2e.py <firefox-binary>  (or set INVPW_BINARY_PATH)", file=sys.stderr)
+        return 2
+    if not Path(binary).exists():
+        print(f"ERROR: binary not found: {binary}", file=sys.stderr)
+        return 2
+
+    env = dict(os.environ)
+    # One setting drives the whole suite: conftest's firefox_binary fixture and
+    # the webrtc e2e both resolve from these.
+    env["INVPW_BINARY_PATH"] = binary
+    env["STEALTHFOX_E2E_BINARY"] = binary
+
+    repo = Path(__file__).resolve().parent.parent
+    cmd = [
+        sys.executable, "-m", "pytest",
+        "-m", "e2e",
+        "-o", "addopts=",            # override the default 'not e2e' deselection
+        "--reruns", "2", "--reruns-delay", "1",
+        "--only-rerun", _RERUN_SIGNATURES,
+        "-p", "no:cacheprovider",
+        "-q", "--tb=short",
+    ] + sys.argv[2:]
+    print(f"[run_e2e] binary={binary}")
+    print(f"[run_e2e] {' '.join(cmd)}")
+    return subprocess.run(cmd, cwd=repo, env=env).returncode
+
+
+if __name__ == "__main__":
+    sys.exit(main())

src/invisible_playwright/cli.py

@@ -10,7 +10,15 @@ from .constants import BINARY_VERSION, FIREFOX_UPSTREAM_VERSION
 from .download import cache_root, ensure_binary
 
 
-def _cmd_fetch(_args: argparse.Namespace) -> int:
+def _cmd_fetch(args: argparse.Namespace) -> int:
+    # --force: re-download even if already cached (drop the cached version dir,
+    # then let ensure_binary fetch it fresh). Useful to recover a corrupted cache
+    # or re-pull after a re-published release.
+    if getattr(args, "force", False):
+        from .download import cache_dir_for_version
+        d = cache_dir_for_version()
+        if d.exists():
+            shutil.rmtree(d, ignore_errors=True)
     path = ensure_binary()
     print(path)
     return 0
@@ -52,7 +60,9 @@ def build_parser() -> argparse.ArgumentParser:
     )
     sub = p.add_subparsers(dest="cmd")
 
-    sub.add_parser("fetch", help="download the patched Firefox binary")
+    fetch_p = sub.add_parser("fetch", help="download the patched Firefox binary")
+    fetch_p.add_argument("--force", action="store_true",
+                         help="re-download even if already cached")
     sub.add_parser("path", help="print the absolute path to the cached binary")
     sub.add_parser("version", help="print wrapper and binary versions")
     sub.add_parser("clear-cache", help="remove all cached binaries")

tests/conftest.py

@@ -1,8 +1,12 @@
+import os
 import random
+import sys
+from pathlib import Path
 
 import pytest
 
 from invisible_playwright._fpforge import generate_profile
+from invisible_playwright.constants import BINARY_ENTRY_REL
 
 
 @pytest.fixture
@@ -15,3 +19,36 @@ def deterministic_rng():
 def sample_profile():
     """A Profile generated from seed=42 for reuse across tests."""
     return generate_profile(seed=42)
+
+
+@pytest.fixture(scope="session")
+def firefox_binary():
+    """Locate the patched Firefox binary for E2E tests, or skip cleanly.
+
+    Single source of truth for every E2E test (previously each test file had its
+    own copy — and three of them silently ignored INVPW_BINARY_PATH, so they kept
+    testing whatever was in the cache even when you pointed the suite at a
+    specific build: a false-confidence trap). Lookup order:
+
+      1. ``INVPW_BINARY_PATH`` env var — point the whole suite at a local build
+         or a freshly-extracted release (this is how the full-suite gate runs).
+      2. Cached binary under ``cache_dir_for_version()`` (post ``fetch``).
+      3. Skip — we never trigger an implicit multi-hundred-MB network download
+         inside a test run.
+    """
+    env_path = os.environ.get("INVPW_BINARY_PATH")
+    if env_path:
+        if Path(env_path).exists():
+            return env_path
+        pytest.skip(f"INVPW_BINARY_PATH={env_path!r} does not exist")
+
+    if sys.platform not in BINARY_ENTRY_REL:
+        pytest.skip(f"unsupported platform: {sys.platform}")
+    from invisible_playwright.download import cache_dir_for_version
+    entry = cache_dir_for_version() / BINARY_ENTRY_REL[sys.platform]
+    if not entry.exists():
+        pytest.skip(
+            "patched Firefox binary not cached and INVPW_BINARY_PATH unset; "
+            "set INVPW_BINARY_PATH=<firefox binary> or run `invisible-playwright fetch`"
+        )
+    return str(entry)

tests/test_cross_origin_iframe.py

@@ -31,7 +31,6 @@ random free ports.
 from __future__ import annotations
 
 import socket
-import sys
 import threading
 from http.server import BaseHTTPRequestHandler, HTTPServer
 
@@ -165,22 +164,6 @@ def cross_origin_harness():
         sb.shutdown()
 
 
-@pytest.fixture(scope="session")
-def firefox_binary():
-    """Locate the cached patched Firefox binary or skip."""
-    from invisible_playwright.constants import BINARY_ENTRY_REL
-    if sys.platform not in BINARY_ENTRY_REL:
-        pytest.skip(f"unsupported platform: {sys.platform}")
-    from invisible_playwright.download import cache_dir_for_version
-    entry = cache_dir_for_version() / BINARY_ENTRY_REL[sys.platform]
-    if not entry.exists():
-        pytest.skip(
-            "patched Firefox binary not cached; run `invisible-playwright fetch` "
-            "to enable E2E tests"
-        )
-    return str(entry)
-
-
 @pytest.mark.e2e
 def test_cross_origin_iframe_url_appears_in_page_frames(firefox_binary, cross_origin_harness):
     """``page.frames`` must list the cross-origin iframe with its real URL.

tests/test_e2e.py

@@ -8,33 +8,9 @@ handling) do not need a binary and always run.
 """
 from __future__ import annotations
 
-import sys
-
 import pytest
 
 from invisible_playwright import InvisiblePlaywright
-from invisible_playwright.constants import BINARY_ENTRY_REL
-
-
-@pytest.fixture(scope="session")
-def firefox_binary():
-    """Locate the patched Firefox binary or skip the calling test.
-
-    We do NOT trigger a network download here: ``ensure_binary`` would
-    pull a multi-hundred-megabyte archive from a private release,
-    which is not appropriate inside a unit/E2E test run. Instead we
-    look for an already-cached binary; if missing we skip.
-    """
-    if sys.platform not in BINARY_ENTRY_REL:
-        pytest.skip(f"unsupported platform: {sys.platform}")
-    from invisible_playwright.download import cache_dir_for_version
-    entry = cache_dir_for_version() / BINARY_ENTRY_REL[sys.platform]
-    if not entry.exists():
-        pytest.skip(
-            "patched Firefox binary not cached; run `invisible-playwright fetch` "
-            "to enable E2E tests"
-        )
-    return str(entry)
 
 
 # ────────────────────────────────────────────────────────────────────

tests/test_fingerprint_consistency.py

@@ -25,12 +25,9 @@ Run only this file:
 """
 from __future__ import annotations
 
-import sys
-
 import pytest
 
 from invisible_playwright import InvisiblePlaywright
-from invisible_playwright.constants import BINARY_ENTRY_REL
 
 
 PIN = {
@@ -45,29 +42,6 @@ PIN = {
 }
 
 
-@pytest.fixture(scope="session")
-def firefox_binary():
-    """See test_fingerprint_surface.firefox_binary for the lookup chain."""
-    import os
-    env_path = os.environ.get("INVPW_BINARY_PATH")
-    if env_path:
-        from pathlib import Path
-        if Path(env_path).exists():
-            return env_path
-        pytest.skip(f"INVPW_BINARY_PATH={env_path!r} does not exist")
-    if sys.platform not in BINARY_ENTRY_REL:
-        pytest.skip(f"unsupported platform: {sys.platform}")
-    from invisible_playwright.download import cache_dir_for_version
-    entry = cache_dir_for_version() / BINARY_ENTRY_REL[sys.platform]
-    if not entry.exists():
-        pytest.skip(
-            "patched Firefox not cached; run "
-            "`python -m invisible_playwright fetch` first, or set "
-            "INVPW_BINARY_PATH to a local build"
-        )
-    return str(entry)
-
-
 @pytest.fixture(scope="module")
 def page(firefox_binary):
     with InvisiblePlaywright(

tests/test_fingerprint_surface.py

@@ -27,12 +27,10 @@ Run only this file:
 from __future__ import annotations
 
 import re
-import sys
 
 import pytest
 
 from invisible_playwright import InvisiblePlaywright
-from invisible_playwright.constants import BINARY_ENTRY_REL
 
 
 # ────────────────────────────────────────────────────────────────────
@@ -53,32 +51,6 @@ PIN = {
 }
 
 
-@pytest.fixture(scope="session")
-def firefox_binary():
-    """Locate the patched Firefox binary. Three lookup paths:
-      1. ``INVPW_BINARY_PATH`` env var (for dev iteration against a local build)
-      2. Cached binary under ``cache_dir_for_version()`` (post-fetch)
-      3. Skip cleanly (no implicit network download)."""
-    import os
-    env_path = os.environ.get("INVPW_BINARY_PATH")
-    if env_path:
-        from pathlib import Path
-        if Path(env_path).exists():
-            return env_path
-        pytest.skip(f"INVPW_BINARY_PATH={env_path!r} does not exist")
-    if sys.platform not in BINARY_ENTRY_REL:
-        pytest.skip(f"unsupported platform: {sys.platform}")
-    from invisible_playwright.download import cache_dir_for_version
-    entry = cache_dir_for_version() / BINARY_ENTRY_REL[sys.platform]
-    if not entry.exists():
-        pytest.skip(
-            "patched Firefox not cached; run "
-            "`python -m invisible_playwright fetch` first, or set "
-            "INVPW_BINARY_PATH to a local build"
-        )
-    return str(entry)
-
-
 @pytest.fixture(scope="module")
 def page(firefox_binary):
     """One headless browser shared across the whole module.

tests/test_mouse.py

@@ -16,24 +16,11 @@ and covers each patched call site:
 """
 from __future__ import annotations
 
-import sys
 import urllib.parse
 
 import pytest
 
 from invisible_playwright import InvisiblePlaywright
-from invisible_playwright.constants import BINARY_ENTRY_REL
-
-
-@pytest.fixture(scope="session")
-def firefox_binary():
-    if sys.platform not in BINARY_ENTRY_REL:
-        pytest.skip(f"unsupported platform: {sys.platform}")
-    from invisible_playwright.download import cache_dir_for_version
-    entry = cache_dir_for_version() / BINARY_ENTRY_REL[sys.platform]
-    if not entry.exists():
-        pytest.skip("patched Firefox binary not cached; run `invisible-playwright fetch`")
-    return str(entry)
 
 
 def _data_url(html: str) -> str:

tests/test_service_worker.py

@@ -22,35 +22,12 @@ For dev iteration:
 from __future__ import annotations
 
 import http.server
-import os
 import socketserver
-import sys
 import threading
 
 import pytest
 
 from invisible_playwright import InvisiblePlaywright
-from invisible_playwright.constants import BINARY_ENTRY_REL
-
-
-@pytest.fixture(scope="session")
-def firefox_binary():
-    env_path = os.environ.get("INVPW_BINARY_PATH")
-    if env_path:
-        from pathlib import Path
-        if Path(env_path).exists():
-            return env_path
-        pytest.skip(f"INVPW_BINARY_PATH={env_path!r} does not exist")
-    if sys.platform not in BINARY_ENTRY_REL:
-        pytest.skip(f"unsupported platform: {sys.platform}")
-    from invisible_playwright.download import cache_dir_for_version
-    entry = cache_dir_for_version() / BINARY_ENTRY_REL[sys.platform]
-    if not entry.exists():
-        pytest.skip(
-            "patched Firefox not cached; run `python -m invisible_playwright fetch` "
-            "or set INVPW_BINARY_PATH"
-        )
-    return str(entry)
 
 
 # ---------------------------------------------------------------------------

tests/test_webrtc_realness.py

@@ -342,7 +342,9 @@ _FAKE_EGRESS = "203.0.113.7"  # RFC 5737 TEST-NET-3
 
 
 def _e2e_binary():
-    cand = os.environ.get("STEALTHFOX_E2E_BINARY")
+    # Honor both env vars so the whole e2e suite targets one binary from a single
+    # setting (INVPW_BINARY_PATH is what conftest's firefox_binary uses).
+    cand = os.environ.get("STEALTHFOX_E2E_BINARY") or os.environ.get("INVPW_BINARY_PATH")
     if cand and os.path.exists(cand):
         return cand
     built = r"C:\ff\source\obj-x86_64-pc-windows-msvc\dist\bin\firefox.exe"