From 86a04d2d347244b092de2267cd756baa14949e04 Mon Sep 17 00:00:00 2001
From: feder-cr <85809106+feder-cr@users.noreply.github.com>
Date: Tue, 9 Jun 2026 12:24:06 +0200
Subject: [PATCH] ci: drive-test every release binary via Playwright, not just
screenshot
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
The old gate ran firefox --headless --screenshot, which renders fine even
when the juggler automation layer is missing from the package — so a binary
Playwright can't actually drive (firefox-8) passed and shipped broken.
Replace it with a real drive gate: a 5-leg matrix that launches each binary
over the juggler pipe on its native runner, loads a page, and round-trips JS
(also asserts navigator.webdriver stays hidden). Headless and no screenshot,
so it stays GPU-free on the hosted runners and needs no proxy or secrets.
Same logic is reusable standalone via verify-assets.yml to drive-test an
existing release's assets without a rebuild.
---
.github/workflows/release.yml | 105 +++++++++++++++++++++-------
.github/workflows/verify-assets.yml | 98 ++++++++++++++++++++++++++
scripts/ci_drive_gate.py | 49 +++++++++++++
3 files changed, 226 insertions(+), 26 deletions(-)
create mode 100644 .github/workflows/verify-assets.yml
create mode 100644 scripts/ci_drive_gate.py
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index b987ad6..ebf7ef8 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -9,12 +9,17 @@
# strip + sanitize + tar at ROOT, then validate_release.py as a HARD
# in-pipeline gate (the exact battle-tested script from the source repo).
# Win → mach package; zip the CONTENTS of dist/firefox (clean tree, NOT
-# dist/bin) so firefox.exe sits at the zip ROOT. Runtime-gated on a real
-# windows-latest runner (headless screenshot).
+# dist/bin) so firefox.exe sits at the zip ROOT.
# macOS → mach package; ad-hoc codesign the .app; PRESERVE its internal relative
# symlinks (a .app legitimately has them — cp -aL would break it); verify
# every symlink is relative+internal; tar the bundle. --version self-gate.
#
+# DRIVE GATE (the firefox-8 catcher): after build, every binary is DRIVEN by
+# Playwright on its native runner (launch via juggler + real page + JS roundtrip,
+# headless, no screenshot → GPU-free, zero proxy). A juggler-less binary renders
+# a screenshot fine but is undrivable — only an actual drive catches that. The
+# proxy realness gate (fppro/webrtc) stays LOCAL — it needs secrets.
+#
# Trigger: push a tag `firefox-N`, or run manually. Hybrid runners, all free.
# ─────────────────────────────────────────────────────────────────────────────
name: release
@@ -233,37 +238,85 @@ jobs:
if-no-files-found: error
retention-days: 7
- # Windows binary is cross-built on Linux → gate it on a real Windows runner.
- gate-windows:
- name: gate-windows
+ # DRIVE GATE — the firefox-8 catcher. A raw `firefox --screenshot` proves
+ # nothing about automation: a juggler-less binary renders fine and ships
+ # broken (firefox-8 did exactly that). So we DRIVE every binary the way users
+ # will: Playwright launches it over the juggler pipe, loads a real page, and
+ # round-trips JS. A binary missing/broken juggler throws TargetClosedError
+ # here and the release never publishes. Headless, NO screenshot → GPU-free,
+ # so it can't false-fail on the GPU-less hosted runners. Zero proxy / zero
+ # secrets → safe in public CI (the proxy realness gate stays local, by design).
+ # Each leg runs on its NATIVE runner so we test the real artifact, not a cross
+ # surrogate. Playwright is pinned to a version validated against this build's
+ # juggler; bump it in lockstep when the juggler is re-synced from upstream.
+ gate:
+ name: gate-${{ matrix.leg }}
needs: build
- runs-on: windows-latest
- timeout-minutes: 20
+ runs-on: ${{ matrix.runner }}
+ timeout-minutes: 25
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - leg: linux-x86_64
+ runner: ubuntu-24.04
+ kind: linux
+ asset: firefox-150.0.1-stealth-linux-x86_64.tar.gz
+ - leg: linux-arm64
+ runner: ubuntu-24.04-arm
+ kind: linux
+ asset: firefox-150.0.1-stealth-linux-arm64.tar.gz
+ - leg: win-x86_64
+ runner: windows-latest
+ kind: win
+ asset: firefox-150.0.1-stealth-win-x86_64.zip
+ - leg: macos-arm64
+ runner: macos-15
+ kind: mac
+ asset: firefox-150.0.1-stealth-macos-arm64.tar.gz
+ - leg: macos-x86_64
+ runner: macos-15-intel
+ kind: mac
+ asset: firefox-150.0.1-stealth-macos-x86_64.tar.gz
steps:
- - name: Download win asset
+ - name: Checkout wrapper (for scripts/ci_drive_gate.py)
+ uses: actions/checkout@v4
+ with: { fetch-depth: 1 }
+ - name: Download asset
uses: actions/download-artifact@v4
- with: { name: asset-win-x86_64, path: art }
- - name: Extract + structure + headless render gate
- shell: pwsh
+ with:
+ name: asset-${{ matrix.leg }}
+ path: art
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with: { python-version: '3.11' }
+ - name: Install Playwright driver (no bundled browser — we override executable_path)
+ run: python -m pip install --quiet "playwright==1.55.0"
+ - name: Linux system deps for headless firefox
+ if: matrix.kind == 'linux'
+ run: sudo "$(which python)" -m playwright install-deps firefox
+ - name: Extract + locate firefox binary
+ shell: bash
run: |
- $zip = Get-ChildItem art -Filter *.zip | Select-Object -First 1
- Expand-Archive $zip.FullName -DestinationPath ff -Force
- foreach ($f in 'firefox.exe','application.ini','dependentlibs.list') {
- if (-not (Test-Path (Join-Path ff $f))) { throw "missing critical file: $f" }
- }
- $exe = Join-Path (Resolve-Path ff) 'firefox.exe'
- Remove-Item shot.png -ErrorAction SilentlyContinue
- $p = Start-Process -FilePath $exe -ArgumentList '--headless','--no-remote','--profile','prof','--screenshot',"$PWD\shot.png",'https://example.com' -PassThru -NoNewWindow
- if (-not $p.WaitForExit(90000)) { $p.Kill(); throw 'win gate TIMEOUT' }
- Start-Sleep 1
- if (-not (Test-Path shot.png) -or (Get-Item shot.png).Length -lt 3000) { throw 'win gate: no/empty screenshot' }
- Write-Output "win gate OK: firefox.exe runs + renders ($((Get-Item shot.png).Length) bytes)"
- - uses: actions/upload-artifact@v4
- with: { name: gate-win-screenshot, path: shot.png, if-no-files-found: warn, retention-days: 7 }
+ set -e
+ mkdir -p ff
+ A="art/${{ matrix.asset }}"
+ case "${{ matrix.kind }}" in
+ win) python -c "import zipfile; zipfile.ZipFile('$A').extractall('ff')"; EXE="ff/firefox.exe";;
+ linux) tar xzf "$A" -C ff; EXE="ff/firefox";;
+ mac) tar xzf "$A" -C ff; EXE="ff/Firefox.app/Contents/MacOS/firefox";;
+ esac
+ [ -e "$EXE" ] || { echo "ERROR: firefox binary not found at $EXE"; exit 1; }
+ chmod +x "$EXE" 2>/dev/null || true
+ echo "FF_EXE=$EXE" >> "$GITHUB_ENV"
+ echo "located: $EXE"
+ - name: DRIVE GATE — Playwright launch via juggler + real page + JS roundtrip
+ shell: bash
+ run: python scripts/ci_drive_gate.py "$FF_EXE"
publish:
name: publish-draft-release
- needs: [build, gate-windows]
+ needs: [build, gate]
runs-on: ubuntu-24.04
permissions:
contents: write
diff --git a/.github/workflows/verify-assets.yml b/.github/workflows/verify-assets.yml
new file mode 100644
index 0000000..fea5805
--- /dev/null
+++ b/.github/workflows/verify-assets.yml
@@ -0,0 +1,98 @@
+# ─────────────────────────────────────────────────────────────────────────────
+# verify-assets.yml — re-runnable DRIVE GATE for an EXISTING release's assets.
+#
+# release.yml drive-gates every binary it builds. This does the same drive test
+# WITHOUT rebuilding: it downloads a release's already-published assets (works on
+# DRAFT releases too via GITHUB_TOKEN) and drives each one on its native runner.
+#
+# Use it to:
+# • drive-test a release that was built before the in-pipeline gate existed
+# (e.g. firefox-9, built on the old release.yml), or
+# • re-verify any shipped release on demand (regression check).
+#
+# Same single-source-of-truth drive logic as release.yml: scripts/ci_drive_gate.py.
+# Headless, no screenshot → GPU-free. Zero proxy / zero secrets.
+# ─────────────────────────────────────────────────────────────────────────────
+name: verify-assets
+
+on:
+ workflow_dispatch:
+ inputs:
+ release_tag:
+ description: 'release tag whose assets to drive-test (e.g. firefox-9)'
+ required: true
+
+permissions:
+ contents: read
+
+jobs:
+ drive:
+ name: drive-${{ matrix.leg }}
+ runs-on: ${{ matrix.runner }}
+ timeout-minutes: 25
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - leg: linux-x86_64
+ runner: ubuntu-24.04
+ kind: linux
+ asset: firefox-150.0.1-stealth-linux-x86_64.tar.gz
+ - leg: linux-arm64
+ runner: ubuntu-24.04-arm
+ kind: linux
+ asset: firefox-150.0.1-stealth-linux-arm64.tar.gz
+ - leg: win-x86_64
+ runner: windows-latest
+ kind: win
+ asset: firefox-150.0.1-stealth-win-x86_64.zip
+ - leg: macos-arm64
+ runner: macos-15
+ kind: mac
+ asset: firefox-150.0.1-stealth-macos-arm64.tar.gz
+ - leg: macos-x86_64
+ runner: macos-15-intel
+ kind: mac
+ asset: firefox-150.0.1-stealth-macos-x86_64.tar.gz
+ steps:
+ - name: Checkout wrapper (for scripts/ci_drive_gate.py)
+ uses: actions/checkout@v4
+ with: { fetch-depth: 1 }
+ - name: Download the release asset (draft releases included)
+ shell: bash
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ run: |
+ set -e
+ mkdir -p art
+ gh release download "${{ github.event.inputs.release_tag }}" \
+ --repo "${{ github.repository }}" \
+ --pattern "${{ matrix.asset }}" \
+ --dir art
+ ls -la art/
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with: { python-version: '3.11' }
+ - name: Install Playwright driver (no bundled browser — we override executable_path)
+ run: python -m pip install --quiet "playwright==1.55.0"
+ - name: Linux system deps for headless firefox
+ if: matrix.kind == 'linux'
+ run: sudo "$(which python)" -m playwright install-deps firefox
+ - name: Extract + locate firefox binary
+ shell: bash
+ run: |
+ set -e
+ mkdir -p ff
+ A="art/${{ matrix.asset }}"
+ case "${{ matrix.kind }}" in
+ win) python -c "import zipfile; zipfile.ZipFile('$A').extractall('ff')"; EXE="ff/firefox.exe";;
+ linux) tar xzf "$A" -C ff; EXE="ff/firefox";;
+ mac) tar xzf "$A" -C ff; EXE="ff/Firefox.app/Contents/MacOS/firefox";;
+ esac
+ [ -e "$EXE" ] || { echo "ERROR: firefox binary not found at $EXE"; exit 1; }
+ chmod +x "$EXE" 2>/dev/null || true
+ echo "FF_EXE=$EXE" >> "$GITHUB_ENV"
+ echo "located: $EXE"
+ - name: DRIVE GATE — Playwright launch via juggler + real page + JS roundtrip
+ shell: bash
+ run: python scripts/ci_drive_gate.py "$FF_EXE"
diff --git a/scripts/ci_drive_gate.py b/scripts/ci_drive_gate.py
new file mode 100644
index 0000000..e81f95f
--- /dev/null
+++ b/scripts/ci_drive_gate.py
@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+"""CI drive gate — the firefox-8 catcher.
+
+A raw `firefox --screenshot` proves nothing about automation: a juggler-less
+binary renders a screenshot just fine and ships broken (firefox-8 did exactly
+that). This DRIVES the binary the way users will — Playwright launches it over
+the juggler pipe, loads a real page, and round-trips JS. A binary with a
+missing/broken juggler throws TargetClosedError here and the gate fails.
+
+Headless, NO screenshot → GPU-free, so it can't false-fail on GPU-less hosted
+runners. Zero proxy / zero secrets → safe in public CI. (The proxy realness
+gate — fppro/webrtc — stays local, it needs secrets.)
+
+Usage: python ci_drive_gate.py /path/to/firefox[.exe | .app/Contents/MacOS/firefox]
+Exit 0 + "DRIVE GATE OK ..." on success; non-zero with a reason on failure.
+"""
+from __future__ import annotations
+
+import sys
+
+from playwright.sync_api import sync_playwright
+
+
+def main(exe: str) -> int:
+ with sync_playwright() as p:
+ browser = p.firefox.launch(executable_path=exe, headless=True)
+ page = browser.new_page()
+ # data: URL → real HTML parse + DOM + JS, fully offline (no network/proxy).
+ page.goto("data:text/html,dthello-drive
")
+ ua = page.evaluate("navigator.userAgent")
+ webdriver = page.evaluate("navigator.webdriver")
+ text = page.evaluate("() => document.getElementById('x').textContent")
+ browser.close()
+
+ assert "Firefox" in ua, f"unexpected UA (binary not driving correctly): {ua!r}"
+ assert text == "hello-drive", f"DOM/JS roundtrip failed: {text!r}"
+ # Free stealth smoke: the patched build hides navigator.webdriver even when
+ # driven by bare Playwright. A True here is a stealth regression, not a crash.
+ assert not webdriver, f"navigator.webdriver leaked True (stealth regression): {webdriver!r}"
+
+ print(f"DRIVE GATE OK | UA={ua} | webdriver={webdriver} | dom-roundtrip=ok")
+ return 0
+
+
+if __name__ == "__main__":
+ if len(sys.argv) != 2:
+ print("usage: ci_drive_gate.py ", file=sys.stderr)
+ sys.exit(2)
+ sys.exit(main(sys.argv[1]))