mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-05-13 17:52:38 +02:00
chore: add playwright cursor skill
This commit is contained in:
parent
25aad38ca4
commit
d52225c18d
57 changed files with 25244 additions and 0 deletions
468
.cursor/skills/playwright-testing/infrastructure-ci-cd/ci-cd.md
Normal file
468
.cursor/skills/playwright-testing/infrastructure-ci-cd/ci-cd.md
Normal file
|
|
@ -0,0 +1,468 @@
|
|||
# CI/CD Integration
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [GitHub Actions](#github-actions)
|
||||
2. [Docker](#docker)
|
||||
3. [Reporting](#reporting)
|
||||
4. [Sharding](#sharding)
|
||||
5. [Environment Management](#environment-management)
|
||||
6. [Caching](#caching)
|
||||
|
||||
## GitHub Actions
|
||||
|
||||
### Basic Workflow
|
||||
|
||||
```yaml
|
||||
# .github/workflows/playwright.yml
|
||||
name: Playwright Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
jobs:
|
||||
test:
|
||||
timeout-minutes: 60
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
cache: "npm"
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Install Playwright browsers
|
||||
run: npx playwright install --with-deps
|
||||
|
||||
- name: Run Playwright tests
|
||||
run: npx playwright test
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: ${{ !cancelled() }}
|
||||
with:
|
||||
name: playwright-report
|
||||
path: playwright-report/
|
||||
retention-days: 30
|
||||
```
|
||||
|
||||
### With Sharding
|
||||
|
||||
```yaml
|
||||
name: Playwright Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
|
||||
jobs:
|
||||
test:
|
||||
timeout-minutes: 60
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shardIndex: [1, 2, 3, 4]
|
||||
shardTotal: [4]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
cache: "npm"
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Install Playwright browsers
|
||||
run: npx playwright install --with-deps
|
||||
|
||||
- name: Run Playwright tests
|
||||
run: npx playwright test --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
|
||||
|
||||
- name: Upload blob report
|
||||
if: ${{ !cancelled() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: blob-report-${{ matrix.shardIndex }}
|
||||
path: blob-report
|
||||
retention-days: 1
|
||||
|
||||
merge-reports:
|
||||
if: ${{ !cancelled() }}
|
||||
needs: [test]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
cache: "npm"
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Download blob reports
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: all-blob-reports
|
||||
pattern: blob-report-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Merge reports
|
||||
run: npx playwright merge-reports --reporter html ./all-blob-reports
|
||||
|
||||
- name: Upload HTML report
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: html-report
|
||||
path: playwright-report
|
||||
retention-days: 14
|
||||
```
|
||||
|
||||
### With Container
|
||||
|
||||
```yaml
|
||||
jobs:
|
||||
test:
|
||||
timeout-minutes: 60
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
# Use latest or more appropriate playwright version (match package.json)
|
||||
image: mcr.microsoft.com/playwright:v1.40.0-jammy
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
cache: "npm"
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Run tests
|
||||
run: npx playwright test
|
||||
env:
|
||||
HOME: /root
|
||||
```
|
||||
|
||||
## Docker
|
||||
|
||||
### Dockerfile
|
||||
|
||||
```dockerfile
|
||||
FROM mcr.microsoft.com/playwright:v1.40.0-jammy
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY package*.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY . .
|
||||
|
||||
CMD ["npx", "playwright", "test"]
|
||||
```
|
||||
|
||||
### Docker Compose
|
||||
|
||||
```yaml
|
||||
# docker-compose.yml
|
||||
version: "3.8"
|
||||
|
||||
services:
|
||||
playwright:
|
||||
build: .
|
||||
volumes:
|
||||
- ./playwright-report:/app/playwright-report
|
||||
- ./test-results:/app/test-results
|
||||
environment:
|
||||
- CI=true
|
||||
- BASE_URL=http://app:3000
|
||||
depends_on:
|
||||
- app
|
||||
|
||||
app:
|
||||
build: ./app
|
||||
ports:
|
||||
- "3000:3000"
|
||||
```
|
||||
|
||||
### Run with Docker
|
||||
|
||||
```bash
|
||||
# Build and run
|
||||
docker build -t playwright-tests .
|
||||
docker run --rm -v $(pwd)/playwright-report:/app/playwright-report playwright-tests
|
||||
|
||||
# With docker-compose
|
||||
docker-compose run --rm playwright
|
||||
```
|
||||
|
||||
## Reporting
|
||||
|
||||
### Configuration
|
||||
|
||||
```typescript
|
||||
// playwright.config.ts
|
||||
export default defineConfig({
|
||||
reporter: [
|
||||
// Always generate
|
||||
["html", { outputFolder: "playwright-report" }],
|
||||
|
||||
// Console output
|
||||
["list"],
|
||||
|
||||
// CI-friendly
|
||||
["github"], // GitHub Actions annotations
|
||||
|
||||
// JUnit for CI integration
|
||||
["junit", { outputFile: "results.xml" }],
|
||||
|
||||
// JSON for custom processing
|
||||
["json", { outputFile: "results.json" }],
|
||||
|
||||
// Blob for merging shards
|
||||
["blob", { outputDir: "blob-report" }],
|
||||
],
|
||||
});
|
||||
```
|
||||
|
||||
### CI-Specific Reporter
|
||||
|
||||
```typescript
|
||||
export default defineConfig({
|
||||
reporter: process.env.CI
|
||||
? [["github"], ["blob"], ["html"]]
|
||||
: [["list"], ["html"]],
|
||||
});
|
||||
```
|
||||
|
||||
## Sharding
|
||||
|
||||
### Command Line
|
||||
|
||||
```bash
|
||||
# Split into 4 shards, run shard 1
|
||||
npx playwright test --shard=1/4
|
||||
|
||||
# Run shard 2
|
||||
npx playwright test --shard=2/4
|
||||
```
|
||||
|
||||
### Configuration
|
||||
|
||||
```typescript
|
||||
// playwright.config.ts
|
||||
export default defineConfig({
|
||||
// Evenly distribute tests across shards
|
||||
fullyParallel: true,
|
||||
|
||||
// For blob reporter to merge later
|
||||
reporter: process.env.CI ? [["blob"]] : [["html"]],
|
||||
});
|
||||
```
|
||||
|
||||
### Merge Sharded Reports
|
||||
|
||||
```bash
|
||||
# After all shards complete, merge blob reports
|
||||
npx playwright merge-reports --reporter html ./all-blob-reports
|
||||
```
|
||||
|
||||
## Environment Management
|
||||
|
||||
### Environment Variables
|
||||
|
||||
```typescript
|
||||
// playwright.config.ts
|
||||
import { defineConfig } from "@playwright/test";
|
||||
import dotenv from "dotenv";
|
||||
|
||||
// Load env file based on environment
|
||||
dotenv.config({ path: `.env.${process.env.NODE_ENV || "development"}` });
|
||||
|
||||
export default defineConfig({
|
||||
use: {
|
||||
baseURL: process.env.BASE_URL || "http://localhost:3000",
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
### Multiple Environments
|
||||
|
||||
```yaml
|
||||
# .github/workflows/playwright.yml
|
||||
jobs:
|
||||
test:
|
||||
strategy:
|
||||
matrix:
|
||||
environment: [staging, production]
|
||||
steps:
|
||||
- name: Run tests
|
||||
run: npx playwright test
|
||||
env:
|
||||
BASE_URL: ${{ matrix.environment == 'staging' && 'https://staging.example.com' || 'https://example.com' }}
|
||||
TEST_USER: ${{ secrets[format('TEST_USER_{0}', matrix.environment)] }}
|
||||
```
|
||||
|
||||
### Secrets Management
|
||||
|
||||
```yaml
|
||||
# GitHub Actions secrets
|
||||
- name: Run tests
|
||||
run: npx playwright test
|
||||
env:
|
||||
TEST_EMAIL: ${{ secrets.TEST_EMAIL }}
|
||||
TEST_PASSWORD: ${{ secrets.TEST_PASSWORD }}
|
||||
```
|
||||
|
||||
```typescript
|
||||
// tests use environment variables
|
||||
test("login", async ({ page }) => {
|
||||
await page.getByLabel("Email").fill(process.env.TEST_EMAIL!);
|
||||
await page.getByLabel("Password").fill(process.env.TEST_PASSWORD!);
|
||||
});
|
||||
```
|
||||
|
||||
## Caching
|
||||
|
||||
### Cache Playwright Browsers
|
||||
|
||||
```yaml
|
||||
- name: Cache Playwright browsers
|
||||
uses: actions/cache@v4
|
||||
id: playwright-cache
|
||||
with:
|
||||
path: ~/.cache/ms-playwright
|
||||
key: playwright-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
|
||||
|
||||
- name: Install Playwright browsers
|
||||
if: steps.playwright-cache.outputs.cache-hit != 'true'
|
||||
run: npx playwright install --with-deps
|
||||
|
||||
- name: Install system deps only
|
||||
if: steps.playwright-cache.outputs.cache-hit == 'true'
|
||||
run: npx playwright install-deps
|
||||
```
|
||||
|
||||
### Cache Node Modules
|
||||
|
||||
```yaml
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
cache: "npm"
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
```
|
||||
|
||||
## Tag-Based Test Filtering
|
||||
|
||||
### Run Specific Tags in CI
|
||||
|
||||
```yaml
|
||||
# Run smoke tests on PR
|
||||
- name: Run smoke tests
|
||||
run: npx playwright test --grep @smoke
|
||||
|
||||
# Run full regression nightly
|
||||
- name: Run regression
|
||||
run: npx playwright test --grep @regression
|
||||
|
||||
# Exclude flaky tests
|
||||
- name: Run stable tests
|
||||
run: npx playwright test --grep-invert @flaky
|
||||
```
|
||||
|
||||
### PR vs Nightly Strategy
|
||||
|
||||
```yaml
|
||||
# .github/workflows/pr.yml - Fast feedback
|
||||
- name: Run critical tests
|
||||
run: npx playwright test --grep "@smoke|@critical"
|
||||
|
||||
# .github/workflows/nightly.yml - Full coverage
|
||||
- name: Run all tests
|
||||
run: npx playwright test --grep-invert @flaky
|
||||
```
|
||||
|
||||
### Tag Filtering in Config
|
||||
|
||||
```typescript
|
||||
// playwright.config.ts
|
||||
export default defineConfig({
|
||||
grep: process.env.CI ? /@smoke|@critical/ : undefined,
|
||||
grepInvert: process.env.CI ? /@flaky/ : undefined,
|
||||
});
|
||||
```
|
||||
|
||||
### Project-Based Tag Filtering
|
||||
|
||||
```typescript
|
||||
// playwright.config.ts
|
||||
export default defineConfig({
|
||||
projects: [
|
||||
{
|
||||
name: "smoke",
|
||||
grep: /@smoke/,
|
||||
},
|
||||
{
|
||||
name: "regression",
|
||||
grepInvert: /@smoke/,
|
||||
},
|
||||
],
|
||||
});
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
|
||||
| Practice | Benefit |
|
||||
| ----------------------------- | ------------------------- |
|
||||
| Use `npm ci` | Deterministic installs |
|
||||
| Run headless in CI | Faster, no display needed |
|
||||
| Set retries in CI only | Handle flakiness |
|
||||
| Upload artifacts on failure | Debug failures |
|
||||
| Use sharding for large suites | Faster execution |
|
||||
| Cache browsers | Faster setup |
|
||||
| Use blob reporter for shards | Merge reports correctly |
|
||||
| Use tags for PR vs nightly | Fast feedback + coverage |
|
||||
| Exclude @flaky in CI | Stable pipeline |
|
||||
|
||||
## CI Configuration Reference
|
||||
|
||||
```typescript
|
||||
// playwright.config.ts - CI optimized
|
||||
export default defineConfig({
|
||||
testDir: "./tests",
|
||||
fullyParallel: true,
|
||||
forbidOnly: !!process.env.CI,
|
||||
retries: process.env.CI ? 2 : 0,
|
||||
workers: process.env.CI ? 1 : undefined,
|
||||
reporter: process.env.CI
|
||||
? [["github"], ["blob"], ["html"]]
|
||||
: [["list"], ["html"]],
|
||||
use: {
|
||||
baseURL: process.env.BASE_URL || "http://localhost:3000",
|
||||
trace: "on-first-retry",
|
||||
screenshot: "only-on-failure",
|
||||
video: "on-first-retry",
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
## Related References
|
||||
|
||||
- **Test tags**: See [test-tags.md](../core/test-tags.md) for tagging and filtering patterns
|
||||
- **Performance optimization**: See [performance.md](performance.md) for sharding and parallelization
|
||||
- **Debugging CI failures**: See [debugging.md](../debugging/debugging.md) for troubleshooting
|
||||
- **Test reporting**: See [debugging.md](../debugging/debugging.md) for trace viewer usage
|
||||
283
.cursor/skills/playwright-testing/infrastructure-ci-cd/docker.md
Normal file
283
.cursor/skills/playwright-testing/infrastructure-ci-cd/docker.md
Normal file
|
|
@ -0,0 +1,283 @@
|
|||
# Container-Based Testing
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [Patterns](#patterns)
|
||||
2. [Decision Guide](#decision-guide)
|
||||
3. [Anti-Patterns](#anti-patterns)
|
||||
4. [Troubleshooting](#troubleshooting)
|
||||
|
||||
> **When to use**: Running tests in containers for reproducible environments, CI pipelines, or consistent browser versions across team machines.
|
||||
|
||||
## Patterns
|
||||
|
||||
### Official Image Usage
|
||||
|
||||
Run tests without building a custom image:
|
||||
|
||||
```bash
|
||||
docker run --rm \
|
||||
-v $(pwd):/app \
|
||||
-w /app \
|
||||
-e CI=true \
|
||||
-e BASE_URL=http://host.docker.internal:3000 \
|
||||
mcr.microsoft.com/playwright:v1.48.0-noble \
|
||||
bash -c "npm ci && npx playwright test"
|
||||
```
|
||||
|
||||
Extract reports with bind mounts:
|
||||
|
||||
```bash
|
||||
docker run --rm \
|
||||
-v $(pwd):/app \
|
||||
-v $(pwd)/playwright-report:/app/playwright-report \
|
||||
-v $(pwd)/test-results:/app/test-results \
|
||||
-w /app \
|
||||
mcr.microsoft.com/playwright:v1.48.0-noble \
|
||||
bash -c "npm ci && npx playwright test"
|
||||
```
|
||||
|
||||
### Custom Dockerfile
|
||||
|
||||
Build a custom image when you need additional dependencies or pre-installed packages:
|
||||
|
||||
```dockerfile
|
||||
FROM mcr.microsoft.com/playwright:v1.48.0-noble
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY package.json package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY . .
|
||||
|
||||
CMD ["npx", "playwright", "test"]
|
||||
```
|
||||
|
||||
Chromium-only slim image:
|
||||
|
||||
```dockerfile
|
||||
FROM node:latest-slim
|
||||
|
||||
RUN npx playwright install --with-deps chromium
|
||||
|
||||
WORKDIR /app
|
||||
COPY package.json package-lock.json ./
|
||||
RUN npm ci
|
||||
COPY . .
|
||||
|
||||
CMD ["npx", "playwright", "test", "--project=chromium"]
|
||||
```
|
||||
|
||||
### Docker Compose Stack
|
||||
|
||||
Full application stack with database, cache, and test runner:
|
||||
|
||||
```yaml
|
||||
services:
|
||||
app:
|
||||
build: .
|
||||
ports:
|
||||
- "3000:3000"
|
||||
environment:
|
||||
- NODE_ENV=test
|
||||
- DATABASE_URL=postgresql://postgres:postgres@db:5432/test
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
|
||||
db:
|
||||
image: postgres:latest-alpine
|
||||
environment:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: test
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U postgres"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
tmpfs:
|
||||
- /var/lib/postgresql/data
|
||||
|
||||
e2e:
|
||||
image: mcr.microsoft.com/playwright:v1.48.0-noble
|
||||
working_dir: /app
|
||||
volumes:
|
||||
- .:/app
|
||||
- /app/node_modules
|
||||
environment:
|
||||
- CI=true
|
||||
- BASE_URL=http://app:3000
|
||||
depends_on:
|
||||
- app
|
||||
command: bash -c "npm ci && npx playwright test"
|
||||
profiles:
|
||||
- test
|
||||
```
|
||||
|
||||
Run commands:
|
||||
|
||||
```bash
|
||||
docker compose --profile test up --abort-on-container-exit --exit-code-from e2e
|
||||
|
||||
docker compose --profile test down -v
|
||||
```
|
||||
|
||||
### CI Container Jobs
|
||||
|
||||
**GitHub Actions:**
|
||||
|
||||
```yaml
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: mcr.microsoft.com/playwright:v1.48.0-noble
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- run: npm ci
|
||||
- run: npx playwright test
|
||||
env:
|
||||
HOME: /root
|
||||
```
|
||||
|
||||
**GitLab CI:**
|
||||
|
||||
```yaml
|
||||
test:
|
||||
image: mcr.microsoft.com/playwright:v1.48.0-noble
|
||||
script:
|
||||
- npm ci
|
||||
- npx playwright test
|
||||
```
|
||||
|
||||
**Jenkins:**
|
||||
|
||||
```groovy
|
||||
pipeline {
|
||||
agent {
|
||||
docker {
|
||||
image 'mcr.microsoft.com/playwright:v1.48.0-noble'
|
||||
args '-u root'
|
||||
}
|
||||
}
|
||||
stages {
|
||||
stage('Test') {
|
||||
steps {
|
||||
sh 'npm ci'
|
||||
sh 'npx playwright test'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Dev Container Setup
|
||||
|
||||
VS Code Dev Container or GitHub Codespaces configuration:
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "Playwright Dev",
|
||||
"image": "mcr.microsoft.com/playwright:v1.48.0-noble",
|
||||
"features": {
|
||||
"ghcr.io/devcontainers/features/node:latest": {
|
||||
"version": "20"
|
||||
}
|
||||
},
|
||||
"postCreateCommand": "npm ci",
|
||||
"customizations": {
|
||||
"vscode": {
|
||||
"extensions": ["ms-playwright.playwright"]
|
||||
}
|
||||
},
|
||||
"forwardPorts": [3000, 9323],
|
||||
"remoteUser": "root"
|
||||
}
|
||||
```
|
||||
|
||||
## Decision Guide
|
||||
|
||||
| Scenario | Approach |
|
||||
|---|---|
|
||||
| Simple CI pipeline | Official image as CI container |
|
||||
| Tests need database + cache | Docker Compose with app, db, e2e services |
|
||||
| Team needs identical environments | Dev Container or custom Dockerfile |
|
||||
| Only testing Chromium | Slim image with `install --with-deps chromium` |
|
||||
| Cross-browser testing | Official image (all browsers pre-installed) |
|
||||
| Local development | Run directly on host for faster iteration |
|
||||
|
||||
## Anti-Patterns
|
||||
|
||||
| Anti-Pattern | Problem | Solution |
|
||||
|---|---|---|
|
||||
| Installing browsers at runtime | Wastes 60-90 seconds per run | Use official image or bake browsers into custom image |
|
||||
| Running as non-root without sandbox config | Chromium sandbox permission errors | Run as root or disable sandbox |
|
||||
| Bind-mounting `node_modules` from host | Platform-specific binary crashes | Use anonymous volume: `-v /app/node_modules` |
|
||||
| No health checks on dependent services | Tests start before database ready | Add `healthcheck` with `depends_on: condition: service_healthy` |
|
||||
| Building application inside Playwright container | Large image, slow builds | Separate app and e2e containers |
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### "browserType.launch: Executable doesn't exist"
|
||||
|
||||
Playwright version mismatch with Docker image. Ensure `@playwright/test` version matches image tag:
|
||||
|
||||
```bash
|
||||
npm ls @playwright/test
|
||||
docker pull mcr.microsoft.com/playwright:v<matching-version>-noble
|
||||
```
|
||||
|
||||
### "net::ERR_CONNECTION_REFUSED" in docker-compose
|
||||
|
||||
Tests trying to reach `localhost` instead of service name. Configure `baseURL`:
|
||||
|
||||
```typescript
|
||||
import { defineConfig } from '@playwright/test';
|
||||
|
||||
export default defineConfig({
|
||||
use: {
|
||||
baseURL: process.env.BASE_URL || 'http://localhost:3000',
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
```yaml
|
||||
e2e:
|
||||
environment:
|
||||
- BASE_URL=http://app:3000
|
||||
```
|
||||
|
||||
### Permission denied on mounted volumes
|
||||
|
||||
Match user IDs or run as root:
|
||||
|
||||
```bash
|
||||
docker run --rm -u $(id -u):$(id -g) \
|
||||
-v $(pwd):/app -w /app \
|
||||
mcr.microsoft.com/playwright:v1.48.0-noble \
|
||||
npx playwright test
|
||||
```
|
||||
|
||||
### Slow container tests on macOS/Windows
|
||||
|
||||
Docker Desktop I/O overhead. Copy files instead of mounting:
|
||||
|
||||
```dockerfile
|
||||
FROM mcr.microsoft.com/playwright:v1.48.0-noble
|
||||
WORKDIR /app
|
||||
COPY . .
|
||||
RUN npm ci
|
||||
CMD ["npx", "playwright", "test"]
|
||||
```
|
||||
|
||||
Or use delegated mount:
|
||||
|
||||
```bash
|
||||
docker run --rm \
|
||||
-v $(pwd):/app:delegated \
|
||||
-w /app \
|
||||
mcr.microsoft.com/playwright:v1.48.0-noble \
|
||||
bash -c "npm ci && npx playwright test"
|
||||
```
|
||||
|
|
@ -0,0 +1,546 @@
|
|||
# GitHub Actions for Playwright
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [CLI Commands](#cli-commands)
|
||||
2. [Workflow Patterns](#workflow-patterns)
|
||||
3. [Scenario Guide](#scenario-guide)
|
||||
4. [Common Mistakes](#common-mistakes)
|
||||
5. [Troubleshooting](#troubleshooting)
|
||||
6. [Related](#related)
|
||||
|
||||
> **When to use**: Automating Playwright tests on pull requests, main branch merges, or scheduled runs.
|
||||
|
||||
## CLI Commands
|
||||
|
||||
```bash
|
||||
npx playwright install --with-deps # browsers + OS dependencies
|
||||
npx playwright test --shard=1/4 # run shard 1 of 4
|
||||
npx playwright test --reporter=github # PR annotations
|
||||
npx playwright merge-reports ./blob-report # combine shard reports
|
||||
```
|
||||
|
||||
## Workflow Patterns
|
||||
|
||||
### Basic Workflow
|
||||
|
||||
**Use when**: Starting a new project or running a small test suite.
|
||||
|
||||
```yaml
|
||||
# .github/workflows/e2e.yml
|
||||
name: E2E Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
concurrency:
|
||||
group: e2e-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
CI: true
|
||||
|
||||
jobs:
|
||||
test:
|
||||
timeout-minutes: 30
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- run: npm ci
|
||||
|
||||
- name: Cache browsers
|
||||
id: browser-cache
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/ms-playwright
|
||||
key: pw-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
|
||||
|
||||
- name: Install browsers
|
||||
if: steps.browser-cache.outputs.cache-hit != 'true'
|
||||
run: npx playwright install --with-deps
|
||||
|
||||
- name: Install OS dependencies
|
||||
if: steps.browser-cache.outputs.cache-hit == 'true'
|
||||
run: npx playwright install-deps
|
||||
|
||||
- run: npx playwright test
|
||||
|
||||
- name: Upload report
|
||||
uses: actions/upload-artifact@v4
|
||||
if: ${{ !cancelled() }}
|
||||
with:
|
||||
name: test-report
|
||||
path: playwright-report/
|
||||
retention-days: 14
|
||||
|
||||
- name: Upload traces
|
||||
uses: actions/upload-artifact@v4
|
||||
if: failure()
|
||||
with:
|
||||
name: traces
|
||||
path: test-results/
|
||||
retention-days: 7
|
||||
```
|
||||
|
||||
### Sharded Execution
|
||||
|
||||
**Use when**: Test suite exceeds 10 minutes. Sharding cuts wall-clock time significantly.
|
||||
**Avoid when**: Suite runs under 5 minutes—sharding overhead negates benefits.
|
||||
|
||||
```yaml
|
||||
# .github/workflows/e2e-sharded.yml
|
||||
name: E2E Tests (Sharded)
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
concurrency:
|
||||
group: e2e-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
CI: true
|
||||
|
||||
jobs:
|
||||
test:
|
||||
timeout-minutes: 20
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shard: [1/4, 2/4, 3/4, 4/4]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- run: npm ci
|
||||
|
||||
- name: Cache browsers
|
||||
id: browser-cache
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/ms-playwright
|
||||
key: pw-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
|
||||
|
||||
- name: Install browsers
|
||||
if: steps.browser-cache.outputs.cache-hit != 'true'
|
||||
run: npx playwright install --with-deps
|
||||
|
||||
- name: Install OS dependencies
|
||||
if: steps.browser-cache.outputs.cache-hit == 'true'
|
||||
run: npx playwright install-deps
|
||||
|
||||
- name: Run tests (shard ${{ matrix.shard }})
|
||||
run: npx playwright test --shard=${{ matrix.shard }}
|
||||
|
||||
- name: Upload blob report
|
||||
uses: actions/upload-artifact@v4
|
||||
if: ${{ !cancelled() }}
|
||||
with:
|
||||
name: blob-${{ strategy.job-index }}
|
||||
path: blob-report/
|
||||
retention-days: 1
|
||||
|
||||
merge:
|
||||
if: ${{ !cancelled() }}
|
||||
needs: test
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- run: npm ci
|
||||
|
||||
- name: Download blob reports
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: all-blobs
|
||||
pattern: blob-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Merge reports
|
||||
run: npx playwright merge-reports --reporter=html ./all-blobs
|
||||
|
||||
- name: Upload merged report
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: test-report
|
||||
path: playwright-report/
|
||||
retention-days: 14
|
||||
```
|
||||
|
||||
**Config for sharding**—enable blob reporter:
|
||||
|
||||
```typescript
|
||||
// playwright.config.ts
|
||||
import { defineConfig } from '@playwright/test';
|
||||
|
||||
export default defineConfig({
|
||||
reporter: process.env.CI
|
||||
? [['blob'], ['github']]
|
||||
: [['html', { open: 'on-failure' }]],
|
||||
});
|
||||
```
|
||||
|
||||
### Container-Based Execution
|
||||
|
||||
**Use when**: Reproducible environment matching local Docker setup, or runner OS dependencies cause issues.
|
||||
**Avoid when**: Standard `ubuntu-latest` with `--with-deps` works fine.
|
||||
|
||||
```yaml
|
||||
# .github/workflows/e2e-container.yml
|
||||
name: E2E Tests (Container)
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
jobs:
|
||||
test:
|
||||
timeout-minutes: 30
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: mcr.microsoft.com/playwright:v1.48.0-noble
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- run: npm ci
|
||||
|
||||
- name: Run tests
|
||||
run: npx playwright test
|
||||
env:
|
||||
HOME: /root
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: ${{ !cancelled() }}
|
||||
with:
|
||||
name: test-report
|
||||
path: playwright-report/
|
||||
retention-days: 14
|
||||
```
|
||||
|
||||
### Environment Secrets
|
||||
|
||||
**Use when**: Tests target staging/production with credentials.
|
||||
**Avoid when**: Tests only run against local dev server.
|
||||
|
||||
```yaml
|
||||
# .github/workflows/e2e-staging.yml
|
||||
name: Staging Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
test:
|
||||
timeout-minutes: 30
|
||||
runs-on: ubuntu-latest
|
||||
environment: staging
|
||||
|
||||
env:
|
||||
CI: true
|
||||
BASE_URL: ${{ vars.STAGING_URL }}
|
||||
TEST_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
|
||||
API_TOKEN: ${{ secrets.API_TOKEN }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- run: npm ci
|
||||
|
||||
- name: Cache browsers
|
||||
id: browser-cache
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/ms-playwright
|
||||
key: pw-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
|
||||
|
||||
- name: Install browsers
|
||||
if: steps.browser-cache.outputs.cache-hit != 'true'
|
||||
run: npx playwright install --with-deps
|
||||
|
||||
- name: Install OS dependencies
|
||||
if: steps.browser-cache.outputs.cache-hit == 'true'
|
||||
run: npx playwright install-deps
|
||||
|
||||
- name: Run smoke tests
|
||||
run: npx playwright test --grep @smoke
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: ${{ !cancelled() }}
|
||||
with:
|
||||
name: staging-report
|
||||
path: playwright-report/
|
||||
retention-days: 14
|
||||
```
|
||||
|
||||
### Scheduled Runs
|
||||
|
||||
**Use when**: Full regression suite is too slow for every PR—run nightly instead.
|
||||
**Avoid when**: Suite runs under 15 minutes and can run on every PR.
|
||||
|
||||
```yaml
|
||||
# .github/workflows/nightly.yml
|
||||
name: Nightly Regression
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 3 * * 1-5'
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
test:
|
||||
timeout-minutes: 60
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
env:
|
||||
CI: true
|
||||
BASE_URL: ${{ vars.STAGING_URL }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- run: npm ci
|
||||
|
||||
- name: Install browsers
|
||||
run: npx playwright install --with-deps
|
||||
|
||||
- name: Run full regression
|
||||
run: npx playwright test --grep @regression
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: ${{ !cancelled() }}
|
||||
with:
|
||||
name: nightly-${{ github.run_number }}
|
||||
path: playwright-report/
|
||||
retention-days: 30
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
uses: slackapi/slack-github-action@latest
|
||||
with:
|
||||
payload: |
|
||||
{
|
||||
"text": "Nightly regression failed: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
}
|
||||
env:
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
```
|
||||
|
||||
### Reusable Workflow
|
||||
|
||||
**Use when**: Multiple repositories share the same Playwright setup.
|
||||
**Avoid when**: Single repo with one workflow.
|
||||
|
||||
```yaml
|
||||
# .github/workflows/pw-reusable.yml
|
||||
name: Playwright Reusable
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
node-version:
|
||||
type: string
|
||||
default: 'lts/*'
|
||||
test-command:
|
||||
type: string
|
||||
default: 'npx playwright test'
|
||||
secrets:
|
||||
BASE_URL:
|
||||
required: false
|
||||
TEST_PASSWORD:
|
||||
required: false
|
||||
|
||||
jobs:
|
||||
test:
|
||||
timeout-minutes: 30
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
env:
|
||||
CI: true
|
||||
BASE_URL: ${{ secrets.BASE_URL }}
|
||||
TEST_PASSWORD: ${{ secrets.TEST_PASSWORD }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: ${{ inputs.node-version }}
|
||||
cache: npm
|
||||
|
||||
- run: npm ci
|
||||
|
||||
- name: Cache browsers
|
||||
id: browser-cache
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/ms-playwright
|
||||
key: pw-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
|
||||
|
||||
- name: Install browsers
|
||||
if: steps.browser-cache.outputs.cache-hit != 'true'
|
||||
run: npx playwright install --with-deps
|
||||
|
||||
- name: Install OS dependencies
|
||||
if: steps.browser-cache.outputs.cache-hit == 'true'
|
||||
run: npx playwright install-deps
|
||||
|
||||
- name: Run tests
|
||||
run: ${{ inputs.test-command }}
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: ${{ !cancelled() }}
|
||||
with:
|
||||
name: test-report
|
||||
path: playwright-report/
|
||||
retention-days: 14
|
||||
```
|
||||
|
||||
**Calling the reusable workflow:**
|
||||
|
||||
```yaml
|
||||
# .github/workflows/ci.yml
|
||||
name: CI
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
jobs:
|
||||
e2e:
|
||||
uses: ./.github/workflows/pw-reusable.yml
|
||||
with:
|
||||
node-version: 'lts/*'
|
||||
secrets:
|
||||
BASE_URL: ${{ secrets.STAGING_URL }}
|
||||
TEST_PASSWORD: ${{ secrets.TEST_PASSWORD }}
|
||||
```
|
||||
|
||||
## Scenario Guide
|
||||
|
||||
| Scenario | Approach |
|
||||
|---|---|
|
||||
| Small suite (< 5 min) | Single job, no sharding |
|
||||
| Medium suite (5-20 min) | 2-4 shards with matrix |
|
||||
| Large suite (20+ min) | 4-8 shards + blob merge |
|
||||
| Cross-browser on PRs | Chromium only on PRs; all browsers on main |
|
||||
| Staging/prod smoke tests | Separate workflow with `environment:` |
|
||||
| Nightly full regression | `schedule` trigger + `workflow_dispatch` |
|
||||
| Multiple repos, same setup | Reusable workflow with `workflow_call` |
|
||||
| Reproducible env needed | Container job with Playwright image |
|
||||
|
||||
## Common Mistakes
|
||||
|
||||
| Mistake | Problem | Fix |
|
||||
|---|---|---|
|
||||
| No `concurrency` group | Duplicate runs waste minutes | Add `concurrency: { group: ..., cancel-in-progress: true }` |
|
||||
| `fail-fast: true` with sharding | One failure cancels others | Set `fail-fast: false` |
|
||||
| No browser caching | 60-90 seconds wasted per run | Cache `~/.cache/ms-playwright` |
|
||||
| No `timeout-minutes` | Stuck jobs run for 6 hours | Set explicit timeout: 20-30 minutes |
|
||||
| Artifacts only on failure | No report when tests pass | Use `if: ${{ !cancelled() }}` |
|
||||
| Hardcoded secrets | Security risk | Use GitHub Secrets and Environments |
|
||||
| All browsers on every PR | 3x CI cost | Chromium on PR; cross-browser on main |
|
||||
| No artifact retention | Default 90-day fills storage | Set `retention-days: 7-14` |
|
||||
| Missing `--with-deps` | Browser launch failures | Always use `npx playwright install --with-deps` |
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Browser launch fails: "Missing dependencies"
|
||||
|
||||
**Cause**: Browsers restored from cache but OS dependencies weren't cached.
|
||||
|
||||
**Fix**: Run `npx playwright install-deps` on cache hit:
|
||||
|
||||
```yaml
|
||||
- name: Install OS dependencies
|
||||
if: steps.browser-cache.outputs.cache-hit == 'true'
|
||||
run: npx playwright install-deps
|
||||
```
|
||||
|
||||
### Tests pass locally but timeout in CI
|
||||
|
||||
**Cause**: CI runners have fewer resources than dev machines.
|
||||
|
||||
**Fix**: Reduce workers and increase timeouts:
|
||||
|
||||
```typescript
|
||||
// playwright.config.ts
|
||||
import { defineConfig } from '@playwright/test';
|
||||
|
||||
export default defineConfig({
|
||||
workers: process.env.CI ? '50%' : undefined,
|
||||
use: {
|
||||
actionTimeout: process.env.CI ? 15_000 : 10_000,
|
||||
navigationTimeout: process.env.CI ? 30_000 : 15_000,
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
### Sharded reports incomplete
|
||||
|
||||
**Cause**: Artifact names collide or `merge-multiple` not set.
|
||||
|
||||
**Fix**: Unique names per shard and enable merge:
|
||||
|
||||
```yaml
|
||||
# Upload in each shard
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: blob-${{ strategy.job-index }}
|
||||
path: blob-report/
|
||||
|
||||
# Download in merge job
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: all-blobs
|
||||
pattern: blob-*
|
||||
merge-multiple: true
|
||||
```
|
||||
|
||||
### `webServer` fails: "port already in use"
|
||||
|
||||
**Cause**: Zombie process from previous run.
|
||||
|
||||
**Fix**: Kill stale processes before starting:
|
||||
|
||||
```yaml
|
||||
- name: Kill stale processes
|
||||
run: lsof -ti:3000 | xargs kill -9 2>/dev/null || true
|
||||
```
|
||||
|
||||
### No PR annotations
|
||||
|
||||
**Cause**: `github` reporter not configured.
|
||||
|
||||
**Fix**: Add `github` reporter for CI:
|
||||
|
||||
```typescript
|
||||
// playwright.config.ts
|
||||
import { defineConfig } from '@playwright/test';
|
||||
|
||||
export default defineConfig({
|
||||
reporter: process.env.CI
|
||||
? [['html', { open: 'never' }], ['github']]
|
||||
: [['html', { open: 'on-failure' }]],
|
||||
});
|
||||
```
|
||||
|
||||
## Related
|
||||
|
||||
- [test-tags.md](../core/test-tags.md) — tagging and filtering tests
|
||||
- [parallel-sharding.md](parallel-sharding.md) — sharding strategies
|
||||
- [reporting.md](reporting.md) — reporter configuration
|
||||
- [docker.md](docker.md) — container images
|
||||
- [gitlab.md](gitlab.md) — GitLab CI equivalent
|
||||
- [other-providers.md](other-providers.md) — CircleCI, Azure DevOps, Jenkins
|
||||
397
.cursor/skills/playwright-testing/infrastructure-ci-cd/gitlab.md
Normal file
397
.cursor/skills/playwright-testing/infrastructure-ci-cd/gitlab.md
Normal file
|
|
@ -0,0 +1,397 @@
|
|||
# GitLab CI/CD Configuration
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [Key Commands](#key-commands)
|
||||
2. [Patterns](#patterns)
|
||||
3. [Decision Guide](#decision-guide)
|
||||
4. [Anti-Patterns](#anti-patterns)
|
||||
5. [Troubleshooting](#troubleshooting)
|
||||
|
||||
> **When to use**: Running Playwright tests in GitLab pipelines on merge requests, merges to main, or scheduled pipelines.
|
||||
|
||||
## Key Commands
|
||||
|
||||
```bash
|
||||
npx playwright install --with-deps # install browsers + OS deps
|
||||
npx playwright test --shard=1/4 # run 1 of 4 parallel shards
|
||||
npx playwright merge-reports ./blob-report # merge shard results
|
||||
npx playwright test --reporter=dot # minimal output for CI logs
|
||||
```
|
||||
|
||||
## Patterns
|
||||
|
||||
### Basic Pipeline Configuration
|
||||
|
||||
**Use when**: Any GitLab project with Playwright tests.
|
||||
|
||||
```yaml
|
||||
# .gitlab-ci.yml
|
||||
image: mcr.microsoft.com/playwright:v1.48.0-noble
|
||||
|
||||
stages:
|
||||
- install
|
||||
- test
|
||||
- report
|
||||
|
||||
variables:
|
||||
CI: "true"
|
||||
npm_config_cache: "$CI_PROJECT_DIR/.npm"
|
||||
|
||||
cache:
|
||||
key:
|
||||
files:
|
||||
- package-lock.json
|
||||
paths:
|
||||
- .npm/
|
||||
- node_modules/
|
||||
|
||||
setup:
|
||||
stage: install
|
||||
script:
|
||||
- npm ci
|
||||
artifacts:
|
||||
paths:
|
||||
- node_modules/
|
||||
expire_in: 1 hour
|
||||
|
||||
e2e:
|
||||
stage: test
|
||||
needs: [setup]
|
||||
script:
|
||||
- npx playwright test
|
||||
artifacts:
|
||||
when: always
|
||||
paths:
|
||||
- playwright-report/
|
||||
- test-results/
|
||||
expire_in: 14 days
|
||||
rules:
|
||||
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
||||
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
|
||||
```
|
||||
|
||||
### Sharded Parallel Execution
|
||||
|
||||
**Use when**: Test suite exceeds 10 minutes. GitLab's `parallel` keyword splits across jobs automatically.
|
||||
**Avoid when**: Suite runs under 5 minutes.
|
||||
|
||||
```yaml
|
||||
image: mcr.microsoft.com/playwright:v1.48.0-noble
|
||||
|
||||
stages:
|
||||
- install
|
||||
- test
|
||||
- report
|
||||
|
||||
variables:
|
||||
CI: "true"
|
||||
npm_config_cache: "$CI_PROJECT_DIR/.npm"
|
||||
|
||||
cache:
|
||||
key:
|
||||
files:
|
||||
- package-lock.json
|
||||
paths:
|
||||
- .npm/
|
||||
- node_modules/
|
||||
|
||||
setup:
|
||||
stage: install
|
||||
script:
|
||||
- npm ci
|
||||
artifacts:
|
||||
paths:
|
||||
- node_modules/
|
||||
expire_in: 1 hour
|
||||
|
||||
e2e:
|
||||
stage: test
|
||||
needs: [setup]
|
||||
parallel: 4
|
||||
script:
|
||||
- npx playwright test --shard=$CI_NODE_INDEX/$CI_NODE_TOTAL
|
||||
artifacts:
|
||||
when: always
|
||||
paths:
|
||||
- blob-report/
|
||||
expire_in: 1 hour
|
||||
rules:
|
||||
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
||||
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
|
||||
|
||||
combine-reports:
|
||||
stage: report
|
||||
needs: [e2e]
|
||||
when: always
|
||||
script:
|
||||
- npx playwright merge-reports --reporter=html ./blob-report
|
||||
artifacts:
|
||||
when: always
|
||||
paths:
|
||||
- playwright-report/
|
||||
expire_in: 14 days
|
||||
```
|
||||
|
||||
**Config for sharded pipelines:**
|
||||
|
||||
```typescript
|
||||
// playwright.config.ts
|
||||
export default defineConfig({
|
||||
reporter: process.env.CI
|
||||
? [["blob"], ["dot"]]
|
||||
: [["html", { open: "on-failure" }]],
|
||||
});
|
||||
```
|
||||
|
||||
### Environment Variables and Secrets
|
||||
|
||||
**Use when**: Tests need secrets (API keys, passwords) and should only run on merge requests or the default branch.
|
||||
|
||||
```yaml
|
||||
image: mcr.microsoft.com/playwright:v1.48.0-noble
|
||||
|
||||
stages:
|
||||
- test
|
||||
|
||||
variables:
|
||||
CI: "true"
|
||||
|
||||
e2e:staging:
|
||||
stage: test
|
||||
variables:
|
||||
BASE_URL: $STAGING_URL
|
||||
TEST_PASSWORD: $TEST_PASSWORD
|
||||
API_KEY: $API_KEY
|
||||
before_script:
|
||||
- npm ci
|
||||
script:
|
||||
- npx playwright test
|
||||
artifacts:
|
||||
when: always
|
||||
paths:
|
||||
- playwright-report/
|
||||
- test-results/
|
||||
expire_in: 14 days
|
||||
rules:
|
||||
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
||||
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
|
||||
- when: manual
|
||||
allow_failure: true
|
||||
```
|
||||
|
||||
**Setting variables in GitLab:**
|
||||
Navigate to **Settings > CI/CD > Variables** and add:
|
||||
|
||||
- `STAGING_URL` -- not masked, not protected
|
||||
- `TEST_PASSWORD` -- masked, protected
|
||||
- `API_KEY` -- masked, protected
|
||||
|
||||
### Multi-Browser Matrix
|
||||
|
||||
**Use when**: Running Chromium on MRs and all browsers on the default branch.
|
||||
|
||||
```yaml
|
||||
image: mcr.microsoft.com/playwright:v1.48.0-noble
|
||||
|
||||
stages:
|
||||
- install
|
||||
- test
|
||||
|
||||
variables:
|
||||
CI: "true"
|
||||
|
||||
setup:
|
||||
stage: install
|
||||
script:
|
||||
- npm ci
|
||||
artifacts:
|
||||
paths:
|
||||
- node_modules/
|
||||
expire_in: 1 hour
|
||||
|
||||
e2e:chromium:
|
||||
stage: test
|
||||
needs: [setup]
|
||||
script:
|
||||
- npx playwright test --project=chromium
|
||||
artifacts:
|
||||
when: always
|
||||
paths:
|
||||
- playwright-report/
|
||||
- test-results/
|
||||
expire_in: 14 days
|
||||
rules:
|
||||
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
||||
|
||||
e2e:all-browsers:
|
||||
stage: test
|
||||
needs: [setup]
|
||||
parallel:
|
||||
matrix:
|
||||
- PROJECT: [chromium, firefox, webkit]
|
||||
script:
|
||||
- npx playwright test --project=$PROJECT
|
||||
artifacts:
|
||||
when: always
|
||||
paths:
|
||||
- playwright-report/
|
||||
- test-results/
|
||||
expire_in: 14 days
|
||||
rules:
|
||||
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
|
||||
```
|
||||
|
||||
### Services Integration (Database, Cache)
|
||||
|
||||
**Use when**: Tests need the application running alongside Playwright, or you need external services.
|
||||
|
||||
```yaml
|
||||
stages:
|
||||
- test
|
||||
|
||||
e2e:integration:
|
||||
stage: test
|
||||
image: mcr.microsoft.com/playwright:v1.48.0-noble
|
||||
services:
|
||||
- name: postgres:latest
|
||||
alias: db
|
||||
- name: redis:latest
|
||||
alias: cache
|
||||
variables:
|
||||
CI: "true"
|
||||
DATABASE_URL: "postgresql://postgres:postgres@db:5432/testdb"
|
||||
REDIS_URL: "redis://cache:6379"
|
||||
POSTGRES_PASSWORD: "postgres"
|
||||
POSTGRES_DB: "testdb"
|
||||
before_script:
|
||||
- npm ci
|
||||
- npx prisma db push
|
||||
- npx prisma db seed
|
||||
script:
|
||||
- npx playwright test
|
||||
artifacts:
|
||||
when: always
|
||||
paths:
|
||||
- playwright-report/
|
||||
- test-results/
|
||||
expire_in: 14 days
|
||||
rules:
|
||||
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
||||
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
|
||||
```
|
||||
|
||||
### Scheduled Nightly Regression
|
||||
|
||||
**Use when**: Full regression is too slow for every MR.
|
||||
|
||||
```yaml
|
||||
e2e:nightly:
|
||||
stage: test
|
||||
image: mcr.microsoft.com/playwright:v1.48.0-noble
|
||||
before_script:
|
||||
- npm ci
|
||||
script:
|
||||
- npx playwright test --grep @regression
|
||||
artifacts:
|
||||
when: always
|
||||
paths:
|
||||
- playwright-report/
|
||||
expire_in: 30 days
|
||||
rules:
|
||||
- if: $CI_PIPELINE_SOURCE == "schedule"
|
||||
```
|
||||
|
||||
Set up the schedule in **CI/CD > Schedules**: `0 3 * * 1-5` (3 AM UTC, weekdays).
|
||||
|
||||
## Decision Guide
|
||||
|
||||
| Scenario | Approach | Why |
|
||||
| ------------------------------------ | ------------------------------------------------------ | --------------------------------------------------- |
|
||||
| Simple project, < 5 min suite | Single `test` job using Playwright Docker image | No sharding overhead; artifacts capture report |
|
||||
| Suite > 10 min | `parallel: N` with `--shard` | GitLab auto-assigns `CI_NODE_INDEX`/`CI_NODE_TOTAL` |
|
||||
| Merge request fast feedback | Chromium only on MRs; all browsers on main | 3x fewer pipeline minutes on MRs |
|
||||
| External services needed (DB, Redis) | `services:` keyword with Postgres/Redis images | GitLab manages service lifecycle |
|
||||
| Secrets for staging environment | GitLab CI/CD Variables (masked + protected) | Never hardcode secrets in `.gitlab-ci.yml` |
|
||||
| Full nightly regression | Pipeline schedule (`CI_PIPELINE_SOURCE == "schedule"`) | Avoids blocking MR pipelines |
|
||||
| Report browsing | `artifacts:` with `paths: [playwright-report/]` | Browse directly in GitLab job artifacts UI |
|
||||
|
||||
## Anti-Patterns
|
||||
|
||||
| Anti-Pattern | Problem | Do This Instead |
|
||||
| ---------------------------------------------------- | ------------------------------------------------------------------ | ------------------------------------------------------------------------- |
|
||||
| Not using the Playwright Docker image | Installing browsers every run adds 1-2 minutes | Use `mcr.microsoft.com/playwright:v1.48.0-noble` as base image |
|
||||
| `artifacts: when: on_failure` only | No report when tests pass; can't verify results | Use `when: always` to capture reports regardless |
|
||||
| No `expire_in` on artifacts | Artifacts accumulate and consume storage | Set `expire_in: 14 days` for reports, `1 hour` for intermediate artifacts |
|
||||
| Hardcoding `CI_NODE_TOTAL` in shard flag | Breaks when you change `parallel:` value | Use `--shard=$CI_NODE_INDEX/$CI_NODE_TOTAL` |
|
||||
| Skipping `needs:` between stages | Jobs wait for all previous stage jobs, not just their dependencies | Use `needs:` for precise dependency graphs |
|
||||
| Large `cache:` including `node_modules/` without key | Stale cache causes version conflicts | Key cache on `package-lock.json` hash |
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Browser launch fails: "Failed to launch browser"
|
||||
|
||||
**Cause**: Not using the Playwright Docker image, or using a version that doesn't match your `@playwright/test` version.
|
||||
|
||||
**Fix**: Match the Docker image tag to your Playwright version:
|
||||
|
||||
```yaml
|
||||
# Check your version: npm ls @playwright/test
|
||||
image: mcr.microsoft.com/playwright:v1.48.0-noble
|
||||
```
|
||||
|
||||
### Tests hang in GitLab runner: "Navigation timeout exceeded"
|
||||
|
||||
**Cause**: GitLab shared runners may have limited resources.
|
||||
|
||||
**Fix**: Reduce workers and increase timeouts:
|
||||
|
||||
```typescript
|
||||
export default defineConfig({
|
||||
workers: process.env.CI ? 2 : undefined,
|
||||
use: {
|
||||
navigationTimeout: process.env.CI ? 30_000 : 15_000,
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
### Pipeline runs on every push, not just merge requests
|
||||
|
||||
**Cause**: Missing `rules:` configuration.
|
||||
|
||||
**Fix**: Add explicit rules:
|
||||
|
||||
```yaml
|
||||
rules:
|
||||
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
||||
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
|
||||
```
|
||||
|
||||
### Services (Postgres/Redis) not reachable from tests
|
||||
|
||||
**Cause**: Using `localhost` instead of the service alias.
|
||||
|
||||
**Fix**: Use the service alias as hostname:
|
||||
|
||||
```yaml
|
||||
services:
|
||||
- name: postgres:latest
|
||||
alias: db
|
||||
|
||||
variables:
|
||||
DATABASE_URL: "postgresql://postgres:postgres@db:5432/testdb"
|
||||
```
|
||||
|
||||
### Merged report is empty after sharded run
|
||||
|
||||
**Cause**: Each shard job needs the `blob` reporter, not `html`.
|
||||
|
||||
**Fix**: Configure blob reporter for CI:
|
||||
|
||||
```typescript
|
||||
export default defineConfig({
|
||||
reporter: process.env.CI
|
||||
? [["blob"], ["dot"]]
|
||||
: [["html", { open: "on-failure" }]],
|
||||
});
|
||||
```
|
||||
|
|
@ -0,0 +1,521 @@
|
|||
# CI: CircleCI, Azure DevOps, and Jenkins
|
||||
|
||||
> **When to use**: Running Playwright tests in CI platforms other than GitHub Actions or GitLab.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [Common Commands](#common-commands)
|
||||
2. [Jenkins](#jenkins)
|
||||
3. [CircleCI](#circleci)
|
||||
4. [Azure DevOps](#azure-devops)
|
||||
5. [JUnit Reporter Config](#junit-reporter-config)
|
||||
6. [Platform Comparison](#platform-comparison)
|
||||
7. [Troubleshooting](#troubleshooting)
|
||||
8. [Anti-Patterns](#anti-patterns)
|
||||
|
||||
---
|
||||
|
||||
## Common Commands
|
||||
|
||||
```bash
|
||||
npx playwright install --with-deps # browsers + OS dependencies
|
||||
npx playwright test --shard=1/4 # parallel sharding
|
||||
npx playwright merge-reports ./blob-report # combine shard results
|
||||
npx playwright test --reporter=dot,html # multiple reporters
|
||||
```
|
||||
|
||||
## Jenkins
|
||||
|
||||
### Declarative Pipeline
|
||||
|
||||
```groovy
|
||||
// Jenkinsfile
|
||||
pipeline {
|
||||
agent {
|
||||
docker {
|
||||
image 'mcr.microsoft.com/playwright:v1.48.0-noble'
|
||||
args '-u root'
|
||||
}
|
||||
}
|
||||
|
||||
environment {
|
||||
CI = 'true'
|
||||
HOME = '/root'
|
||||
npm_config_cache = "${WORKSPACE}/.npm"
|
||||
}
|
||||
|
||||
options {
|
||||
timeout(time: 30, unit: 'MINUTES')
|
||||
disableConcurrentBuilds()
|
||||
}
|
||||
|
||||
stages {
|
||||
stage('Install') {
|
||||
steps {
|
||||
sh 'npm ci'
|
||||
}
|
||||
}
|
||||
|
||||
stage('Test') {
|
||||
steps {
|
||||
sh 'npx playwright test'
|
||||
}
|
||||
post {
|
||||
always {
|
||||
junit allowEmptyResults: true,
|
||||
testResults: 'results/junit.xml'
|
||||
archiveArtifacts artifacts: 'pw-report/**',
|
||||
allowEmptyArchive: true
|
||||
archiveArtifacts artifacts: 'results/**',
|
||||
allowEmptyArchive: true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
post {
|
||||
failure {
|
||||
echo 'Tests failed!'
|
||||
}
|
||||
cleanup {
|
||||
cleanWs()
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Parallel Shards
|
||||
|
||||
```groovy
|
||||
// Jenkinsfile (sharded)
|
||||
pipeline {
|
||||
agent none
|
||||
|
||||
environment {
|
||||
CI = 'true'
|
||||
HOME = '/root'
|
||||
}
|
||||
|
||||
options {
|
||||
timeout(time: 30, unit: 'MINUTES')
|
||||
}
|
||||
|
||||
stages {
|
||||
stage('Test') {
|
||||
parallel {
|
||||
stage('Shard 1') {
|
||||
agent {
|
||||
docker {
|
||||
image 'mcr.microsoft.com/playwright:v1.48.0-noble'
|
||||
args '-u root'
|
||||
}
|
||||
}
|
||||
steps {
|
||||
sh 'npm ci'
|
||||
sh 'npx playwright test --shard=1/4'
|
||||
}
|
||||
post {
|
||||
always {
|
||||
archiveArtifacts artifacts: 'blob-report/**',
|
||||
allowEmptyArchive: true
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Shard 2') {
|
||||
agent {
|
||||
docker {
|
||||
image 'mcr.microsoft.com/playwright:v1.48.0-noble'
|
||||
args '-u root'
|
||||
}
|
||||
}
|
||||
steps {
|
||||
sh 'npm ci'
|
||||
sh 'npx playwright test --shard=2/4'
|
||||
}
|
||||
post {
|
||||
always {
|
||||
archiveArtifacts artifacts: 'blob-report/**',
|
||||
allowEmptyArchive: true
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Shard 3') {
|
||||
agent {
|
||||
docker {
|
||||
image 'mcr.microsoft.com/playwright:v1.48.0-noble'
|
||||
args '-u root'
|
||||
}
|
||||
}
|
||||
steps {
|
||||
sh 'npm ci'
|
||||
sh 'npx playwright test --shard=3/4'
|
||||
}
|
||||
post {
|
||||
always {
|
||||
archiveArtifacts artifacts: 'blob-report/**',
|
||||
allowEmptyArchive: true
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Shard 4') {
|
||||
agent {
|
||||
docker {
|
||||
image 'mcr.microsoft.com/playwright:v1.48.0-noble'
|
||||
args '-u root'
|
||||
}
|
||||
}
|
||||
steps {
|
||||
sh 'npm ci'
|
||||
sh 'npx playwright test --shard=4/4'
|
||||
}
|
||||
post {
|
||||
always {
|
||||
archiveArtifacts artifacts: 'blob-report/**',
|
||||
allowEmptyArchive: true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## CircleCI
|
||||
|
||||
### Basic Pipeline
|
||||
|
||||
```yaml
|
||||
# .circleci/config.yml
|
||||
version: 2.1
|
||||
|
||||
executors:
|
||||
pw:
|
||||
docker:
|
||||
- image: mcr.microsoft.com/playwright:v1.48.0-noble
|
||||
working_directory: ~/app
|
||||
|
||||
jobs:
|
||||
install:
|
||||
executor: pw
|
||||
steps:
|
||||
- checkout
|
||||
- restore_cache:
|
||||
keys:
|
||||
- deps-{{ checksum "package-lock.json" }}
|
||||
- run: npm ci
|
||||
- save_cache:
|
||||
key: deps-{{ checksum "package-lock.json" }}
|
||||
paths:
|
||||
- node_modules
|
||||
- persist_to_workspace:
|
||||
root: .
|
||||
paths:
|
||||
- node_modules
|
||||
|
||||
test:
|
||||
executor: pw
|
||||
parallelism: 4
|
||||
steps:
|
||||
- checkout
|
||||
- attach_workspace:
|
||||
at: .
|
||||
- run:
|
||||
name: Run tests
|
||||
command: |
|
||||
npx playwright test --shard=$((CIRCLE_NODE_INDEX + 1))/$CIRCLE_NODE_TOTAL
|
||||
- store_artifacts:
|
||||
path: pw-report
|
||||
destination: pw-report
|
||||
- store_artifacts:
|
||||
path: results
|
||||
destination: results
|
||||
- store_test_results:
|
||||
path: results/junit.xml
|
||||
|
||||
workflows:
|
||||
test:
|
||||
jobs:
|
||||
- install
|
||||
- test:
|
||||
requires:
|
||||
- install
|
||||
```
|
||||
|
||||
### Using Orbs
|
||||
|
||||
```yaml
|
||||
# .circleci/config.yml
|
||||
version: 2.1
|
||||
|
||||
orbs:
|
||||
node: circleci/node@latest
|
||||
|
||||
executors:
|
||||
pw:
|
||||
docker:
|
||||
- image: mcr.microsoft.com/playwright:v1.48.0-noble
|
||||
|
||||
jobs:
|
||||
e2e:
|
||||
executor: pw
|
||||
parallelism: 4
|
||||
steps:
|
||||
- checkout
|
||||
- node/install-packages
|
||||
- run:
|
||||
name: Run tests
|
||||
command: npx playwright test --shard=$((CIRCLE_NODE_INDEX + 1))/$CIRCLE_NODE_TOTAL
|
||||
- store_artifacts:
|
||||
path: pw-report
|
||||
- store_test_results:
|
||||
path: results/junit.xml
|
||||
|
||||
workflows:
|
||||
main:
|
||||
jobs:
|
||||
- e2e
|
||||
```
|
||||
|
||||
## Azure DevOps
|
||||
|
||||
### Basic Pipeline
|
||||
|
||||
```yaml
|
||||
# azure-pipelines.yml
|
||||
trigger:
|
||||
branches:
|
||||
include:
|
||||
- main
|
||||
|
||||
pr:
|
||||
branches:
|
||||
include:
|
||||
- main
|
||||
|
||||
pool:
|
||||
vmImage: "ubuntu-latest"
|
||||
|
||||
variables:
|
||||
CI: "true"
|
||||
npm_config_cache: $(Pipeline.Workspace)/.npm
|
||||
|
||||
steps:
|
||||
- task: NodeTool@0
|
||||
inputs:
|
||||
versionSpec: "20.x"
|
||||
displayName: "Install Node.js"
|
||||
|
||||
- task: Cache@2
|
||||
inputs:
|
||||
key: 'npm | "$(Agent.OS)" | package-lock.json'
|
||||
restoreKeys: |
|
||||
npm | "$(Agent.OS)"
|
||||
path: $(npm_config_cache)
|
||||
displayName: "Cache npm"
|
||||
|
||||
- script: npm ci
|
||||
displayName: "Install dependencies"
|
||||
|
||||
- script: npx playwright install --with-deps
|
||||
displayName: "Install browsers"
|
||||
|
||||
- script: npx playwright test
|
||||
displayName: "Run tests"
|
||||
|
||||
- task: PublishTestResults@2
|
||||
condition: always()
|
||||
inputs:
|
||||
testResultsFormat: "JUnit"
|
||||
testResultsFiles: "results/junit.xml"
|
||||
mergeTestResults: true
|
||||
testRunTitle: "E2E Tests"
|
||||
displayName: "Publish results"
|
||||
|
||||
- task: PublishPipelineArtifact@1
|
||||
condition: always()
|
||||
inputs:
|
||||
targetPath: pw-report
|
||||
artifact: pw-report
|
||||
publishLocation: "pipeline"
|
||||
displayName: "Upload report"
|
||||
```
|
||||
|
||||
### With Sharding
|
||||
|
||||
```yaml
|
||||
# azure-pipelines.yml
|
||||
trigger:
|
||||
branches:
|
||||
include:
|
||||
- main
|
||||
|
||||
pr:
|
||||
branches:
|
||||
include:
|
||||
- main
|
||||
|
||||
variables:
|
||||
CI: "true"
|
||||
|
||||
stages:
|
||||
- stage: Test
|
||||
jobs:
|
||||
- job: E2E
|
||||
pool:
|
||||
vmImage: "ubuntu-latest"
|
||||
strategy:
|
||||
matrix:
|
||||
shard1:
|
||||
SHARD: "1/4"
|
||||
shard2:
|
||||
SHARD: "2/4"
|
||||
shard3:
|
||||
SHARD: "3/4"
|
||||
shard4:
|
||||
SHARD: "4/4"
|
||||
steps:
|
||||
- task: NodeTool@0
|
||||
inputs:
|
||||
versionSpec: "20.x"
|
||||
|
||||
- script: npm ci
|
||||
displayName: "Install dependencies"
|
||||
|
||||
- script: npx playwright install --with-deps
|
||||
displayName: "Install browsers"
|
||||
|
||||
- script: npx playwright test --shard=$(SHARD)
|
||||
displayName: "Run tests (shard $(SHARD))"
|
||||
|
||||
- task: PublishPipelineArtifact@1
|
||||
condition: always()
|
||||
inputs:
|
||||
targetPath: blob-report
|
||||
artifact: blob-report-$(System.JobPositionInPhase)
|
||||
displayName: "Upload blob report"
|
||||
|
||||
- stage: Report
|
||||
dependsOn: Test
|
||||
condition: always()
|
||||
jobs:
|
||||
- job: MergeReports
|
||||
pool:
|
||||
vmImage: "ubuntu-latest"
|
||||
steps:
|
||||
- task: NodeTool@0
|
||||
inputs:
|
||||
versionSpec: "20.x"
|
||||
|
||||
- script: npm ci
|
||||
displayName: "Install dependencies"
|
||||
|
||||
- task: DownloadPipelineArtifact@2
|
||||
inputs:
|
||||
patterns: "blob-report-*/**"
|
||||
path: all-blob-reports
|
||||
displayName: "Download blob reports"
|
||||
|
||||
- script: npx playwright merge-reports --reporter=html ./all-blob-reports
|
||||
displayName: "Merge reports"
|
||||
|
||||
- task: PublishPipelineArtifact@1
|
||||
inputs:
|
||||
targetPath: pw-report
|
||||
artifact: pw-report
|
||||
displayName: "Upload merged report"
|
||||
```
|
||||
|
||||
## JUnit Reporter Config
|
||||
|
||||
All platforms benefit from JUnit output for native test result display:
|
||||
|
||||
```typescript
|
||||
// playwright.config.ts
|
||||
import { defineConfig } from "@playwright/test";
|
||||
|
||||
export default defineConfig({
|
||||
reporter: process.env.CI
|
||||
? [
|
||||
["dot"],
|
||||
["html", { open: "never" }],
|
||||
["junit", { outputFile: "results/junit.xml" }],
|
||||
]
|
||||
: [["html", { open: "on-failure" }]],
|
||||
});
|
||||
```
|
||||
|
||||
## Platform Comparison
|
||||
|
||||
| Feature | CircleCI | Azure DevOps | Jenkins |
|
||||
| ----------------- | ----------------------------------------------- | -------------------------------- | ---------------------- |
|
||||
| Docker support | `docker:` executor | `vmImage` or container jobs | Docker Pipeline plugin |
|
||||
| Parallelism | `parallelism: N` + `CIRCLE_NODE_INDEX` | `strategy.matrix` | `parallel` stages |
|
||||
| Artifact upload | `store_artifacts` | `PublishPipelineArtifact@1` | `archiveArtifacts` |
|
||||
| JUnit integration | `store_test_results` | `PublishTestResults@2` | `junit` step |
|
||||
| Shard variable | `$((CIRCLE_NODE_INDEX + 1))/$CIRCLE_NODE_TOTAL` | Define in matrix: `SHARD: '1/4'` | Hardcode per stage |
|
||||
| Cache key | `checksum "package-lock.json"` | `Cache@2` with key template | `stash`/`unstash` |
|
||||
| Secrets | Context + env variables | Variable groups | Credentials plugin |
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Jenkins: "Browser closed unexpectedly"
|
||||
|
||||
Running as non-root in container causes sandbox issues.
|
||||
|
||||
```groovy
|
||||
agent {
|
||||
docker {
|
||||
image 'mcr.microsoft.com/playwright:v1.48.0-noble'
|
||||
args '-u root'
|
||||
}
|
||||
}
|
||||
environment {
|
||||
HOME = '/root'
|
||||
}
|
||||
```
|
||||
|
||||
### CircleCI: "Executable doesn't exist"
|
||||
|
||||
Image version mismatch with `@playwright/test` version. Use `latest` tag or match versions:
|
||||
|
||||
```yaml
|
||||
docker:
|
||||
- image: mcr.microsoft.com/playwright:v1.48.0-noble
|
||||
```
|
||||
|
||||
### Azure DevOps: Test results not showing
|
||||
|
||||
Missing JUnit reporter or `PublishTestResults@2` task:
|
||||
|
||||
```typescript
|
||||
reporter: [['junit', { outputFile: 'results/junit.xml' }]],
|
||||
```
|
||||
|
||||
```yaml
|
||||
- task: PublishTestResults@2
|
||||
condition: always()
|
||||
inputs:
|
||||
testResultsFormat: "JUnit"
|
||||
testResultsFiles: "results/junit.xml"
|
||||
```
|
||||
|
||||
### Shard index off by one
|
||||
|
||||
CircleCI's `CIRCLE_NODE_INDEX` is 0-based, Playwright's `--shard` is 1-based:
|
||||
|
||||
```yaml
|
||||
# CircleCI - add 1
|
||||
command: npx playwright test --shard=$((CIRCLE_NODE_INDEX + 1))/$CIRCLE_NODE_TOTAL
|
||||
```
|
||||
|
||||
## Anti-Patterns
|
||||
|
||||
| Anti-Pattern | Problem | Solution |
|
||||
| ----------------------------------- | ----------------------------------------- | ---------------------------------------------------- |
|
||||
| Missing `--with-deps` on bare metal | OS libs missing, browser launch fails | Use Playwright Docker image or `--with-deps` |
|
||||
| No JUnit reporter | CI can't display test results | Add `['junit', { outputFile: 'results/junit.xml' }]` |
|
||||
| No job timeout | Hung tests consume resources indefinitely | Set explicit timeout (20-30 min) |
|
||||
| No artifact upload on success | Can't verify passing results | Always upload reports (`condition: always()`) |
|
||||
| Non-root in container without setup | Permission errors on browser binaries | Run as root or configure permissions |
|
||||
| Hardcoded shard count | Must update multiple places | Use CI-native variables |
|
||||
|
|
@ -0,0 +1,371 @@
|
|||
# Sharding and Parallel Execution
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [CLI Commands](#cli-commands)
|
||||
2. [Patterns](#patterns)
|
||||
3. [Decision Guide](#decision-guide)
|
||||
4. [Anti-Patterns](#anti-patterns)
|
||||
5. [Troubleshooting](#troubleshooting)
|
||||
|
||||
> **When to use**: Speeding up test suites by running tests concurrently on one machine (workers) or splitting across multiple CI jobs (sharding).
|
||||
|
||||
## CLI Commands
|
||||
|
||||
```bash
|
||||
# Parallelism within one machine
|
||||
npx playwright test --workers=4
|
||||
npx playwright test --workers=50%
|
||||
|
||||
# Splitting across CI jobs
|
||||
npx playwright test --shard=1/4
|
||||
npx playwright test --shard=2/4
|
||||
|
||||
# Merging shard outputs
|
||||
npx playwright merge-reports ./blob-report
|
||||
npx playwright merge-reports --reporter=html,json ./blob-report
|
||||
|
||||
# Override config for single run
|
||||
npx playwright test --fully-parallel
|
||||
```
|
||||
|
||||
## Patterns
|
||||
|
||||
### Worker Configuration
|
||||
|
||||
**Use when**: Controlling concurrent test execution on a single machine.
|
||||
|
||||
```ts
|
||||
// playwright.config.ts
|
||||
import { defineConfig } from "@playwright/test";
|
||||
|
||||
export default defineConfig({
|
||||
// Tests WITHIN a file also run in parallel
|
||||
fullyParallel: true,
|
||||
|
||||
// Worker count options:
|
||||
// - undefined: auto-detect (half CPU cores)
|
||||
// - number: fixed count
|
||||
// - string: percentage of cores
|
||||
workers: process.env.CI ? "50%" : undefined,
|
||||
});
|
||||
```
|
||||
|
||||
**`fullyParallel` behavior:**
|
||||
|
||||
| Setting | Files parallel | Tests in file parallel |
|
||||
| -------------------------------- | -------------- | ---------------------- |
|
||||
| `fullyParallel: false` (default) | Yes | No (serial) |
|
||||
| `fullyParallel: true` | Yes | Yes |
|
||||
|
||||
**Serial execution for specific files:**
|
||||
|
||||
```ts
|
||||
// tests/checkout-flow.spec.ts
|
||||
import { test, expect } from "@playwright/test";
|
||||
|
||||
test.describe.configure({ mode: "serial" });
|
||||
|
||||
test("add items to cart", async ({ page }) => {
|
||||
// ...
|
||||
});
|
||||
|
||||
test("complete payment", async ({ page }) => {
|
||||
// ...
|
||||
});
|
||||
```
|
||||
|
||||
### Sharding Across CI Machines
|
||||
|
||||
**Use when**: Suite exceeds 5 minutes even with maximum workers.
|
||||
|
||||
```bash
|
||||
# Job 1 Job 2 Job 3 Job 4
|
||||
--shard=1/4 --shard=2/4 --shard=3/4 --shard=4/4
|
||||
```
|
||||
|
||||
**Config for sharded runs:**
|
||||
|
||||
```ts
|
||||
// playwright.config.ts
|
||||
import { defineConfig } from "@playwright/test";
|
||||
|
||||
export default defineConfig({
|
||||
fullyParallel: true,
|
||||
workers: process.env.CI ? "50%" : undefined,
|
||||
|
||||
reporter: process.env.CI
|
||||
? [["blob"], ["github"]]
|
||||
: [["html", { open: "on-failure" }]],
|
||||
});
|
||||
```
|
||||
|
||||
### Merging Shard Reports
|
||||
|
||||
**Use when**: Combining blob reports from multiple shards into a unified report.
|
||||
|
||||
```bash
|
||||
# Merge all blobs into HTML
|
||||
npx playwright merge-reports --reporter=html ./all-blob-reports
|
||||
|
||||
# Multiple formats
|
||||
npx playwright merge-reports --reporter=html,json,junit ./all-blob-reports
|
||||
|
||||
# Custom output location
|
||||
PLAYWRIGHT_HTML_REPORT=merged-report npx playwright merge-reports --reporter=html ./all-blob-reports
|
||||
```
|
||||
|
||||
**GitHub Actions merge job:**
|
||||
|
||||
```yaml
|
||||
merge-reports:
|
||||
if: ${{ !cancelled() }}
|
||||
needs: test
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- run: npm ci
|
||||
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: all-blob-reports
|
||||
pattern: blob-report-*
|
||||
merge-multiple: true
|
||||
|
||||
- run: npx playwright merge-reports --reporter=html ./all-blob-reports
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: playwright-report
|
||||
path: playwright-report/
|
||||
retention-days: 14
|
||||
```
|
||||
|
||||
### Worker-Scoped Fixtures
|
||||
|
||||
**Use when**: Expensive resources (DB connections, auth tokens) should be created once per worker, not per test.
|
||||
|
||||
```ts
|
||||
// fixtures.ts
|
||||
import { test as base } from "@playwright/test";
|
||||
|
||||
type WorkerFixtures = {
|
||||
dbClient: DatabaseClient;
|
||||
apiToken: string;
|
||||
};
|
||||
|
||||
export const test = base.extend<{}, WorkerFixtures>({
|
||||
dbClient: [
|
||||
async ({}, use) => {
|
||||
const client = await DatabaseClient.connect(process.env.DB_URL!);
|
||||
await use(client);
|
||||
await client.disconnect();
|
||||
},
|
||||
{ scope: "worker" },
|
||||
],
|
||||
|
||||
apiToken: [
|
||||
async ({}, use, workerInfo) => {
|
||||
const res = await fetch(`${process.env.API_URL}/auth`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
user: `test-user-${workerInfo.workerIndex}`,
|
||||
password: process.env.TEST_PASSWORD,
|
||||
}),
|
||||
});
|
||||
const { token } = await res.json();
|
||||
await use(token);
|
||||
},
|
||||
{ scope: "worker" },
|
||||
],
|
||||
});
|
||||
|
||||
export { expect } from "@playwright/test";
|
||||
```
|
||||
|
||||
### Test Isolation for Parallelism
|
||||
|
||||
**Use when**: Preparing tests to run without interference.
|
||||
|
||||
Each test must create its own state. No test should depend on or modify shared state.
|
||||
|
||||
```ts
|
||||
// BAD: Shared user causes race conditions
|
||||
test("edit settings", async ({ page }) => {
|
||||
await page.goto("/users/test-user/settings");
|
||||
await page.getByLabel("Email").fill("new@example.com");
|
||||
await page.getByRole("button", { name: "Save" }).click();
|
||||
});
|
||||
|
||||
// GOOD: Unique user per test
|
||||
test("edit settings", async ({ page, request }) => {
|
||||
const res = await request.post("/api/users", {
|
||||
data: { name: `user-${Date.now()}`, email: `${Date.now()}@test.com` },
|
||||
});
|
||||
const user = await res.json();
|
||||
|
||||
await page.goto(`/users/${user.id}/settings`);
|
||||
await page.getByLabel("Email").fill("updated@example.com");
|
||||
await page.getByRole("button", { name: "Save" }).click();
|
||||
await expect(page.getByLabel("Email")).toHaveValue("updated@example.com");
|
||||
|
||||
await request.delete(`/api/users/${user.id}`);
|
||||
});
|
||||
```
|
||||
|
||||
**Using `testInfo` for unique identifiers:**
|
||||
|
||||
```ts
|
||||
import { test, expect } from "@playwright/test";
|
||||
|
||||
test("submit order", async ({ page }, testInfo) => {
|
||||
const orderId = `order-${testInfo.workerIndex}-${Date.now()}`;
|
||||
await page.goto(`/orders/new?ref=${orderId}`);
|
||||
// ...
|
||||
});
|
||||
```
|
||||
|
||||
### Dynamic Shard Count
|
||||
|
||||
**Use when**: Automatically adjusting shards based on test count.
|
||||
|
||||
```yaml
|
||||
# .github/workflows/playwright.yml
|
||||
jobs:
|
||||
calculate-shards:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
shard-count: ${{ steps.calc.outputs.count }}
|
||||
shard-matrix: ${{ steps.calc.outputs.matrix }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- run: npm ci
|
||||
- id: calc
|
||||
run: |
|
||||
TEST_COUNT=$(npx playwright test --list --reporter=json 2>/dev/null | node -e "
|
||||
const data = require('fs').readFileSync('/dev/stdin', 'utf8');
|
||||
const parsed = JSON.parse(data);
|
||||
console.log(parsed.suites?.reduce((acc, s) => acc + (s.specs?.length || 0), 0) || 0);
|
||||
")
|
||||
# 1 shard per 20 tests, min 1, max 8
|
||||
SHARDS=$(( (TEST_COUNT + 19) / 20 ))
|
||||
SHARDS=$(( SHARDS > 8 ? 8 : SHARDS ))
|
||||
SHARDS=$(( SHARDS < 1 ? 1 : SHARDS ))
|
||||
MATRIX="["
|
||||
for i in $(seq 1 $SHARDS); do
|
||||
[ $i -gt 1 ] && MATRIX+=","
|
||||
MATRIX+="\"$i/$SHARDS\""
|
||||
done
|
||||
MATRIX+="]"
|
||||
echo "count=$SHARDS" >> $GITHUB_OUTPUT
|
||||
echo "matrix=$MATRIX" >> $GITHUB_OUTPUT
|
||||
|
||||
test:
|
||||
needs: calculate-shards
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shard: ${{ fromJson(needs.calculate-shards.outputs.shard-matrix) }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- run: npm ci
|
||||
- run: npx playwright install --with-deps
|
||||
- run: npx playwright test --shard=${{ matrix.shard }}
|
||||
```
|
||||
|
||||
## Decision Guide
|
||||
|
||||
| Scenario | Workers | Shards | Reason |
|
||||
| -------------------------------- | -------------- | ------ | --------------------------------------- |
|
||||
| < 50 tests, < 5 min | Auto (default) | None | No optimization needed |
|
||||
| 50-200 tests, 5-15 min | `'50%'` in CI | 2-4 | Balance speed and cost |
|
||||
| 200+ tests, > 15 min | `'50%'` in CI | 4-8 | Keep feedback under 10 min |
|
||||
| Flaky due to resource contention | Reduce to 2 | Keep | Less CPU/memory pressure |
|
||||
| Tests modify shared database | 1 or isolate | Useful | Sharding splits files; workers run them |
|
||||
| CI has limited resources | 1 or `'25%'` | More | Compensate with more machines |
|
||||
|
||||
| Aspect | Workers (in-process) | Shards (across machines) |
|
||||
| -------------- | ------------------------- | -------------------------- |
|
||||
| What it splits | Tests across CPU cores | Test files across CI jobs |
|
||||
| Controlled by | Config or `--workers` CLI | `--shard=X/Y` CLI flag |
|
||||
| Shares memory | Yes | No |
|
||||
| Report merging | Not needed | Required (`merge-reports`) |
|
||||
| Cost | Free (same machine) | More CI minutes |
|
||||
|
||||
## Anti-Patterns
|
||||
|
||||
| Anti-Pattern | Problem | Solution |
|
||||
| --------------------------------------- | ---------------------------------------- | ---------------------------------------------------- |
|
||||
| `fullyParallel: false` without reason | Tests in files run serially | Set `fullyParallel: true` unless tests need serial |
|
||||
| `workers: 1` in CI "for safety" | Negates parallelism | Fix isolation issues; use `workers: '50%'` |
|
||||
| Hardcoded shared user account | Race conditions in parallel runs | Each test creates unique data |
|
||||
| Sharding without blob reporter | Each shard produces separate HTML report | Configure `reporter: [['blob']]` for CI |
|
||||
| Sharding with 3 tests | Setup overhead exceeds time saved | Only shard when suite > 5 minutes |
|
||||
| `test.describe.serial()` everywhere | Kills parallelism, creates dependencies | Use only when tests genuinely need prior state |
|
||||
| Workers > CPU cores | Context switching overhead | Use `'50%'` or auto-detect |
|
||||
| Missing `fail-fast: false` in CI matrix | One shard failure cancels others | Always set `fail-fast: false` for sharded strategies |
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Tests pass solo but fail together
|
||||
|
||||
- **Shared state**. Make test data unique:
|
||||
```ts
|
||||
test("create item", async ({ request }, ti) => {
|
||||
await request.post("/api/items", {
|
||||
data: { name: `Item-${ti.workerIndex}-${Date.now()}` },
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
### "No tests found" in some shards
|
||||
|
||||
- **Too many shards**. Never exceed file count:
|
||||
```bash
|
||||
npx playwright test --shard=1/10 # ok if 10 files
|
||||
npx playwright test --shard=1/20 # too many, some shards empty
|
||||
```
|
||||
|
||||
### Merged report missing results
|
||||
|
||||
- **Blob reports collide**. Use unique names:
|
||||
```yaml
|
||||
# Each shard
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: blob-report-${{ strategy.job-index }}
|
||||
path: blob-report/
|
||||
# Merge step
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
pattern: blob-report-*
|
||||
merge-multiple: true
|
||||
path: all-blob-reports
|
||||
```
|
||||
|
||||
### Worker-scoped fixture not working
|
||||
|
||||
- **Missing `{ scope: 'worker' }`**. Fix:
|
||||
```ts
|
||||
export const test = base.extend({
|
||||
resource: [
|
||||
async ({}, use) => {
|
||||
const r = await Resource.create();
|
||||
await use(r);
|
||||
await r.destroy();
|
||||
},
|
||||
{ scope: "worker" },
|
||||
],
|
||||
});
|
||||
```
|
||||
|
||||
### More workers = Slower
|
||||
|
||||
- **Too many workers thrash**. Limit in CI:
|
||||
```ts
|
||||
export default defineConfig({
|
||||
workers: process.env.CI ? 2 : undefined,
|
||||
});
|
||||
```
|
||||
|
|
@ -0,0 +1,453 @@
|
|||
# Performance & Parallelization
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [Parallel Execution](#parallel-execution)
|
||||
2. [Sharding](#sharding)
|
||||
3. [Test Optimization](#test-optimization)
|
||||
4. [Network Optimization](#network-optimization)
|
||||
5. [Isolation and Parallel Execution](#isolation-and-parallel-execution)
|
||||
6. [Resource Management](#resource-management)
|
||||
7. [Benchmarking](#benchmarking)
|
||||
|
||||
## Parallel Execution
|
||||
|
||||
### Configuration
|
||||
|
||||
```typescript
|
||||
// playwright.config.ts
|
||||
export default defineConfig({
|
||||
// Run test files in parallel
|
||||
fullyParallel: true,
|
||||
|
||||
// Number of worker processes
|
||||
workers: process.env.CI ? 1 : undefined, // undefined = half CPU cores
|
||||
|
||||
// Or explicit count
|
||||
// workers: 4,
|
||||
// workers: '50%', // Percentage of CPU cores
|
||||
});
|
||||
```
|
||||
|
||||
### Serial Execution When Needed
|
||||
|
||||
```typescript
|
||||
// Entire file serial
|
||||
test.describe.configure({ mode: "serial" });
|
||||
|
||||
test.describe("Sequential Tests", () => {
|
||||
test("first", async ({ page }) => {
|
||||
// Runs first
|
||||
});
|
||||
|
||||
test("second", async ({ page }) => {
|
||||
// Runs after first
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
```typescript
|
||||
// Single describe block serial
|
||||
test.describe("Parallel Tests", () => {
|
||||
test("a", async () => {}); // Parallel
|
||||
test("b", async () => {}); // Parallel
|
||||
});
|
||||
|
||||
test.describe.serial("Serial Tests", () => {
|
||||
test("c", async () => {}); // Serial
|
||||
test("d", async () => {}); // Serial
|
||||
});
|
||||
```
|
||||
|
||||
### Parallel Projects
|
||||
|
||||
```typescript
|
||||
// playwright.config.ts
|
||||
export default defineConfig({
|
||||
projects: [
|
||||
{ name: "chromium", use: { ...devices["Desktop Chrome"] } },
|
||||
{ name: "firefox", use: { ...devices["Desktop Firefox"] } },
|
||||
{ name: "webkit", use: { ...devices["Desktop Safari"] } },
|
||||
],
|
||||
});
|
||||
```
|
||||
|
||||
```bash
|
||||
# Run all projects in parallel
|
||||
npx playwright test
|
||||
|
||||
# Run specific project
|
||||
npx playwright test --project=chromium
|
||||
```
|
||||
|
||||
## Sharding
|
||||
|
||||
### Basic Sharding
|
||||
|
||||
```bash
|
||||
# Split tests across 4 machines
|
||||
# Machine 1:
|
||||
npx playwright test --shard=1/4
|
||||
|
||||
# Machine 2:
|
||||
npx playwright test --shard=2/4
|
||||
|
||||
# Machine 3:
|
||||
npx playwright test --shard=3/4
|
||||
|
||||
# Machine 4:
|
||||
npx playwright test --shard=4/4
|
||||
```
|
||||
|
||||
### Sharding Strategy
|
||||
|
||||
Tests are distributed evenly by file. For optimal sharding:
|
||||
|
||||
- Keep test files similar in size
|
||||
- Use `fullyParallel: true` for even distribution
|
||||
- Balance slow tests across files
|
||||
|
||||
### CI Sharding Pattern
|
||||
|
||||
```yaml
|
||||
# GitHub Actions
|
||||
jobs:
|
||||
test:
|
||||
strategy:
|
||||
matrix:
|
||||
shard: [1, 2, 3, 4]
|
||||
steps:
|
||||
- run: npx playwright test --shard=${{ matrix.shard }}/4
|
||||
```
|
||||
|
||||
> **For comprehensive CI sharding** (blob reports, merging sharded results, full workflows), see [ci-cd.md](ci-cd.md#sharding).
|
||||
|
||||
## Test Optimization
|
||||
|
||||
### Reuse Authentication
|
||||
|
||||
Avoid logging in for every test. Use setup projects with storage state to authenticate once and reuse the session.
|
||||
|
||||
> **For authentication patterns** (storage state, multiple auth states, setup projects), see [fixtures-hooks.md](fixtures-hooks.md#authentication-patterns).
|
||||
|
||||
### Reuse Page State (serial only — trade-off with isolation)
|
||||
|
||||
Sharing a single page/context across tests with `beforeAll`/`afterAll` is **not recommended** for most suites: it breaks test isolation, causes state leak between tests, and makes failures harder to debug. Prefer a fresh `page` per test (Playwright default). Use shared page only when you explicitly need serial execution and accept no isolation.
|
||||
|
||||
```typescript
|
||||
// ⚠️ Serial only, no isolation: state from one test leaks into the next.
|
||||
// Prefer test.describe.configure({ mode: 'serial' }) + fresh page per test, or beforeEach + page.goto().
|
||||
test.describe.configure({ mode: "serial" });
|
||||
test.describe("Dashboard", () => {
|
||||
let page: Page;
|
||||
|
||||
test.beforeAll(async ({ browser }) => {
|
||||
const context = await browser.newContext({
|
||||
storageState: ".auth/user.json",
|
||||
});
|
||||
page = await context.newPage();
|
||||
await page.goto("/dashboard");
|
||||
});
|
||||
|
||||
test.afterAll(async () => {
|
||||
await page?.close();
|
||||
});
|
||||
|
||||
test("shows stats", async () => {
|
||||
await expect(page.getByTestId("stats")).toBeVisible();
|
||||
});
|
||||
|
||||
test("shows chart", async () => {
|
||||
await expect(page.getByTestId("chart")).toBeVisible();
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
### Lazy Navigation
|
||||
|
||||
```typescript
|
||||
// Bad: Navigate in every test
|
||||
test("check header", async ({ page }) => {
|
||||
await page.goto("/products");
|
||||
await expect(page.getByRole("heading")).toBeVisible();
|
||||
});
|
||||
|
||||
test("check footer", async ({ page }) => {
|
||||
await page.goto("/products");
|
||||
await expect(page.getByRole("contentinfo")).toBeVisible();
|
||||
});
|
||||
|
||||
// Good: Share navigation
|
||||
test.describe("Products Page", () => {
|
||||
test.beforeEach(async ({ page }) => {
|
||||
await page.goto("/products");
|
||||
});
|
||||
|
||||
test("check header", async ({ page }) => {
|
||||
await expect(page.getByRole("heading")).toBeVisible();
|
||||
});
|
||||
|
||||
test("check footer", async ({ page }) => {
|
||||
await expect(page.getByRole("contentinfo")).toBeVisible();
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
### Skip Unnecessary Setup
|
||||
|
||||
```typescript
|
||||
// Use test.skip for conditional execution
|
||||
test("admin feature", async ({ page }) => {
|
||||
test.skip(!process.env.ADMIN_ENABLED, "Admin features disabled");
|
||||
// ...
|
||||
});
|
||||
|
||||
// Use test.fixme for known broken tests
|
||||
test.fixme("broken feature", async ({ page }) => {
|
||||
// Skipped but tracked
|
||||
});
|
||||
```
|
||||
|
||||
## Network Optimization
|
||||
|
||||
### Mock APIs
|
||||
|
||||
```typescript
|
||||
test.beforeEach(async ({ page }) => {
|
||||
// Mock slow/heavy endpoints
|
||||
await page.route("**/api/analytics", (route) =>
|
||||
route.fulfill({ json: { views: 1000 } }),
|
||||
);
|
||||
|
||||
await page.route("**/api/recommendations", (route) =>
|
||||
route.fulfill({ json: [] }),
|
||||
);
|
||||
});
|
||||
```
|
||||
|
||||
### Block Unnecessary Resources
|
||||
|
||||
```typescript
|
||||
test.beforeEach(async ({ page }) => {
|
||||
// Block analytics, ads, tracking
|
||||
await page.route("**/*", (route) => {
|
||||
const url = route.request().url();
|
||||
if (
|
||||
url.includes("google-analytics") ||
|
||||
url.includes("facebook") ||
|
||||
url.includes("hotjar")
|
||||
) {
|
||||
return route.abort();
|
||||
}
|
||||
return route.continue();
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
### Block Resource Types
|
||||
|
||||
```typescript
|
||||
// Block images and fonts for faster tests
|
||||
await page.route("**/*", (route) => {
|
||||
const resourceType = route.request().resourceType();
|
||||
if (["image", "font", "stylesheet"].includes(resourceType)) {
|
||||
return route.abort();
|
||||
}
|
||||
return route.continue();
|
||||
});
|
||||
```
|
||||
|
||||
### Cache API Responses
|
||||
|
||||
```typescript
|
||||
const apiCache = new Map<string, object>();
|
||||
|
||||
test.beforeEach(async ({ page }) => {
|
||||
await page.route("**/api/**", async (route) => {
|
||||
const url = route.request().url();
|
||||
|
||||
if (apiCache.has(url)) {
|
||||
return route.fulfill({ json: apiCache.get(url) });
|
||||
}
|
||||
|
||||
const response = await route.fetch();
|
||||
const json = await response.json();
|
||||
apiCache.set(url, json);
|
||||
return route.fulfill({ json });
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
## Isolation and Parallel Execution
|
||||
|
||||
### Default: one context per test
|
||||
|
||||
Playwright gives each test its own browser context (and page). That gives isolation: no shared cookies, storage, or DOM between tests, so failures don’t carry over and you can run tests in any order or in parallel. Keep this default unless you have a clear reason to share state.
|
||||
|
||||
### Avoiding state leak in parallel runs
|
||||
|
||||
- **Do not** rely on shared mutable state (e.g. a single `page` or `context` in `beforeAll`) when tests can run in parallel. State from one test can leak into another and cause flaky, order-dependent failures.
|
||||
- Use **fixtures** for setup/teardown and **`beforeEach`** for per-test navigation so each test gets a fresh page or a clean slate.
|
||||
- For **backend or DB state** shared across tests, isolate per worker so parallel workers don’t collide. Use a worker-scoped fixture and `testInfo.workerIndex` (or `process.env.TEST_WORKER_INDEX`) to create unique data per worker (e.g. unique user or DB prefix). See [fixtures-hooks.md](../core/fixtures-hooks.md) for worker-scoped fixtures and [debugging.md](../debugging/debugging.md) for debugging flaky parallel runs.
|
||||
|
||||
### Debugging flaky parallel runs
|
||||
|
||||
If a test is flaky only with multiple workers:
|
||||
|
||||
1. **Reproduce**: Run with default workers and `--repeat-each=10` (or `--repeat-each=100 --max-failures=1`).
|
||||
2. **Confirm parallel-specific**: Run with `--workers=1`. If the failure disappears, the cause is likely shared state or non-isolated backend/DB data.
|
||||
3. **Fix**: Remove shared page/context; use per-test fixtures and `beforeEach`; isolate test data per worker with `workerIndex` in a worker-scoped fixture.
|
||||
|
||||
Workers are restarted after a test failure so subsequent tests in that worker get a clean environment; fixing isolation still prevents the initial flakiness.
|
||||
|
||||
## Resource Management
|
||||
|
||||
### Browser Contexts
|
||||
|
||||
```typescript
|
||||
// Recommended: One context per test (default) — full isolation
|
||||
test("isolated test", async ({ page }) => {
|
||||
// Fresh context automatically
|
||||
});
|
||||
|
||||
// Manual context for specific needs
|
||||
test("multiple tabs", async ({ browser }) => {
|
||||
const context = await browser.newContext();
|
||||
const page1 = await context.newPage();
|
||||
const page2 = await context.newPage();
|
||||
|
||||
// Clean up
|
||||
await context.close();
|
||||
});
|
||||
```
|
||||
|
||||
### Memory Management
|
||||
|
||||
```typescript
|
||||
// playwright.config.ts
|
||||
export default defineConfig({
|
||||
// Limit concurrent workers
|
||||
workers: 2,
|
||||
|
||||
// Limit parallel tests per worker
|
||||
use: {
|
||||
// Lower memory usage
|
||||
launchOptions: {
|
||||
args: ["--disable-dev-shm-usage"],
|
||||
},
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
### Timeouts
|
||||
|
||||
```typescript
|
||||
// playwright.config.ts
|
||||
export default defineConfig({
|
||||
// Global test timeout
|
||||
timeout: 30000,
|
||||
|
||||
// Assertion timeout
|
||||
expect: {
|
||||
timeout: 5000,
|
||||
},
|
||||
|
||||
// Navigation timeout
|
||||
use: {
|
||||
navigationTimeout: 15000,
|
||||
actionTimeout: 10000,
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
## Benchmarking
|
||||
|
||||
### Measure Test Duration
|
||||
|
||||
```typescript
|
||||
test("performance test", async ({ page }, testInfo) => {
|
||||
const startTime = Date.now();
|
||||
|
||||
await page.goto("/");
|
||||
|
||||
const loadTime = Date.now() - startTime;
|
||||
console.log(`Page load: ${loadTime}ms`);
|
||||
|
||||
// Add to test report
|
||||
testInfo.annotations.push({
|
||||
type: "performance",
|
||||
description: `Load time: ${loadTime}ms`,
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
### Performance Metrics
|
||||
|
||||
```typescript
|
||||
test("collect metrics", async ({ page }) => {
|
||||
await page.goto("/");
|
||||
|
||||
const metrics = await page.evaluate(() => ({
|
||||
// Navigation timing
|
||||
loadTime:
|
||||
performance.timing.loadEventEnd - performance.timing.navigationStart,
|
||||
domContentLoaded:
|
||||
performance.timing.domContentLoadedEventEnd -
|
||||
performance.timing.navigationStart,
|
||||
|
||||
// Performance entries
|
||||
resources: performance.getEntriesByType("resource").length,
|
||||
|
||||
// Memory (Chrome only)
|
||||
// @ts-ignore
|
||||
memory: performance.memory?.usedJSHeapSize,
|
||||
}));
|
||||
|
||||
console.log("Metrics:", metrics);
|
||||
expect(metrics.loadTime).toBeLessThan(3000);
|
||||
});
|
||||
```
|
||||
|
||||
### Lighthouse Integration
|
||||
|
||||
```typescript
|
||||
import { playAudit } from "playwright-lighthouse";
|
||||
|
||||
test("lighthouse audit", async ({ page }) => {
|
||||
await page.goto("/");
|
||||
|
||||
const audit = await playAudit({
|
||||
page,
|
||||
thresholds: {
|
||||
performance: 80,
|
||||
accessibility: 90,
|
||||
"best-practices": 80,
|
||||
seo: 80,
|
||||
},
|
||||
port: 9222,
|
||||
});
|
||||
|
||||
expect(audit.lhr.categories.performance.score * 100).toBeGreaterThanOrEqual(
|
||||
80,
|
||||
);
|
||||
});
|
||||
```
|
||||
|
||||
## Performance Checklist
|
||||
|
||||
| Optimization | Impact |
|
||||
| ------------------------------ | ---------- |
|
||||
| Enable `fullyParallel` | High |
|
||||
| Reuse authentication | High |
|
||||
| Mock heavy APIs | High |
|
||||
| Block tracking scripts | Medium |
|
||||
| Use sharding in CI | High |
|
||||
| Reduce workers if memory-bound | Medium |
|
||||
| Cache API responses | Medium |
|
||||
| Skip unnecessary tests | Low-Medium |
|
||||
|
||||
## Related References
|
||||
|
||||
- **CI/CD sharding**: See [ci-cd.md](ci-cd.md) for CI configuration
|
||||
- **Test organization**: See [test-suite-structure.md](../core/test-suite-structure.md) for structuring tests
|
||||
- **Fixtures for reuse**: See [fixtures-hooks.md](../core/fixtures-hooks.md) for authentication patterns
|
||||
|
|
@ -0,0 +1,424 @@
|
|||
# Test Reports & Artifacts
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [CLI Commands](#cli-commands)
|
||||
2. [Reporter Configuration](#reporter-configuration)
|
||||
3. [Custom Reporter](#custom-reporter)
|
||||
4. [Trace Configuration](#trace-configuration)
|
||||
5. [Screenshot & Video Settings](#screenshot--video-settings)
|
||||
6. [Artifact Directory Structure](#artifact-directory-structure)
|
||||
7. [CI Artifact Upload](#ci-artifact-upload)
|
||||
8. [Decision Guide](#decision-guide)
|
||||
9. [Anti-Patterns](#anti-patterns)
|
||||
10. [Troubleshooting](#troubleshooting)
|
||||
|
||||
> **When to use**: Configuring test output for debugging, CI dashboards, and team visibility.
|
||||
|
||||
## CLI Commands
|
||||
|
||||
```bash
|
||||
# Display last HTML report
|
||||
npx playwright show-report
|
||||
|
||||
# Specify reporter
|
||||
npx playwright test --reporter=html
|
||||
npx playwright test --reporter=dot # minimal CI output
|
||||
npx playwright test --reporter=line # one line per test
|
||||
npx playwright test --reporter=json # machine-readable
|
||||
npx playwright test --reporter=junit # CI integration
|
||||
|
||||
# Combine reporters
|
||||
npx playwright test --reporter=dot,html
|
||||
|
||||
# Merge sharded reports
|
||||
npx playwright merge-reports --reporter=html ./blob-report
|
||||
```
|
||||
|
||||
## Reporter Configuration
|
||||
|
||||
### Environment-Based Setup
|
||||
|
||||
```typescript
|
||||
// playwright.config.ts
|
||||
import { defineConfig } from '@playwright/test';
|
||||
|
||||
export default defineConfig({
|
||||
reporter: process.env.CI
|
||||
? [
|
||||
['dot'],
|
||||
['html', { open: 'never' }],
|
||||
['junit', { outputFile: 'results/junit.xml' }],
|
||||
['github'],
|
||||
]
|
||||
: [
|
||||
['list'],
|
||||
['html', { open: 'on-failure' }],
|
||||
],
|
||||
});
|
||||
```
|
||||
|
||||
### Reporter Types
|
||||
|
||||
| Reporter | Output | Use Case |
|
||||
|---|---|---|
|
||||
| `list` | One line per test | Local development |
|
||||
| `line` | Single updating line | Local, less verbose |
|
||||
| `dot` | `.` pass, `F` fail | CI logs |
|
||||
| `html` | Interactive HTML page | Post-run analysis |
|
||||
| `json` | Machine-readable JSON | Custom tooling |
|
||||
| `junit` | JUnit XML | CI platforms |
|
||||
| `github` | PR annotations | GitHub Actions |
|
||||
| `blob` | Binary archive | Shard merging |
|
||||
|
||||
### JSON Output to File
|
||||
|
||||
```typescript
|
||||
import { defineConfig } from '@playwright/test';
|
||||
|
||||
export default defineConfig({
|
||||
reporter: [
|
||||
['json', { outputFile: 'results/output.json' }],
|
||||
],
|
||||
});
|
||||
```
|
||||
|
||||
### JUnit Customization
|
||||
|
||||
```typescript
|
||||
import { defineConfig } from '@playwright/test';
|
||||
|
||||
export default defineConfig({
|
||||
reporter: [
|
||||
['junit', {
|
||||
outputFile: 'results/junit.xml',
|
||||
stripANSIControlSequences: true,
|
||||
includeProjectInTestName: true,
|
||||
}],
|
||||
],
|
||||
});
|
||||
```
|
||||
|
||||
## Custom Reporter
|
||||
|
||||
Build custom reporters for Slack notifications, database logging, or dashboards.
|
||||
|
||||
```typescript
|
||||
// reporters/notification-reporter.ts
|
||||
import type {
|
||||
FullResult,
|
||||
Reporter,
|
||||
TestCase,
|
||||
TestResult,
|
||||
} from '@playwright/test/reporter';
|
||||
|
||||
class NotificationReporter implements Reporter {
|
||||
private passed = 0;
|
||||
private failed = 0;
|
||||
private skipped = 0;
|
||||
private failures: string[] = [];
|
||||
|
||||
onTestEnd(test: TestCase, result: TestResult) {
|
||||
switch (result.status) {
|
||||
case 'passed':
|
||||
this.passed++;
|
||||
break;
|
||||
case 'failed':
|
||||
case 'timedOut':
|
||||
this.failed++;
|
||||
this.failures.push(`${test.title}: ${result.error?.message?.split('\n')[0]}`);
|
||||
break;
|
||||
case 'skipped':
|
||||
this.skipped++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
async onEnd(result: FullResult) {
|
||||
const total = this.passed + this.failed + this.skipped;
|
||||
const status = this.failed > 0 ? 'FAILED' : 'PASSED';
|
||||
const message = [
|
||||
`Tests ${status}`,
|
||||
`Passed: ${this.passed} | Failed: ${this.failed} | Skipped: ${this.skipped}`,
|
||||
`Duration: ${(result.duration / 1000).toFixed(1)}s`,
|
||||
];
|
||||
|
||||
if (this.failures.length > 0) {
|
||||
message.push('', 'Failures:');
|
||||
this.failures.slice(0, 5).forEach((f) => message.push(` - ${f}`));
|
||||
if (this.failures.length > 5) {
|
||||
message.push(` ...and ${this.failures.length - 5} more`);
|
||||
}
|
||||
}
|
||||
|
||||
const webhookUrl = process.env.NOTIFICATION_WEBHOOK;
|
||||
if (webhookUrl) {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), 5000);
|
||||
try {
|
||||
await fetch(webhookUrl, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ text: message.join('\n') }),
|
||||
signal: controller.signal,
|
||||
});
|
||||
} catch (error) {
|
||||
// Intentionally swallow notifier failures to avoid blocking test completion
|
||||
console.warn('Webhook notification failed:', error.message);
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default NotificationReporter;
|
||||
```
|
||||
|
||||
**Register custom reporter:**
|
||||
|
||||
```typescript
|
||||
import { defineConfig } from '@playwright/test';
|
||||
|
||||
export default defineConfig({
|
||||
reporter: [
|
||||
['dot'],
|
||||
['html', { open: 'never' }],
|
||||
['./reporters/notification-reporter.ts'],
|
||||
],
|
||||
});
|
||||
```
|
||||
|
||||
## Trace Configuration
|
||||
|
||||
Traces capture actions, network requests, DOM snapshots, and console logs.
|
||||
|
||||
```typescript
|
||||
import { defineConfig } from '@playwright/test';
|
||||
|
||||
export default defineConfig({
|
||||
retries: process.env.CI ? 2 : 0,
|
||||
use: {
|
||||
trace: 'on-first-retry',
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
### Trace Options
|
||||
|
||||
| Value | Behavior | Overhead |
|
||||
|---|---|---|
|
||||
| `'off'` | Never records | None |
|
||||
| `'on'` | Every test | High |
|
||||
| `'on-first-retry'` | On first retry after failure | Minimal |
|
||||
| `'retain-on-failure'` | Records all, keeps failures | Medium |
|
||||
| `'retain-on-first-failure'` | Records all, keeps first failure | Medium |
|
||||
|
||||
### Viewing Traces
|
||||
|
||||
```bash
|
||||
# Local trace viewer
|
||||
npx playwright show-trace results/my-test/trace.zip
|
||||
|
||||
# From HTML report (click Traces tab)
|
||||
npx playwright show-report
|
||||
|
||||
# Online viewer: https://trace.playwright.dev
|
||||
```
|
||||
|
||||
## Screenshot & Video Settings
|
||||
|
||||
```typescript
|
||||
import { defineConfig } from '@playwright/test';
|
||||
|
||||
export default defineConfig({
|
||||
use: {
|
||||
screenshot: 'only-on-failure',
|
||||
video: 'retain-on-failure',
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
### Video with Custom Size
|
||||
|
||||
```typescript
|
||||
use: {
|
||||
video: {
|
||||
mode: 'retain-on-failure',
|
||||
size: { width: 1280, height: 720 },
|
||||
},
|
||||
},
|
||||
```
|
||||
|
||||
### Screenshot Options
|
||||
|
||||
| Value | Captures | Disk Cost |
|
||||
|---|---|---|
|
||||
| `'off'` | Never | None |
|
||||
| `'on'` | Every test | High |
|
||||
| `'only-on-failure'` | Failed tests | Low |
|
||||
|
||||
### Video Options
|
||||
|
||||
| Value | Records | Keeps | Disk Cost |
|
||||
|---|---|---|---|
|
||||
| `'off'` | Never | — | None |
|
||||
| `'on'` | Every test | All | Very high |
|
||||
| `'on-first-retry'` | On retry | Retried | Low |
|
||||
| `'retain-on-failure'` | Every test | Failed | Medium |
|
||||
|
||||
## Artifact Directory Structure
|
||||
|
||||
```text
|
||||
test-results/
|
||||
├── checkout-test-chromium/
|
||||
│ ├── trace.zip
|
||||
│ ├── test-failed-1.png
|
||||
│ └── video.webm
|
||||
├── login-test-firefox/
|
||||
│ ├── trace.zip
|
||||
│ └── test-failed-1.png
|
||||
└── junit.xml
|
||||
|
||||
playwright-report/
|
||||
├── index.html
|
||||
└── data/
|
||||
|
||||
blob-report/
|
||||
└── report-1.zip
|
||||
```
|
||||
|
||||
## CI Artifact Upload
|
||||
|
||||
### GitHub Actions
|
||||
|
||||
```yaml
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: ${{ !cancelled() }}
|
||||
with:
|
||||
name: playwright-report
|
||||
path: playwright-report/
|
||||
retention-days: 14
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: failure()
|
||||
with:
|
||||
name: test-traces
|
||||
path: |
|
||||
test-results/**/trace.zip
|
||||
test-results/**/*.png
|
||||
test-results/**/*.webm
|
||||
retention-days: 7
|
||||
```
|
||||
|
||||
## Decision Guide
|
||||
|
||||
| Scenario | Reporter Configuration |
|
||||
|---|---|
|
||||
| Local development | `[['list'], ['html', { open: 'on-failure' }]]` |
|
||||
| GitHub Actions | `[['dot'], ['html'], ['github']]` |
|
||||
| GitLab CI | `[['dot'], ['html'], ['junit']]` |
|
||||
| Azure DevOps / Jenkins | `[['dot'], ['html'], ['junit']]` |
|
||||
| Sharded CI | `[['blob'], ['github']]` |
|
||||
| Custom dashboard | `[['json', { outputFile: '...' }]]` + custom reporter |
|
||||
|
||||
| Artifact | When to Collect | Retention | Upload Condition |
|
||||
|---|---|---|---|
|
||||
| HTML report | Always | 14 days | `if: ${{ !cancelled() }}` |
|
||||
| Traces | On failure | 7 days | `if: failure()` |
|
||||
| Screenshots | On failure | 7 days | `if: failure()` |
|
||||
| Videos | On failure | 7 days | `if: failure()` |
|
||||
| JUnit XML | Always | 14 days | `if: ${{ !cancelled() }}` |
|
||||
| Blob report | Always (sharded) | 1 day | `if: ${{ !cancelled() }}` |
|
||||
|
||||
## Anti-Patterns
|
||||
|
||||
| Anti-Pattern | Problem | Solution |
|
||||
|---|---|---|
|
||||
| No reporter configured | Default `list` only; no persistent report | Configure `html` + CI reporter |
|
||||
| `trace: 'on'` in CI | Massive artifacts, slow uploads | Use `trace: 'on-first-retry'` |
|
||||
| `video: 'on'` in CI | Enormous storage, slower tests | Use `video: 'retain-on-failure'` |
|
||||
| Upload artifacts only on failure | No report when tests pass | Upload with `if: ${{ !cancelled() }}` |
|
||||
| No retention limits | CI storage fills quickly | Set `retention-days: 7-14` |
|
||||
| Only `dot` reporter | Cannot drill into failures | Pair `dot` with `html` |
|
||||
| JUnit to stdout | Interferes with console output | Write to file |
|
||||
| Blocking `onEnd` in custom reporter | Slow HTTP calls delay pipeline | Use `Promise.race` with timeout |
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Empty HTML Report
|
||||
|
||||
Check reporter config. HTML report defaults to `playwright-report/`:
|
||||
|
||||
```typescript
|
||||
import { defineConfig } from '@playwright/test';
|
||||
|
||||
export default defineConfig({
|
||||
reporter: [['html', { outputFolder: 'playwright-report', open: 'never' }]],
|
||||
});
|
||||
```
|
||||
|
||||
### Traces Too Large
|
||||
|
||||
Switch from `trace: 'on'` to `'on-first-retry'` with retries enabled:
|
||||
|
||||
```typescript
|
||||
import { defineConfig } from '@playwright/test';
|
||||
|
||||
export default defineConfig({
|
||||
retries: process.env.CI ? 2 : 0,
|
||||
use: {
|
||||
trace: 'on-first-retry',
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
### JUnit XML Not Recognized
|
||||
|
||||
Ensure path matches CI configuration:
|
||||
|
||||
```typescript
|
||||
reporter: [['junit', { outputFile: 'results/junit.xml' }]],
|
||||
```
|
||||
|
||||
```yaml
|
||||
# GitHub Actions
|
||||
- uses: dorny/test-reporter@latest
|
||||
with:
|
||||
path: results/junit.xml
|
||||
reporter: java-junit
|
||||
|
||||
# Azure DevOps
|
||||
- task: PublishTestResults@latest
|
||||
inputs:
|
||||
testResultsFiles: 'results/junit.xml'
|
||||
|
||||
# Jenkins
|
||||
junit 'results/junit.xml'
|
||||
```
|
||||
|
||||
### Empty Merged Report
|
||||
|
||||
Use `blob` reporter for sharded runs (not `html`):
|
||||
|
||||
```typescript
|
||||
import { defineConfig } from '@playwright/test';
|
||||
|
||||
export default defineConfig({
|
||||
reporter: process.env.CI
|
||||
? [['blob'], ['dot']]
|
||||
: [['html', { open: 'on-failure' }]],
|
||||
});
|
||||
```
|
||||
|
||||
### Missing Screenshots in Report
|
||||
|
||||
Enable screenshots and keep both directories:
|
||||
|
||||
```typescript
|
||||
use: {
|
||||
screenshot: 'only-on-failure',
|
||||
},
|
||||
```
|
||||
|
||||
The HTML report embeds screenshots from `test-results/`. Deleting that directory removes screenshots from the report.
|
||||
|
|
@ -0,0 +1,497 @@
|
|||
# Test Coverage
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [Coverage Setup](#coverage-setup)
|
||||
2. [Collecting Coverage](#collecting-coverage)
|
||||
3. [Coverage Reports](#coverage-reports)
|
||||
4. [Coverage Thresholds](#coverage-thresholds)
|
||||
5. [Advanced Patterns](#advanced-patterns)
|
||||
6. [CI Integration](#ci-integration)
|
||||
|
||||
## Coverage Setup
|
||||
|
||||
### Install Dependencies
|
||||
|
||||
```bash
|
||||
# For V8 coverage (built into Playwright)
|
||||
# No additional dependencies needed
|
||||
|
||||
# For Istanbul-based coverage (more features)
|
||||
npm install -D nyc @istanbuljs/nyc-config-typescript
|
||||
```
|
||||
|
||||
### Basic Configuration
|
||||
|
||||
```typescript
|
||||
// playwright.config.ts
|
||||
import { defineConfig } from "@playwright/test";
|
||||
|
||||
export default defineConfig({
|
||||
use: {
|
||||
// Enable coverage collection
|
||||
contextOptions: {
|
||||
// V8 coverage is automatic with the API below
|
||||
},
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
### V8 Coverage Fixture
|
||||
|
||||
```typescript
|
||||
// fixtures/coverage.ts
|
||||
import { test as base, expect } from "@playwright/test";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { randomUUID } from "crypto";
|
||||
|
||||
export const test = base.extend<{}, { collectCoverage: void }>({
|
||||
collectCoverage: [
|
||||
async ({ browser }, use) => {
|
||||
// Start coverage for all pages
|
||||
const context = await browser.newContext();
|
||||
const page = await context.newPage();
|
||||
|
||||
await page.coverage.startJSCoverage();
|
||||
await page.coverage.startCSSCoverage();
|
||||
|
||||
await use();
|
||||
|
||||
// Collect coverage
|
||||
const [jsCoverage, cssCoverage] = await Promise.all([
|
||||
page.coverage.stopJSCoverage(),
|
||||
page.coverage.stopCSSCoverage(),
|
||||
]);
|
||||
|
||||
// Save coverage data
|
||||
const coverageDir = "./coverage";
|
||||
if (!fs.existsSync(coverageDir)) {
|
||||
fs.mkdirSync(coverageDir, { recursive: true });
|
||||
}
|
||||
|
||||
fs.writeFileSync(
|
||||
path.join(coverageDir, `coverage-${randomUUID()}.json`),
|
||||
JSON.stringify([...jsCoverage, ...cssCoverage])
|
||||
);
|
||||
|
||||
await context.close();
|
||||
},
|
||||
{ scope: "worker", auto: true },
|
||||
],
|
||||
});
|
||||
```
|
||||
|
||||
## Collecting Coverage
|
||||
|
||||
### Per-Test Coverage
|
||||
|
||||
```typescript
|
||||
test("collect coverage for single test", async ({ page }) => {
|
||||
// Start coverage collection
|
||||
await page.coverage.startJSCoverage({
|
||||
resetOnNavigation: false,
|
||||
});
|
||||
|
||||
// Run test
|
||||
await page.goto("/app");
|
||||
await page.getByRole("button", { name: "Submit" }).click();
|
||||
await expect(page.getByText("Success")).toBeVisible();
|
||||
|
||||
// Stop and get coverage
|
||||
const coverage = await page.coverage.stopJSCoverage();
|
||||
|
||||
// Filter to only your source files
|
||||
const appCoverage = coverage.filter((entry) => entry.url.includes("/src/"));
|
||||
|
||||
console.log(`Covered ${appCoverage.length} source files`);
|
||||
});
|
||||
```
|
||||
|
||||
### Coverage for Specific Files
|
||||
|
||||
```typescript
|
||||
test("track specific module coverage", async ({ page }) => {
|
||||
await page.coverage.startJSCoverage();
|
||||
|
||||
await page.goto("/checkout");
|
||||
await page.getByRole("button", { name: "Pay" }).click();
|
||||
|
||||
const coverage = await page.coverage.stopJSCoverage();
|
||||
|
||||
// Find coverage for checkout module
|
||||
const checkoutCoverage = coverage.find((c) => c.url.includes("checkout.js"));
|
||||
|
||||
if (checkoutCoverage) {
|
||||
const totalBytes = checkoutCoverage.text?.length || 0;
|
||||
const coveredBytes = checkoutCoverage.ranges.reduce(
|
||||
(sum, range) => sum + (range.end - range.start),
|
||||
0
|
||||
);
|
||||
const percentage = (coveredBytes / totalBytes) * 100;
|
||||
|
||||
console.log(`Checkout module: ${percentage.toFixed(1)}% covered`);
|
||||
expect(percentage).toBeGreaterThan(80);
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
### CSS Coverage
|
||||
|
||||
```typescript
|
||||
test("collect CSS coverage", async ({ page }) => {
|
||||
await page.coverage.startCSSCoverage();
|
||||
|
||||
await page.goto("/app");
|
||||
|
||||
// Interact to trigger different CSS states
|
||||
await page.getByRole("button").hover();
|
||||
await page.getByRole("dialog").waitFor();
|
||||
|
||||
const cssCoverage = await page.coverage.stopCSSCoverage();
|
||||
|
||||
// Find unused CSS
|
||||
for (const entry of cssCoverage) {
|
||||
const totalBytes = entry.text?.length || 0;
|
||||
const usedBytes = entry.ranges.reduce(
|
||||
(sum, range) => sum + (range.end - range.start),
|
||||
0
|
||||
);
|
||||
const unusedPercentage = ((totalBytes - usedBytes) / totalBytes) * 100;
|
||||
|
||||
if (unusedPercentage > 50) {
|
||||
console.warn(`${entry.url}: ${unusedPercentage.toFixed(1)}% unused CSS`);
|
||||
}
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
## Coverage Reports
|
||||
|
||||
### Converting to Istanbul Format
|
||||
|
||||
```typescript
|
||||
// scripts/convert-coverage.ts
|
||||
import { execSync } from "child_process";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import v8ToIstanbul from "v8-to-istanbul";
|
||||
|
||||
async function convertCoverage() {
|
||||
const coverageDir = "./coverage";
|
||||
const files = fs.readdirSync(coverageDir).filter((f) => f.endsWith(".json"));
|
||||
|
||||
const istanbulCoverage: any = {};
|
||||
|
||||
for (const file of files) {
|
||||
const coverageData = JSON.parse(
|
||||
fs.readFileSync(path.join(coverageDir, file), "utf-8")
|
||||
);
|
||||
|
||||
for (const entry of coverageData) {
|
||||
if (!entry.url.startsWith("file://")) continue;
|
||||
|
||||
const filePath = entry.url.replace("file://", "");
|
||||
const converter = v8ToIstanbul(filePath);
|
||||
|
||||
await converter.load();
|
||||
converter.applyCoverage(entry.functions || []);
|
||||
|
||||
const istanbul = converter.toIstanbul();
|
||||
Object.assign(istanbulCoverage, istanbul);
|
||||
}
|
||||
}
|
||||
|
||||
fs.writeFileSync(
|
||||
path.join(coverageDir, "coverage-final.json"),
|
||||
JSON.stringify(istanbulCoverage)
|
||||
);
|
||||
}
|
||||
|
||||
convertCoverage();
|
||||
```
|
||||
|
||||
### Generating HTML Report
|
||||
|
||||
```bash
|
||||
# Using nyc to generate report
|
||||
npx nyc report --reporter=html --reporter=text --temp-dir=./coverage
|
||||
```
|
||||
|
||||
```typescript
|
||||
// package.json scripts
|
||||
{
|
||||
"scripts": {
|
||||
"test": "playwright test",
|
||||
"test:coverage": "playwright test && npm run coverage:report",
|
||||
"coverage:report": "npx nyc report --reporter=html --reporter=lcov --temp-dir=./coverage"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Custom Coverage Reporter
|
||||
|
||||
```typescript
|
||||
// reporters/coverage-reporter.ts
|
||||
import type { Reporter, FullResult } from "@playwright/test/reporter";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
|
||||
class CoverageReporter implements Reporter {
|
||||
private coverageData: any[] = [];
|
||||
|
||||
onEnd(result: FullResult) {
|
||||
// Aggregate all coverage files
|
||||
const coverageDir = "./coverage";
|
||||
const files = fs
|
||||
.readdirSync(coverageDir)
|
||||
.filter((f) => f.endsWith(".json"));
|
||||
|
||||
for (const file of files) {
|
||||
const data = JSON.parse(
|
||||
fs.readFileSync(path.join(coverageDir, file), "utf-8")
|
||||
);
|
||||
this.coverageData.push(...data);
|
||||
}
|
||||
|
||||
// Generate summary
|
||||
const summary = this.generateSummary();
|
||||
console.log("\n📊 Coverage Summary:");
|
||||
console.log(` Files: ${summary.totalFiles}`);
|
||||
console.log(` Lines: ${summary.lineCoverage.toFixed(1)}%`);
|
||||
console.log(` Bytes: ${summary.byteCoverage.toFixed(1)}%`);
|
||||
|
||||
if (summary.lineCoverage < 80) {
|
||||
console.warn("⚠️ Coverage below 80% threshold!");
|
||||
}
|
||||
}
|
||||
|
||||
private generateSummary() {
|
||||
let totalBytes = 0;
|
||||
let coveredBytes = 0;
|
||||
const files = new Set<string>();
|
||||
|
||||
for (const entry of this.coverageData) {
|
||||
if (entry.url.includes("/src/")) {
|
||||
files.add(entry.url);
|
||||
totalBytes += entry.text?.length || 0;
|
||||
coveredBytes += entry.ranges.reduce(
|
||||
(sum: number, r: any) => sum + (r.end - r.start),
|
||||
0
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
totalFiles: files.size,
|
||||
byteCoverage: (coveredBytes / totalBytes) * 100,
|
||||
lineCoverage: (coveredBytes / totalBytes) * 100, // Simplified
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export default CoverageReporter;
|
||||
```
|
||||
|
||||
## Coverage Thresholds
|
||||
|
||||
### Enforcing Minimum Coverage
|
||||
|
||||
```typescript
|
||||
// tests/coverage.spec.ts
|
||||
import { test, expect } from "@playwright/test";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
|
||||
test.afterAll(async () => {
|
||||
const coverageDir = "./coverage";
|
||||
const files = fs.readdirSync(coverageDir).filter((f) => f.endsWith(".json"));
|
||||
|
||||
let totalBytes = 0;
|
||||
let coveredBytes = 0;
|
||||
|
||||
for (const file of files) {
|
||||
const coverage = JSON.parse(
|
||||
fs.readFileSync(path.join(coverageDir, file), "utf-8")
|
||||
);
|
||||
|
||||
for (const entry of coverage) {
|
||||
if (!entry.url.includes("/src/")) continue;
|
||||
totalBytes += entry.text?.length || 0;
|
||||
coveredBytes += entry.ranges.reduce(
|
||||
(sum: number, r: any) => sum + (r.end - r.start),
|
||||
0
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const coveragePercent = (coveredBytes / totalBytes) * 100;
|
||||
|
||||
// Enforce threshold
|
||||
expect(coveragePercent).toBeGreaterThan(80);
|
||||
});
|
||||
```
|
||||
|
||||
### Per-Directory Thresholds
|
||||
|
||||
```typescript
|
||||
// coverage-check.ts
|
||||
interface CoverageThreshold {
|
||||
pattern: RegExp;
|
||||
minCoverage: number;
|
||||
}
|
||||
|
||||
const thresholds: CoverageThreshold[] = [
|
||||
{ pattern: /\/src\/core\//, minCoverage: 90 },
|
||||
{ pattern: /\/src\/utils\//, minCoverage: 85 },
|
||||
{ pattern: /\/src\/components\//, minCoverage: 70 },
|
||||
{ pattern: /\/src\/pages\//, minCoverage: 60 },
|
||||
];
|
||||
|
||||
function checkThresholds(coverage: any[]): string[] {
|
||||
const violations: string[] = [];
|
||||
|
||||
for (const threshold of thresholds) {
|
||||
const matchingFiles = coverage.filter((c) => threshold.pattern.test(c.url));
|
||||
|
||||
let total = 0;
|
||||
let covered = 0;
|
||||
|
||||
for (const file of matchingFiles) {
|
||||
total += file.text?.length || 0;
|
||||
covered += file.ranges.reduce(
|
||||
(sum: number, r: any) => sum + (r.end - r.start),
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
const percent = total > 0 ? (covered / total) * 100 : 0;
|
||||
|
||||
if (percent < threshold.minCoverage) {
|
||||
violations.push(
|
||||
`${threshold.pattern}: ${percent.toFixed(1)}% < ${
|
||||
threshold.minCoverage
|
||||
}%`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return violations;
|
||||
}
|
||||
```
|
||||
|
||||
## Advanced Patterns
|
||||
|
||||
### Merging Coverage Across Shards
|
||||
|
||||
```typescript
|
||||
// scripts/merge-coverage.ts
|
||||
import fs from "fs";
|
||||
import { glob } from "glob";
|
||||
|
||||
async function mergeCoverage() {
|
||||
const files = await glob("shard-*/coverage/*.json");
|
||||
const merged = new Map<string, any>();
|
||||
|
||||
for (const file of files) {
|
||||
const data = JSON.parse(fs.readFileSync(file, "utf-8"));
|
||||
for (const entry of data) {
|
||||
if (merged.has(entry.url)) {
|
||||
const existing = merged.get(entry.url);
|
||||
existing.ranges.push(...entry.ranges);
|
||||
} else {
|
||||
merged.set(entry.url, { ...entry });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fs.writeFileSync(
|
||||
"./coverage/merged.json",
|
||||
JSON.stringify([...merged.values()])
|
||||
);
|
||||
}
|
||||
|
||||
mergeCoverage();
|
||||
```
|
||||
|
||||
### Incremental Coverage
|
||||
|
||||
```typescript
|
||||
// Check coverage only for changed files in CI
|
||||
import { execSync } from "child_process";
|
||||
import fs from "fs";
|
||||
|
||||
const changedFiles = execSync("git diff --name-only HEAD~1")
|
||||
.toString()
|
||||
.split("\n")
|
||||
.filter((f) => f.endsWith(".ts"));
|
||||
|
||||
const coverage = JSON.parse(fs.readFileSync("./coverage/merged.json", "utf-8"));
|
||||
|
||||
for (const file of changedFiles) {
|
||||
const entry = coverage.find((c: any) => c.url.includes(file));
|
||||
if (entry) {
|
||||
const percent =
|
||||
(entry.ranges.reduce((s: number, r: any) => s + r.end - r.start, 0) /
|
||||
(entry.text?.length || 1)) *
|
||||
100;
|
||||
console.log(`${file}: ${percent.toFixed(1)}%`);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## CI Integration
|
||||
|
||||
### GitHub Actions
|
||||
|
||||
```yaml
|
||||
# .github/workflows/test.yml
|
||||
name: Tests with Coverage
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
|
||||
- run: npm ci
|
||||
- run: npx playwright install --with-deps
|
||||
|
||||
- name: Run tests with coverage
|
||||
run: npm run test:coverage
|
||||
|
||||
- name: Upload coverage to Codecov
|
||||
uses: codecov/codecov-action@v3
|
||||
with:
|
||||
files: ./coverage/lcov.info
|
||||
fail_ci_if_error: true
|
||||
|
||||
- name: Check coverage threshold
|
||||
run: |
|
||||
COVERAGE=$(cat coverage/coverage-summary.json | jq '.total.lines.pct')
|
||||
if (( $(echo "$COVERAGE < 80" | bc -l) )); then
|
||||
echo "Coverage $COVERAGE% is below 80% threshold"
|
||||
exit 1
|
||||
fi
|
||||
```
|
||||
|
||||
## Anti-Patterns to Avoid
|
||||
|
||||
| Anti-Pattern | Problem | Solution |
|
||||
| ---------------------------- | -------------------------------------- | --------------------------- |
|
||||
| Coverage for coverage's sake | Gaming metrics | Focus on critical paths |
|
||||
| 100% coverage target | Diminishing returns, tests for getters | Set realistic thresholds |
|
||||
| Ignoring coverage drops | Technical debt | Enforce thresholds in CI |
|
||||
| No source map support | Wrong line numbers | Enable source maps in build |
|
||||
| Coverage only in CI | Late feedback | Run locally too |
|
||||
|
||||
## Related References
|
||||
|
||||
- **CI/CD**: See [ci-cd.md](ci-cd.md) for pipeline configuration
|
||||
- **Performance**: See [performance.md](performance.md) for optimizing coverage collection
|
||||
Loading…
Add table
Add a link
Reference in a new issue