chore: add playwright cursor skill

This commit is contained in:
Anish Sarkar 2026-05-10 04:19:55 +05:30
parent 25aad38ca4
commit d52225c18d
57 changed files with 25244 additions and 0 deletions

View file

@ -0,0 +1,468 @@
# CI/CD Integration
## Table of Contents
1. [GitHub Actions](#github-actions)
2. [Docker](#docker)
3. [Reporting](#reporting)
4. [Sharding](#sharding)
5. [Environment Management](#environment-management)
6. [Caching](#caching)
## GitHub Actions
### Basic Workflow
```yaml
# .github/workflows/playwright.yml
name: Playwright Tests
on:
push:
branches: [main]
pull_request:
branches: [main]
jobs:
test:
timeout-minutes: 60
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 22
cache: "npm"
- name: Install dependencies
run: npm ci
- name: Install Playwright browsers
run: npx playwright install --with-deps
- name: Run Playwright tests
run: npx playwright test
- uses: actions/upload-artifact@v4
if: ${{ !cancelled() }}
with:
name: playwright-report
path: playwright-report/
retention-days: 30
```
### With Sharding
```yaml
name: Playwright Tests
on:
push:
branches: [main]
jobs:
test:
timeout-minutes: 60
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
shardIndex: [1, 2, 3, 4]
shardTotal: [4]
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 22
cache: "npm"
- name: Install dependencies
run: npm ci
- name: Install Playwright browsers
run: npx playwright install --with-deps
- name: Run Playwright tests
run: npx playwright test --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
- name: Upload blob report
if: ${{ !cancelled() }}
uses: actions/upload-artifact@v4
with:
name: blob-report-${{ matrix.shardIndex }}
path: blob-report
retention-days: 1
merge-reports:
if: ${{ !cancelled() }}
needs: [test]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 22
cache: "npm"
- name: Install dependencies
run: npm ci
- name: Download blob reports
uses: actions/download-artifact@v4
with:
path: all-blob-reports
pattern: blob-report-*
merge-multiple: true
- name: Merge reports
run: npx playwright merge-reports --reporter html ./all-blob-reports
- name: Upload HTML report
uses: actions/upload-artifact@v4
with:
name: html-report
path: playwright-report
retention-days: 14
```
### With Container
```yaml
jobs:
test:
timeout-minutes: 60
runs-on: ubuntu-latest
container:
# Use latest or more appropriate playwright version (match package.json)
image: mcr.microsoft.com/playwright:v1.40.0-jammy
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 22
cache: "npm"
- name: Install dependencies
run: npm ci
- name: Run tests
run: npx playwright test
env:
HOME: /root
```
## Docker
### Dockerfile
```dockerfile
FROM mcr.microsoft.com/playwright:v1.40.0-jammy
WORKDIR /app
COPY package*.json ./
RUN npm ci
COPY . .
CMD ["npx", "playwright", "test"]
```
### Docker Compose
```yaml
# docker-compose.yml
version: "3.8"
services:
playwright:
build: .
volumes:
- ./playwright-report:/app/playwright-report
- ./test-results:/app/test-results
environment:
- CI=true
- BASE_URL=http://app:3000
depends_on:
- app
app:
build: ./app
ports:
- "3000:3000"
```
### Run with Docker
```bash
# Build and run
docker build -t playwright-tests .
docker run --rm -v $(pwd)/playwright-report:/app/playwright-report playwright-tests
# With docker-compose
docker-compose run --rm playwright
```
## Reporting
### Configuration
```typescript
// playwright.config.ts
export default defineConfig({
reporter: [
// Always generate
["html", { outputFolder: "playwright-report" }],
// Console output
["list"],
// CI-friendly
["github"], // GitHub Actions annotations
// JUnit for CI integration
["junit", { outputFile: "results.xml" }],
// JSON for custom processing
["json", { outputFile: "results.json" }],
// Blob for merging shards
["blob", { outputDir: "blob-report" }],
],
});
```
### CI-Specific Reporter
```typescript
export default defineConfig({
reporter: process.env.CI
? [["github"], ["blob"], ["html"]]
: [["list"], ["html"]],
});
```
## Sharding
### Command Line
```bash
# Split into 4 shards, run shard 1
npx playwright test --shard=1/4
# Run shard 2
npx playwright test --shard=2/4
```
### Configuration
```typescript
// playwright.config.ts
export default defineConfig({
// Evenly distribute tests across shards
fullyParallel: true,
// For blob reporter to merge later
reporter: process.env.CI ? [["blob"]] : [["html"]],
});
```
### Merge Sharded Reports
```bash
# After all shards complete, merge blob reports
npx playwright merge-reports --reporter html ./all-blob-reports
```
## Environment Management
### Environment Variables
```typescript
// playwright.config.ts
import { defineConfig } from "@playwright/test";
import dotenv from "dotenv";
// Load env file based on environment
dotenv.config({ path: `.env.${process.env.NODE_ENV || "development"}` });
export default defineConfig({
use: {
baseURL: process.env.BASE_URL || "http://localhost:3000",
},
});
```
### Multiple Environments
```yaml
# .github/workflows/playwright.yml
jobs:
test:
strategy:
matrix:
environment: [staging, production]
steps:
- name: Run tests
run: npx playwright test
env:
BASE_URL: ${{ matrix.environment == 'staging' && 'https://staging.example.com' || 'https://example.com' }}
TEST_USER: ${{ secrets[format('TEST_USER_{0}', matrix.environment)] }}
```
### Secrets Management
```yaml
# GitHub Actions secrets
- name: Run tests
run: npx playwright test
env:
TEST_EMAIL: ${{ secrets.TEST_EMAIL }}
TEST_PASSWORD: ${{ secrets.TEST_PASSWORD }}
```
```typescript
// tests use environment variables
test("login", async ({ page }) => {
await page.getByLabel("Email").fill(process.env.TEST_EMAIL!);
await page.getByLabel("Password").fill(process.env.TEST_PASSWORD!);
});
```
## Caching
### Cache Playwright Browsers
```yaml
- name: Cache Playwright browsers
uses: actions/cache@v4
id: playwright-cache
with:
path: ~/.cache/ms-playwright
key: playwright-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
- name: Install Playwright browsers
if: steps.playwright-cache.outputs.cache-hit != 'true'
run: npx playwright install --with-deps
- name: Install system deps only
if: steps.playwright-cache.outputs.cache-hit == 'true'
run: npx playwright install-deps
```
### Cache Node Modules
```yaml
- uses: actions/setup-node@v4
with:
node-version: 22
cache: "npm"
- name: Install dependencies
run: npm ci
```
## Tag-Based Test Filtering
### Run Specific Tags in CI
```yaml
# Run smoke tests on PR
- name: Run smoke tests
run: npx playwright test --grep @smoke
# Run full regression nightly
- name: Run regression
run: npx playwright test --grep @regression
# Exclude flaky tests
- name: Run stable tests
run: npx playwright test --grep-invert @flaky
```
### PR vs Nightly Strategy
```yaml
# .github/workflows/pr.yml - Fast feedback
- name: Run critical tests
run: npx playwright test --grep "@smoke|@critical"
# .github/workflows/nightly.yml - Full coverage
- name: Run all tests
run: npx playwright test --grep-invert @flaky
```
### Tag Filtering in Config
```typescript
// playwright.config.ts
export default defineConfig({
grep: process.env.CI ? /@smoke|@critical/ : undefined,
grepInvert: process.env.CI ? /@flaky/ : undefined,
});
```
### Project-Based Tag Filtering
```typescript
// playwright.config.ts
export default defineConfig({
projects: [
{
name: "smoke",
grep: /@smoke/,
},
{
name: "regression",
grepInvert: /@smoke/,
},
],
});
```
## Best Practices
| Practice | Benefit |
| ----------------------------- | ------------------------- |
| Use `npm ci` | Deterministic installs |
| Run headless in CI | Faster, no display needed |
| Set retries in CI only | Handle flakiness |
| Upload artifacts on failure | Debug failures |
| Use sharding for large suites | Faster execution |
| Cache browsers | Faster setup |
| Use blob reporter for shards | Merge reports correctly |
| Use tags for PR vs nightly | Fast feedback + coverage |
| Exclude @flaky in CI | Stable pipeline |
## CI Configuration Reference
```typescript
// playwright.config.ts - CI optimized
export default defineConfig({
testDir: "./tests",
fullyParallel: true,
forbidOnly: !!process.env.CI,
retries: process.env.CI ? 2 : 0,
workers: process.env.CI ? 1 : undefined,
reporter: process.env.CI
? [["github"], ["blob"], ["html"]]
: [["list"], ["html"]],
use: {
baseURL: process.env.BASE_URL || "http://localhost:3000",
trace: "on-first-retry",
screenshot: "only-on-failure",
video: "on-first-retry",
},
});
```
## Related References
- **Test tags**: See [test-tags.md](../core/test-tags.md) for tagging and filtering patterns
- **Performance optimization**: See [performance.md](performance.md) for sharding and parallelization
- **Debugging CI failures**: See [debugging.md](../debugging/debugging.md) for troubleshooting
- **Test reporting**: See [debugging.md](../debugging/debugging.md) for trace viewer usage

View file

@ -0,0 +1,283 @@
# Container-Based Testing
## Table of Contents
1. [Patterns](#patterns)
2. [Decision Guide](#decision-guide)
3. [Anti-Patterns](#anti-patterns)
4. [Troubleshooting](#troubleshooting)
> **When to use**: Running tests in containers for reproducible environments, CI pipelines, or consistent browser versions across team machines.
## Patterns
### Official Image Usage
Run tests without building a custom image:
```bash
docker run --rm \
-v $(pwd):/app \
-w /app \
-e CI=true \
-e BASE_URL=http://host.docker.internal:3000 \
mcr.microsoft.com/playwright:v1.48.0-noble \
bash -c "npm ci && npx playwright test"
```
Extract reports with bind mounts:
```bash
docker run --rm \
-v $(pwd):/app \
-v $(pwd)/playwright-report:/app/playwright-report \
-v $(pwd)/test-results:/app/test-results \
-w /app \
mcr.microsoft.com/playwright:v1.48.0-noble \
bash -c "npm ci && npx playwright test"
```
### Custom Dockerfile
Build a custom image when you need additional dependencies or pre-installed packages:
```dockerfile
FROM mcr.microsoft.com/playwright:v1.48.0-noble
WORKDIR /app
COPY package.json package-lock.json ./
RUN npm ci
COPY . .
CMD ["npx", "playwright", "test"]
```
Chromium-only slim image:
```dockerfile
FROM node:latest-slim
RUN npx playwright install --with-deps chromium
WORKDIR /app
COPY package.json package-lock.json ./
RUN npm ci
COPY . .
CMD ["npx", "playwright", "test", "--project=chromium"]
```
### Docker Compose Stack
Full application stack with database, cache, and test runner:
```yaml
services:
app:
build: .
ports:
- "3000:3000"
environment:
- NODE_ENV=test
- DATABASE_URL=postgresql://postgres:postgres@db:5432/test
depends_on:
db:
condition: service_healthy
db:
image: postgres:latest-alpine
environment:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: test
healthcheck:
test: ["CMD-SHELL", "pg_isready -U postgres"]
interval: 5s
timeout: 5s
retries: 5
tmpfs:
- /var/lib/postgresql/data
e2e:
image: mcr.microsoft.com/playwright:v1.48.0-noble
working_dir: /app
volumes:
- .:/app
- /app/node_modules
environment:
- CI=true
- BASE_URL=http://app:3000
depends_on:
- app
command: bash -c "npm ci && npx playwright test"
profiles:
- test
```
Run commands:
```bash
docker compose --profile test up --abort-on-container-exit --exit-code-from e2e
docker compose --profile test down -v
```
### CI Container Jobs
**GitHub Actions:**
```yaml
jobs:
test:
runs-on: ubuntu-latest
container:
image: mcr.microsoft.com/playwright:v1.48.0-noble
steps:
- uses: actions/checkout@v4
- run: npm ci
- run: npx playwright test
env:
HOME: /root
```
**GitLab CI:**
```yaml
test:
image: mcr.microsoft.com/playwright:v1.48.0-noble
script:
- npm ci
- npx playwright test
```
**Jenkins:**
```groovy
pipeline {
agent {
docker {
image 'mcr.microsoft.com/playwright:v1.48.0-noble'
args '-u root'
}
}
stages {
stage('Test') {
steps {
sh 'npm ci'
sh 'npx playwright test'
}
}
}
}
```
### Dev Container Setup
VS Code Dev Container or GitHub Codespaces configuration:
```json
{
"name": "Playwright Dev",
"image": "mcr.microsoft.com/playwright:v1.48.0-noble",
"features": {
"ghcr.io/devcontainers/features/node:latest": {
"version": "20"
}
},
"postCreateCommand": "npm ci",
"customizations": {
"vscode": {
"extensions": ["ms-playwright.playwright"]
}
},
"forwardPorts": [3000, 9323],
"remoteUser": "root"
}
```
## Decision Guide
| Scenario | Approach |
|---|---|
| Simple CI pipeline | Official image as CI container |
| Tests need database + cache | Docker Compose with app, db, e2e services |
| Team needs identical environments | Dev Container or custom Dockerfile |
| Only testing Chromium | Slim image with `install --with-deps chromium` |
| Cross-browser testing | Official image (all browsers pre-installed) |
| Local development | Run directly on host for faster iteration |
## Anti-Patterns
| Anti-Pattern | Problem | Solution |
|---|---|---|
| Installing browsers at runtime | Wastes 60-90 seconds per run | Use official image or bake browsers into custom image |
| Running as non-root without sandbox config | Chromium sandbox permission errors | Run as root or disable sandbox |
| Bind-mounting `node_modules` from host | Platform-specific binary crashes | Use anonymous volume: `-v /app/node_modules` |
| No health checks on dependent services | Tests start before database ready | Add `healthcheck` with `depends_on: condition: service_healthy` |
| Building application inside Playwright container | Large image, slow builds | Separate app and e2e containers |
## Troubleshooting
### "browserType.launch: Executable doesn't exist"
Playwright version mismatch with Docker image. Ensure `@playwright/test` version matches image tag:
```bash
npm ls @playwright/test
docker pull mcr.microsoft.com/playwright:v<matching-version>-noble
```
### "net::ERR_CONNECTION_REFUSED" in docker-compose
Tests trying to reach `localhost` instead of service name. Configure `baseURL`:
```typescript
import { defineConfig } from '@playwright/test';
export default defineConfig({
use: {
baseURL: process.env.BASE_URL || 'http://localhost:3000',
},
});
```
```yaml
e2e:
environment:
- BASE_URL=http://app:3000
```
### Permission denied on mounted volumes
Match user IDs or run as root:
```bash
docker run --rm -u $(id -u):$(id -g) \
-v $(pwd):/app -w /app \
mcr.microsoft.com/playwright:v1.48.0-noble \
npx playwright test
```
### Slow container tests on macOS/Windows
Docker Desktop I/O overhead. Copy files instead of mounting:
```dockerfile
FROM mcr.microsoft.com/playwright:v1.48.0-noble
WORKDIR /app
COPY . .
RUN npm ci
CMD ["npx", "playwright", "test"]
```
Or use delegated mount:
```bash
docker run --rm \
-v $(pwd):/app:delegated \
-w /app \
mcr.microsoft.com/playwright:v1.48.0-noble \
bash -c "npm ci && npx playwright test"
```

View file

@ -0,0 +1,546 @@
# GitHub Actions for Playwright
## Table of Contents
1. [CLI Commands](#cli-commands)
2. [Workflow Patterns](#workflow-patterns)
3. [Scenario Guide](#scenario-guide)
4. [Common Mistakes](#common-mistakes)
5. [Troubleshooting](#troubleshooting)
6. [Related](#related)
> **When to use**: Automating Playwright tests on pull requests, main branch merges, or scheduled runs.
## CLI Commands
```bash
npx playwright install --with-deps # browsers + OS dependencies
npx playwright test --shard=1/4 # run shard 1 of 4
npx playwright test --reporter=github # PR annotations
npx playwright merge-reports ./blob-report # combine shard reports
```
## Workflow Patterns
### Basic Workflow
**Use when**: Starting a new project or running a small test suite.
```yaml
# .github/workflows/e2e.yml
name: E2E Tests
on:
push:
branches: [main]
pull_request:
branches: [main]
concurrency:
group: e2e-${{ github.ref }}
cancel-in-progress: true
env:
CI: true
jobs:
test:
timeout-minutes: 30
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- run: npm ci
- name: Cache browsers
id: browser-cache
uses: actions/cache@v4
with:
path: ~/.cache/ms-playwright
key: pw-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
- name: Install browsers
if: steps.browser-cache.outputs.cache-hit != 'true'
run: npx playwright install --with-deps
- name: Install OS dependencies
if: steps.browser-cache.outputs.cache-hit == 'true'
run: npx playwright install-deps
- run: npx playwright test
- name: Upload report
uses: actions/upload-artifact@v4
if: ${{ !cancelled() }}
with:
name: test-report
path: playwright-report/
retention-days: 14
- name: Upload traces
uses: actions/upload-artifact@v4
if: failure()
with:
name: traces
path: test-results/
retention-days: 7
```
### Sharded Execution
**Use when**: Test suite exceeds 10 minutes. Sharding cuts wall-clock time significantly.
**Avoid when**: Suite runs under 5 minutes—sharding overhead negates benefits.
```yaml
# .github/workflows/e2e-sharded.yml
name: E2E Tests (Sharded)
on:
push:
branches: [main]
pull_request:
branches: [main]
concurrency:
group: e2e-${{ github.ref }}
cancel-in-progress: true
env:
CI: true
jobs:
test:
timeout-minutes: 20
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
shard: [1/4, 2/4, 3/4, 4/4]
steps:
- uses: actions/checkout@v4
- run: npm ci
- name: Cache browsers
id: browser-cache
uses: actions/cache@v4
with:
path: ~/.cache/ms-playwright
key: pw-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
- name: Install browsers
if: steps.browser-cache.outputs.cache-hit != 'true'
run: npx playwright install --with-deps
- name: Install OS dependencies
if: steps.browser-cache.outputs.cache-hit == 'true'
run: npx playwright install-deps
- name: Run tests (shard ${{ matrix.shard }})
run: npx playwright test --shard=${{ matrix.shard }}
- name: Upload blob report
uses: actions/upload-artifact@v4
if: ${{ !cancelled() }}
with:
name: blob-${{ strategy.job-index }}
path: blob-report/
retention-days: 1
merge:
if: ${{ !cancelled() }}
needs: test
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- run: npm ci
- name: Download blob reports
uses: actions/download-artifact@v4
with:
path: all-blobs
pattern: blob-*
merge-multiple: true
- name: Merge reports
run: npx playwright merge-reports --reporter=html ./all-blobs
- name: Upload merged report
uses: actions/upload-artifact@v4
with:
name: test-report
path: playwright-report/
retention-days: 14
```
**Config for sharding**—enable blob reporter:
```typescript
// playwright.config.ts
import { defineConfig } from '@playwright/test';
export default defineConfig({
reporter: process.env.CI
? [['blob'], ['github']]
: [['html', { open: 'on-failure' }]],
});
```
### Container-Based Execution
**Use when**: Reproducible environment matching local Docker setup, or runner OS dependencies cause issues.
**Avoid when**: Standard `ubuntu-latest` with `--with-deps` works fine.
```yaml
# .github/workflows/e2e-container.yml
name: E2E Tests (Container)
on:
pull_request:
branches: [main]
jobs:
test:
timeout-minutes: 30
runs-on: ubuntu-latest
container:
image: mcr.microsoft.com/playwright:v1.48.0-noble
steps:
- uses: actions/checkout@v4
- run: npm ci
- name: Run tests
run: npx playwright test
env:
HOME: /root
- uses: actions/upload-artifact@v4
if: ${{ !cancelled() }}
with:
name: test-report
path: playwright-report/
retention-days: 14
```
### Environment Secrets
**Use when**: Tests target staging/production with credentials.
**Avoid when**: Tests only run against local dev server.
```yaml
# .github/workflows/e2e-staging.yml
name: Staging Tests
on:
push:
branches: [main]
workflow_dispatch:
jobs:
test:
timeout-minutes: 30
runs-on: ubuntu-latest
environment: staging
env:
CI: true
BASE_URL: ${{ vars.STAGING_URL }}
TEST_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
API_TOKEN: ${{ secrets.API_TOKEN }}
steps:
- uses: actions/checkout@v4
- run: npm ci
- name: Cache browsers
id: browser-cache
uses: actions/cache@v4
with:
path: ~/.cache/ms-playwright
key: pw-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
- name: Install browsers
if: steps.browser-cache.outputs.cache-hit != 'true'
run: npx playwright install --with-deps
- name: Install OS dependencies
if: steps.browser-cache.outputs.cache-hit == 'true'
run: npx playwright install-deps
- name: Run smoke tests
run: npx playwright test --grep @smoke
- uses: actions/upload-artifact@v4
if: ${{ !cancelled() }}
with:
name: staging-report
path: playwright-report/
retention-days: 14
```
### Scheduled Runs
**Use when**: Full regression suite is too slow for every PR—run nightly instead.
**Avoid when**: Suite runs under 15 minutes and can run on every PR.
```yaml
# .github/workflows/nightly.yml
name: Nightly Regression
on:
schedule:
- cron: '0 3 * * 1-5'
workflow_dispatch:
jobs:
test:
timeout-minutes: 60
runs-on: ubuntu-latest
env:
CI: true
BASE_URL: ${{ vars.STAGING_URL }}
steps:
- uses: actions/checkout@v4
- run: npm ci
- name: Install browsers
run: npx playwright install --with-deps
- name: Run full regression
run: npx playwright test --grep @regression
- uses: actions/upload-artifact@v4
if: ${{ !cancelled() }}
with:
name: nightly-${{ github.run_number }}
path: playwright-report/
retention-days: 30
- name: Notify on failure
if: failure()
uses: slackapi/slack-github-action@latest
with:
payload: |
{
"text": "Nightly regression failed: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
}
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
```
### Reusable Workflow
**Use when**: Multiple repositories share the same Playwright setup.
**Avoid when**: Single repo with one workflow.
```yaml
# .github/workflows/pw-reusable.yml
name: Playwright Reusable
on:
workflow_call:
inputs:
node-version:
type: string
default: 'lts/*'
test-command:
type: string
default: 'npx playwright test'
secrets:
BASE_URL:
required: false
TEST_PASSWORD:
required: false
jobs:
test:
timeout-minutes: 30
runs-on: ubuntu-latest
env:
CI: true
BASE_URL: ${{ secrets.BASE_URL }}
TEST_PASSWORD: ${{ secrets.TEST_PASSWORD }}
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: ${{ inputs.node-version }}
cache: npm
- run: npm ci
- name: Cache browsers
id: browser-cache
uses: actions/cache@v4
with:
path: ~/.cache/ms-playwright
key: pw-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
- name: Install browsers
if: steps.browser-cache.outputs.cache-hit != 'true'
run: npx playwright install --with-deps
- name: Install OS dependencies
if: steps.browser-cache.outputs.cache-hit == 'true'
run: npx playwright install-deps
- name: Run tests
run: ${{ inputs.test-command }}
- uses: actions/upload-artifact@v4
if: ${{ !cancelled() }}
with:
name: test-report
path: playwright-report/
retention-days: 14
```
**Calling the reusable workflow:**
```yaml
# .github/workflows/ci.yml
name: CI
on:
pull_request:
branches: [main]
jobs:
e2e:
uses: ./.github/workflows/pw-reusable.yml
with:
node-version: 'lts/*'
secrets:
BASE_URL: ${{ secrets.STAGING_URL }}
TEST_PASSWORD: ${{ secrets.TEST_PASSWORD }}
```
## Scenario Guide
| Scenario | Approach |
|---|---|
| Small suite (< 5 min) | Single job, no sharding |
| Medium suite (5-20 min) | 2-4 shards with matrix |
| Large suite (20+ min) | 4-8 shards + blob merge |
| Cross-browser on PRs | Chromium only on PRs; all browsers on main |
| Staging/prod smoke tests | Separate workflow with `environment:` |
| Nightly full regression | `schedule` trigger + `workflow_dispatch` |
| Multiple repos, same setup | Reusable workflow with `workflow_call` |
| Reproducible env needed | Container job with Playwright image |
## Common Mistakes
| Mistake | Problem | Fix |
|---|---|---|
| No `concurrency` group | Duplicate runs waste minutes | Add `concurrency: { group: ..., cancel-in-progress: true }` |
| `fail-fast: true` with sharding | One failure cancels others | Set `fail-fast: false` |
| No browser caching | 60-90 seconds wasted per run | Cache `~/.cache/ms-playwright` |
| No `timeout-minutes` | Stuck jobs run for 6 hours | Set explicit timeout: 20-30 minutes |
| Artifacts only on failure | No report when tests pass | Use `if: ${{ !cancelled() }}` |
| Hardcoded secrets | Security risk | Use GitHub Secrets and Environments |
| All browsers on every PR | 3x CI cost | Chromium on PR; cross-browser on main |
| No artifact retention | Default 90-day fills storage | Set `retention-days: 7-14` |
| Missing `--with-deps` | Browser launch failures | Always use `npx playwright install --with-deps` |
## Troubleshooting
### Browser launch fails: "Missing dependencies"
**Cause**: Browsers restored from cache but OS dependencies weren't cached.
**Fix**: Run `npx playwright install-deps` on cache hit:
```yaml
- name: Install OS dependencies
if: steps.browser-cache.outputs.cache-hit == 'true'
run: npx playwright install-deps
```
### Tests pass locally but timeout in CI
**Cause**: CI runners have fewer resources than dev machines.
**Fix**: Reduce workers and increase timeouts:
```typescript
// playwright.config.ts
import { defineConfig } from '@playwright/test';
export default defineConfig({
workers: process.env.CI ? '50%' : undefined,
use: {
actionTimeout: process.env.CI ? 15_000 : 10_000,
navigationTimeout: process.env.CI ? 30_000 : 15_000,
},
});
```
### Sharded reports incomplete
**Cause**: Artifact names collide or `merge-multiple` not set.
**Fix**: Unique names per shard and enable merge:
```yaml
# Upload in each shard
- uses: actions/upload-artifact@v4
with:
name: blob-${{ strategy.job-index }}
path: blob-report/
# Download in merge job
- uses: actions/download-artifact@v4
with:
path: all-blobs
pattern: blob-*
merge-multiple: true
```
### `webServer` fails: "port already in use"
**Cause**: Zombie process from previous run.
**Fix**: Kill stale processes before starting:
```yaml
- name: Kill stale processes
run: lsof -ti:3000 | xargs kill -9 2>/dev/null || true
```
### No PR annotations
**Cause**: `github` reporter not configured.
**Fix**: Add `github` reporter for CI:
```typescript
// playwright.config.ts
import { defineConfig } from '@playwright/test';
export default defineConfig({
reporter: process.env.CI
? [['html', { open: 'never' }], ['github']]
: [['html', { open: 'on-failure' }]],
});
```
## Related
- [test-tags.md](../core/test-tags.md) — tagging and filtering tests
- [parallel-sharding.md](parallel-sharding.md) — sharding strategies
- [reporting.md](reporting.md) — reporter configuration
- [docker.md](docker.md) — container images
- [gitlab.md](gitlab.md) — GitLab CI equivalent
- [other-providers.md](other-providers.md) — CircleCI, Azure DevOps, Jenkins

View file

@ -0,0 +1,397 @@
# GitLab CI/CD Configuration
## Table of Contents
1. [Key Commands](#key-commands)
2. [Patterns](#patterns)
3. [Decision Guide](#decision-guide)
4. [Anti-Patterns](#anti-patterns)
5. [Troubleshooting](#troubleshooting)
> **When to use**: Running Playwright tests in GitLab pipelines on merge requests, merges to main, or scheduled pipelines.
## Key Commands
```bash
npx playwright install --with-deps # install browsers + OS deps
npx playwright test --shard=1/4 # run 1 of 4 parallel shards
npx playwright merge-reports ./blob-report # merge shard results
npx playwright test --reporter=dot # minimal output for CI logs
```
## Patterns
### Basic Pipeline Configuration
**Use when**: Any GitLab project with Playwright tests.
```yaml
# .gitlab-ci.yml
image: mcr.microsoft.com/playwright:v1.48.0-noble
stages:
- install
- test
- report
variables:
CI: "true"
npm_config_cache: "$CI_PROJECT_DIR/.npm"
cache:
key:
files:
- package-lock.json
paths:
- .npm/
- node_modules/
setup:
stage: install
script:
- npm ci
artifacts:
paths:
- node_modules/
expire_in: 1 hour
e2e:
stage: test
needs: [setup]
script:
- npx playwright test
artifacts:
when: always
paths:
- playwright-report/
- test-results/
expire_in: 14 days
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
```
### Sharded Parallel Execution
**Use when**: Test suite exceeds 10 minutes. GitLab's `parallel` keyword splits across jobs automatically.
**Avoid when**: Suite runs under 5 minutes.
```yaml
image: mcr.microsoft.com/playwright:v1.48.0-noble
stages:
- install
- test
- report
variables:
CI: "true"
npm_config_cache: "$CI_PROJECT_DIR/.npm"
cache:
key:
files:
- package-lock.json
paths:
- .npm/
- node_modules/
setup:
stage: install
script:
- npm ci
artifacts:
paths:
- node_modules/
expire_in: 1 hour
e2e:
stage: test
needs: [setup]
parallel: 4
script:
- npx playwright test --shard=$CI_NODE_INDEX/$CI_NODE_TOTAL
artifacts:
when: always
paths:
- blob-report/
expire_in: 1 hour
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
combine-reports:
stage: report
needs: [e2e]
when: always
script:
- npx playwright merge-reports --reporter=html ./blob-report
artifacts:
when: always
paths:
- playwright-report/
expire_in: 14 days
```
**Config for sharded pipelines:**
```typescript
// playwright.config.ts
export default defineConfig({
reporter: process.env.CI
? [["blob"], ["dot"]]
: [["html", { open: "on-failure" }]],
});
```
### Environment Variables and Secrets
**Use when**: Tests need secrets (API keys, passwords) and should only run on merge requests or the default branch.
```yaml
image: mcr.microsoft.com/playwright:v1.48.0-noble
stages:
- test
variables:
CI: "true"
e2e:staging:
stage: test
variables:
BASE_URL: $STAGING_URL
TEST_PASSWORD: $TEST_PASSWORD
API_KEY: $API_KEY
before_script:
- npm ci
script:
- npx playwright test
artifacts:
when: always
paths:
- playwright-report/
- test-results/
expire_in: 14 days
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
- when: manual
allow_failure: true
```
**Setting variables in GitLab:**
Navigate to **Settings > CI/CD > Variables** and add:
- `STAGING_URL` -- not masked, not protected
- `TEST_PASSWORD` -- masked, protected
- `API_KEY` -- masked, protected
### Multi-Browser Matrix
**Use when**: Running Chromium on MRs and all browsers on the default branch.
```yaml
image: mcr.microsoft.com/playwright:v1.48.0-noble
stages:
- install
- test
variables:
CI: "true"
setup:
stage: install
script:
- npm ci
artifacts:
paths:
- node_modules/
expire_in: 1 hour
e2e:chromium:
stage: test
needs: [setup]
script:
- npx playwright test --project=chromium
artifacts:
when: always
paths:
- playwright-report/
- test-results/
expire_in: 14 days
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
e2e:all-browsers:
stage: test
needs: [setup]
parallel:
matrix:
- PROJECT: [chromium, firefox, webkit]
script:
- npx playwright test --project=$PROJECT
artifacts:
when: always
paths:
- playwright-report/
- test-results/
expire_in: 14 days
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
```
### Services Integration (Database, Cache)
**Use when**: Tests need the application running alongside Playwright, or you need external services.
```yaml
stages:
- test
e2e:integration:
stage: test
image: mcr.microsoft.com/playwright:v1.48.0-noble
services:
- name: postgres:latest
alias: db
- name: redis:latest
alias: cache
variables:
CI: "true"
DATABASE_URL: "postgresql://postgres:postgres@db:5432/testdb"
REDIS_URL: "redis://cache:6379"
POSTGRES_PASSWORD: "postgres"
POSTGRES_DB: "testdb"
before_script:
- npm ci
- npx prisma db push
- npx prisma db seed
script:
- npx playwright test
artifacts:
when: always
paths:
- playwright-report/
- test-results/
expire_in: 14 days
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
```
### Scheduled Nightly Regression
**Use when**: Full regression is too slow for every MR.
```yaml
e2e:nightly:
stage: test
image: mcr.microsoft.com/playwright:v1.48.0-noble
before_script:
- npm ci
script:
- npx playwright test --grep @regression
artifacts:
when: always
paths:
- playwright-report/
expire_in: 30 days
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
```
Set up the schedule in **CI/CD > Schedules**: `0 3 * * 1-5` (3 AM UTC, weekdays).
## Decision Guide
| Scenario | Approach | Why |
| ------------------------------------ | ------------------------------------------------------ | --------------------------------------------------- |
| Simple project, < 5 min suite | Single `test` job using Playwright Docker image | No sharding overhead; artifacts capture report |
| Suite > 10 min | `parallel: N` with `--shard` | GitLab auto-assigns `CI_NODE_INDEX`/`CI_NODE_TOTAL` |
| Merge request fast feedback | Chromium only on MRs; all browsers on main | 3x fewer pipeline minutes on MRs |
| External services needed (DB, Redis) | `services:` keyword with Postgres/Redis images | GitLab manages service lifecycle |
| Secrets for staging environment | GitLab CI/CD Variables (masked + protected) | Never hardcode secrets in `.gitlab-ci.yml` |
| Full nightly regression | Pipeline schedule (`CI_PIPELINE_SOURCE == "schedule"`) | Avoids blocking MR pipelines |
| Report browsing | `artifacts:` with `paths: [playwright-report/]` | Browse directly in GitLab job artifacts UI |
## Anti-Patterns
| Anti-Pattern | Problem | Do This Instead |
| ---------------------------------------------------- | ------------------------------------------------------------------ | ------------------------------------------------------------------------- |
| Not using the Playwright Docker image | Installing browsers every run adds 1-2 minutes | Use `mcr.microsoft.com/playwright:v1.48.0-noble` as base image |
| `artifacts: when: on_failure` only | No report when tests pass; can't verify results | Use `when: always` to capture reports regardless |
| No `expire_in` on artifacts | Artifacts accumulate and consume storage | Set `expire_in: 14 days` for reports, `1 hour` for intermediate artifacts |
| Hardcoding `CI_NODE_TOTAL` in shard flag | Breaks when you change `parallel:` value | Use `--shard=$CI_NODE_INDEX/$CI_NODE_TOTAL` |
| Skipping `needs:` between stages | Jobs wait for all previous stage jobs, not just their dependencies | Use `needs:` for precise dependency graphs |
| Large `cache:` including `node_modules/` without key | Stale cache causes version conflicts | Key cache on `package-lock.json` hash |
## Troubleshooting
### Browser launch fails: "Failed to launch browser"
**Cause**: Not using the Playwright Docker image, or using a version that doesn't match your `@playwright/test` version.
**Fix**: Match the Docker image tag to your Playwright version:
```yaml
# Check your version: npm ls @playwright/test
image: mcr.microsoft.com/playwright:v1.48.0-noble
```
### Tests hang in GitLab runner: "Navigation timeout exceeded"
**Cause**: GitLab shared runners may have limited resources.
**Fix**: Reduce workers and increase timeouts:
```typescript
export default defineConfig({
workers: process.env.CI ? 2 : undefined,
use: {
navigationTimeout: process.env.CI ? 30_000 : 15_000,
},
});
```
### Pipeline runs on every push, not just merge requests
**Cause**: Missing `rules:` configuration.
**Fix**: Add explicit rules:
```yaml
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
```
### Services (Postgres/Redis) not reachable from tests
**Cause**: Using `localhost` instead of the service alias.
**Fix**: Use the service alias as hostname:
```yaml
services:
- name: postgres:latest
alias: db
variables:
DATABASE_URL: "postgresql://postgres:postgres@db:5432/testdb"
```
### Merged report is empty after sharded run
**Cause**: Each shard job needs the `blob` reporter, not `html`.
**Fix**: Configure blob reporter for CI:
```typescript
export default defineConfig({
reporter: process.env.CI
? [["blob"], ["dot"]]
: [["html", { open: "on-failure" }]],
});
```

View file

@ -0,0 +1,521 @@
# CI: CircleCI, Azure DevOps, and Jenkins
> **When to use**: Running Playwright tests in CI platforms other than GitHub Actions or GitLab.
## Table of Contents
1. [Common Commands](#common-commands)
2. [Jenkins](#jenkins)
3. [CircleCI](#circleci)
4. [Azure DevOps](#azure-devops)
5. [JUnit Reporter Config](#junit-reporter-config)
6. [Platform Comparison](#platform-comparison)
7. [Troubleshooting](#troubleshooting)
8. [Anti-Patterns](#anti-patterns)
---
## Common Commands
```bash
npx playwright install --with-deps # browsers + OS dependencies
npx playwright test --shard=1/4 # parallel sharding
npx playwright merge-reports ./blob-report # combine shard results
npx playwright test --reporter=dot,html # multiple reporters
```
## Jenkins
### Declarative Pipeline
```groovy
// Jenkinsfile
pipeline {
agent {
docker {
image 'mcr.microsoft.com/playwright:v1.48.0-noble'
args '-u root'
}
}
environment {
CI = 'true'
HOME = '/root'
npm_config_cache = "${WORKSPACE}/.npm"
}
options {
timeout(time: 30, unit: 'MINUTES')
disableConcurrentBuilds()
}
stages {
stage('Install') {
steps {
sh 'npm ci'
}
}
stage('Test') {
steps {
sh 'npx playwright test'
}
post {
always {
junit allowEmptyResults: true,
testResults: 'results/junit.xml'
archiveArtifacts artifacts: 'pw-report/**',
allowEmptyArchive: true
archiveArtifacts artifacts: 'results/**',
allowEmptyArchive: true
}
}
}
}
post {
failure {
echo 'Tests failed!'
}
cleanup {
cleanWs()
}
}
}
```
### Parallel Shards
```groovy
// Jenkinsfile (sharded)
pipeline {
agent none
environment {
CI = 'true'
HOME = '/root'
}
options {
timeout(time: 30, unit: 'MINUTES')
}
stages {
stage('Test') {
parallel {
stage('Shard 1') {
agent {
docker {
image 'mcr.microsoft.com/playwright:v1.48.0-noble'
args '-u root'
}
}
steps {
sh 'npm ci'
sh 'npx playwright test --shard=1/4'
}
post {
always {
archiveArtifacts artifacts: 'blob-report/**',
allowEmptyArchive: true
}
}
}
stage('Shard 2') {
agent {
docker {
image 'mcr.microsoft.com/playwright:v1.48.0-noble'
args '-u root'
}
}
steps {
sh 'npm ci'
sh 'npx playwright test --shard=2/4'
}
post {
always {
archiveArtifacts artifacts: 'blob-report/**',
allowEmptyArchive: true
}
}
}
stage('Shard 3') {
agent {
docker {
image 'mcr.microsoft.com/playwright:v1.48.0-noble'
args '-u root'
}
}
steps {
sh 'npm ci'
sh 'npx playwright test --shard=3/4'
}
post {
always {
archiveArtifacts artifacts: 'blob-report/**',
allowEmptyArchive: true
}
}
}
stage('Shard 4') {
agent {
docker {
image 'mcr.microsoft.com/playwright:v1.48.0-noble'
args '-u root'
}
}
steps {
sh 'npm ci'
sh 'npx playwright test --shard=4/4'
}
post {
always {
archiveArtifacts artifacts: 'blob-report/**',
allowEmptyArchive: true
}
}
}
}
}
}
}
```
## CircleCI
### Basic Pipeline
```yaml
# .circleci/config.yml
version: 2.1
executors:
pw:
docker:
- image: mcr.microsoft.com/playwright:v1.48.0-noble
working_directory: ~/app
jobs:
install:
executor: pw
steps:
- checkout
- restore_cache:
keys:
- deps-{{ checksum "package-lock.json" }}
- run: npm ci
- save_cache:
key: deps-{{ checksum "package-lock.json" }}
paths:
- node_modules
- persist_to_workspace:
root: .
paths:
- node_modules
test:
executor: pw
parallelism: 4
steps:
- checkout
- attach_workspace:
at: .
- run:
name: Run tests
command: |
npx playwright test --shard=$((CIRCLE_NODE_INDEX + 1))/$CIRCLE_NODE_TOTAL
- store_artifacts:
path: pw-report
destination: pw-report
- store_artifacts:
path: results
destination: results
- store_test_results:
path: results/junit.xml
workflows:
test:
jobs:
- install
- test:
requires:
- install
```
### Using Orbs
```yaml
# .circleci/config.yml
version: 2.1
orbs:
node: circleci/node@latest
executors:
pw:
docker:
- image: mcr.microsoft.com/playwright:v1.48.0-noble
jobs:
e2e:
executor: pw
parallelism: 4
steps:
- checkout
- node/install-packages
- run:
name: Run tests
command: npx playwright test --shard=$((CIRCLE_NODE_INDEX + 1))/$CIRCLE_NODE_TOTAL
- store_artifacts:
path: pw-report
- store_test_results:
path: results/junit.xml
workflows:
main:
jobs:
- e2e
```
## Azure DevOps
### Basic Pipeline
```yaml
# azure-pipelines.yml
trigger:
branches:
include:
- main
pr:
branches:
include:
- main
pool:
vmImage: "ubuntu-latest"
variables:
CI: "true"
npm_config_cache: $(Pipeline.Workspace)/.npm
steps:
- task: NodeTool@0
inputs:
versionSpec: "20.x"
displayName: "Install Node.js"
- task: Cache@2
inputs:
key: 'npm | "$(Agent.OS)" | package-lock.json'
restoreKeys: |
npm | "$(Agent.OS)"
path: $(npm_config_cache)
displayName: "Cache npm"
- script: npm ci
displayName: "Install dependencies"
- script: npx playwright install --with-deps
displayName: "Install browsers"
- script: npx playwright test
displayName: "Run tests"
- task: PublishTestResults@2
condition: always()
inputs:
testResultsFormat: "JUnit"
testResultsFiles: "results/junit.xml"
mergeTestResults: true
testRunTitle: "E2E Tests"
displayName: "Publish results"
- task: PublishPipelineArtifact@1
condition: always()
inputs:
targetPath: pw-report
artifact: pw-report
publishLocation: "pipeline"
displayName: "Upload report"
```
### With Sharding
```yaml
# azure-pipelines.yml
trigger:
branches:
include:
- main
pr:
branches:
include:
- main
variables:
CI: "true"
stages:
- stage: Test
jobs:
- job: E2E
pool:
vmImage: "ubuntu-latest"
strategy:
matrix:
shard1:
SHARD: "1/4"
shard2:
SHARD: "2/4"
shard3:
SHARD: "3/4"
shard4:
SHARD: "4/4"
steps:
- task: NodeTool@0
inputs:
versionSpec: "20.x"
- script: npm ci
displayName: "Install dependencies"
- script: npx playwright install --with-deps
displayName: "Install browsers"
- script: npx playwright test --shard=$(SHARD)
displayName: "Run tests (shard $(SHARD))"
- task: PublishPipelineArtifact@1
condition: always()
inputs:
targetPath: blob-report
artifact: blob-report-$(System.JobPositionInPhase)
displayName: "Upload blob report"
- stage: Report
dependsOn: Test
condition: always()
jobs:
- job: MergeReports
pool:
vmImage: "ubuntu-latest"
steps:
- task: NodeTool@0
inputs:
versionSpec: "20.x"
- script: npm ci
displayName: "Install dependencies"
- task: DownloadPipelineArtifact@2
inputs:
patterns: "blob-report-*/**"
path: all-blob-reports
displayName: "Download blob reports"
- script: npx playwright merge-reports --reporter=html ./all-blob-reports
displayName: "Merge reports"
- task: PublishPipelineArtifact@1
inputs:
targetPath: pw-report
artifact: pw-report
displayName: "Upload merged report"
```
## JUnit Reporter Config
All platforms benefit from JUnit output for native test result display:
```typescript
// playwright.config.ts
import { defineConfig } from "@playwright/test";
export default defineConfig({
reporter: process.env.CI
? [
["dot"],
["html", { open: "never" }],
["junit", { outputFile: "results/junit.xml" }],
]
: [["html", { open: "on-failure" }]],
});
```
## Platform Comparison
| Feature | CircleCI | Azure DevOps | Jenkins |
| ----------------- | ----------------------------------------------- | -------------------------------- | ---------------------- |
| Docker support | `docker:` executor | `vmImage` or container jobs | Docker Pipeline plugin |
| Parallelism | `parallelism: N` + `CIRCLE_NODE_INDEX` | `strategy.matrix` | `parallel` stages |
| Artifact upload | `store_artifacts` | `PublishPipelineArtifact@1` | `archiveArtifacts` |
| JUnit integration | `store_test_results` | `PublishTestResults@2` | `junit` step |
| Shard variable | `$((CIRCLE_NODE_INDEX + 1))/$CIRCLE_NODE_TOTAL` | Define in matrix: `SHARD: '1/4'` | Hardcode per stage |
| Cache key | `checksum "package-lock.json"` | `Cache@2` with key template | `stash`/`unstash` |
| Secrets | Context + env variables | Variable groups | Credentials plugin |
## Troubleshooting
### Jenkins: "Browser closed unexpectedly"
Running as non-root in container causes sandbox issues.
```groovy
agent {
docker {
image 'mcr.microsoft.com/playwright:v1.48.0-noble'
args '-u root'
}
}
environment {
HOME = '/root'
}
```
### CircleCI: "Executable doesn't exist"
Image version mismatch with `@playwright/test` version. Use `latest` tag or match versions:
```yaml
docker:
- image: mcr.microsoft.com/playwright:v1.48.0-noble
```
### Azure DevOps: Test results not showing
Missing JUnit reporter or `PublishTestResults@2` task:
```typescript
reporter: [['junit', { outputFile: 'results/junit.xml' }]],
```
```yaml
- task: PublishTestResults@2
condition: always()
inputs:
testResultsFormat: "JUnit"
testResultsFiles: "results/junit.xml"
```
### Shard index off by one
CircleCI's `CIRCLE_NODE_INDEX` is 0-based, Playwright's `--shard` is 1-based:
```yaml
# CircleCI - add 1
command: npx playwright test --shard=$((CIRCLE_NODE_INDEX + 1))/$CIRCLE_NODE_TOTAL
```
## Anti-Patterns
| Anti-Pattern | Problem | Solution |
| ----------------------------------- | ----------------------------------------- | ---------------------------------------------------- |
| Missing `--with-deps` on bare metal | OS libs missing, browser launch fails | Use Playwright Docker image or `--with-deps` |
| No JUnit reporter | CI can't display test results | Add `['junit', { outputFile: 'results/junit.xml' }]` |
| No job timeout | Hung tests consume resources indefinitely | Set explicit timeout (20-30 min) |
| No artifact upload on success | Can't verify passing results | Always upload reports (`condition: always()`) |
| Non-root in container without setup | Permission errors on browser binaries | Run as root or configure permissions |
| Hardcoded shard count | Must update multiple places | Use CI-native variables |

View file

@ -0,0 +1,371 @@
# Sharding and Parallel Execution
## Table of Contents
1. [CLI Commands](#cli-commands)
2. [Patterns](#patterns)
3. [Decision Guide](#decision-guide)
4. [Anti-Patterns](#anti-patterns)
5. [Troubleshooting](#troubleshooting)
> **When to use**: Speeding up test suites by running tests concurrently on one machine (workers) or splitting across multiple CI jobs (sharding).
## CLI Commands
```bash
# Parallelism within one machine
npx playwright test --workers=4
npx playwright test --workers=50%
# Splitting across CI jobs
npx playwright test --shard=1/4
npx playwright test --shard=2/4
# Merging shard outputs
npx playwright merge-reports ./blob-report
npx playwright merge-reports --reporter=html,json ./blob-report
# Override config for single run
npx playwright test --fully-parallel
```
## Patterns
### Worker Configuration
**Use when**: Controlling concurrent test execution on a single machine.
```ts
// playwright.config.ts
import { defineConfig } from "@playwright/test";
export default defineConfig({
// Tests WITHIN a file also run in parallel
fullyParallel: true,
// Worker count options:
// - undefined: auto-detect (half CPU cores)
// - number: fixed count
// - string: percentage of cores
workers: process.env.CI ? "50%" : undefined,
});
```
**`fullyParallel` behavior:**
| Setting | Files parallel | Tests in file parallel |
| -------------------------------- | -------------- | ---------------------- |
| `fullyParallel: false` (default) | Yes | No (serial) |
| `fullyParallel: true` | Yes | Yes |
**Serial execution for specific files:**
```ts
// tests/checkout-flow.spec.ts
import { test, expect } from "@playwright/test";
test.describe.configure({ mode: "serial" });
test("add items to cart", async ({ page }) => {
// ...
});
test("complete payment", async ({ page }) => {
// ...
});
```
### Sharding Across CI Machines
**Use when**: Suite exceeds 5 minutes even with maximum workers.
```bash
# Job 1 Job 2 Job 3 Job 4
--shard=1/4 --shard=2/4 --shard=3/4 --shard=4/4
```
**Config for sharded runs:**
```ts
// playwright.config.ts
import { defineConfig } from "@playwright/test";
export default defineConfig({
fullyParallel: true,
workers: process.env.CI ? "50%" : undefined,
reporter: process.env.CI
? [["blob"], ["github"]]
: [["html", { open: "on-failure" }]],
});
```
### Merging Shard Reports
**Use when**: Combining blob reports from multiple shards into a unified report.
```bash
# Merge all blobs into HTML
npx playwright merge-reports --reporter=html ./all-blob-reports
# Multiple formats
npx playwright merge-reports --reporter=html,json,junit ./all-blob-reports
# Custom output location
PLAYWRIGHT_HTML_REPORT=merged-report npx playwright merge-reports --reporter=html ./all-blob-reports
```
**GitHub Actions merge job:**
```yaml
merge-reports:
if: ${{ !cancelled() }}
needs: test
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- run: npm ci
- uses: actions/download-artifact@v4
with:
path: all-blob-reports
pattern: blob-report-*
merge-multiple: true
- run: npx playwright merge-reports --reporter=html ./all-blob-reports
- uses: actions/upload-artifact@v4
with:
name: playwright-report
path: playwright-report/
retention-days: 14
```
### Worker-Scoped Fixtures
**Use when**: Expensive resources (DB connections, auth tokens) should be created once per worker, not per test.
```ts
// fixtures.ts
import { test as base } from "@playwright/test";
type WorkerFixtures = {
dbClient: DatabaseClient;
apiToken: string;
};
export const test = base.extend<{}, WorkerFixtures>({
dbClient: [
async ({}, use) => {
const client = await DatabaseClient.connect(process.env.DB_URL!);
await use(client);
await client.disconnect();
},
{ scope: "worker" },
],
apiToken: [
async ({}, use, workerInfo) => {
const res = await fetch(`${process.env.API_URL}/auth`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
user: `test-user-${workerInfo.workerIndex}`,
password: process.env.TEST_PASSWORD,
}),
});
const { token } = await res.json();
await use(token);
},
{ scope: "worker" },
],
});
export { expect } from "@playwright/test";
```
### Test Isolation for Parallelism
**Use when**: Preparing tests to run without interference.
Each test must create its own state. No test should depend on or modify shared state.
```ts
// BAD: Shared user causes race conditions
test("edit settings", async ({ page }) => {
await page.goto("/users/test-user/settings");
await page.getByLabel("Email").fill("new@example.com");
await page.getByRole("button", { name: "Save" }).click();
});
// GOOD: Unique user per test
test("edit settings", async ({ page, request }) => {
const res = await request.post("/api/users", {
data: { name: `user-${Date.now()}`, email: `${Date.now()}@test.com` },
});
const user = await res.json();
await page.goto(`/users/${user.id}/settings`);
await page.getByLabel("Email").fill("updated@example.com");
await page.getByRole("button", { name: "Save" }).click();
await expect(page.getByLabel("Email")).toHaveValue("updated@example.com");
await request.delete(`/api/users/${user.id}`);
});
```
**Using `testInfo` for unique identifiers:**
```ts
import { test, expect } from "@playwright/test";
test("submit order", async ({ page }, testInfo) => {
const orderId = `order-${testInfo.workerIndex}-${Date.now()}`;
await page.goto(`/orders/new?ref=${orderId}`);
// ...
});
```
### Dynamic Shard Count
**Use when**: Automatically adjusting shards based on test count.
```yaml
# .github/workflows/playwright.yml
jobs:
calculate-shards:
runs-on: ubuntu-latest
outputs:
shard-count: ${{ steps.calc.outputs.count }}
shard-matrix: ${{ steps.calc.outputs.matrix }}
steps:
- uses: actions/checkout@v4
- run: npm ci
- id: calc
run: |
TEST_COUNT=$(npx playwright test --list --reporter=json 2>/dev/null | node -e "
const data = require('fs').readFileSync('/dev/stdin', 'utf8');
const parsed = JSON.parse(data);
console.log(parsed.suites?.reduce((acc, s) => acc + (s.specs?.length || 0), 0) || 0);
")
# 1 shard per 20 tests, min 1, max 8
SHARDS=$(( (TEST_COUNT + 19) / 20 ))
SHARDS=$(( SHARDS > 8 ? 8 : SHARDS ))
SHARDS=$(( SHARDS < 1 ? 1 : SHARDS ))
MATRIX="["
for i in $(seq 1 $SHARDS); do
[ $i -gt 1 ] && MATRIX+=","
MATRIX+="\"$i/$SHARDS\""
done
MATRIX+="]"
echo "count=$SHARDS" >> $GITHUB_OUTPUT
echo "matrix=$MATRIX" >> $GITHUB_OUTPUT
test:
needs: calculate-shards
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
shard: ${{ fromJson(needs.calculate-shards.outputs.shard-matrix) }}
steps:
- uses: actions/checkout@v4
- run: npm ci
- run: npx playwright install --with-deps
- run: npx playwright test --shard=${{ matrix.shard }}
```
## Decision Guide
| Scenario | Workers | Shards | Reason |
| -------------------------------- | -------------- | ------ | --------------------------------------- |
| < 50 tests, < 5 min | Auto (default) | None | No optimization needed |
| 50-200 tests, 5-15 min | `'50%'` in CI | 2-4 | Balance speed and cost |
| 200+ tests, > 15 min | `'50%'` in CI | 4-8 | Keep feedback under 10 min |
| Flaky due to resource contention | Reduce to 2 | Keep | Less CPU/memory pressure |
| Tests modify shared database | 1 or isolate | Useful | Sharding splits files; workers run them |
| CI has limited resources | 1 or `'25%'` | More | Compensate with more machines |
| Aspect | Workers (in-process) | Shards (across machines) |
| -------------- | ------------------------- | -------------------------- |
| What it splits | Tests across CPU cores | Test files across CI jobs |
| Controlled by | Config or `--workers` CLI | `--shard=X/Y` CLI flag |
| Shares memory | Yes | No |
| Report merging | Not needed | Required (`merge-reports`) |
| Cost | Free (same machine) | More CI minutes |
## Anti-Patterns
| Anti-Pattern | Problem | Solution |
| --------------------------------------- | ---------------------------------------- | ---------------------------------------------------- |
| `fullyParallel: false` without reason | Tests in files run serially | Set `fullyParallel: true` unless tests need serial |
| `workers: 1` in CI "for safety" | Negates parallelism | Fix isolation issues; use `workers: '50%'` |
| Hardcoded shared user account | Race conditions in parallel runs | Each test creates unique data |
| Sharding without blob reporter | Each shard produces separate HTML report | Configure `reporter: [['blob']]` for CI |
| Sharding with 3 tests | Setup overhead exceeds time saved | Only shard when suite > 5 minutes |
| `test.describe.serial()` everywhere | Kills parallelism, creates dependencies | Use only when tests genuinely need prior state |
| Workers > CPU cores | Context switching overhead | Use `'50%'` or auto-detect |
| Missing `fail-fast: false` in CI matrix | One shard failure cancels others | Always set `fail-fast: false` for sharded strategies |
## Troubleshooting
### Tests pass solo but fail together
- **Shared state**. Make test data unique:
```ts
test("create item", async ({ request }, ti) => {
await request.post("/api/items", {
data: { name: `Item-${ti.workerIndex}-${Date.now()}` },
});
});
```
### "No tests found" in some shards
- **Too many shards**. Never exceed file count:
```bash
npx playwright test --shard=1/10 # ok if 10 files
npx playwright test --shard=1/20 # too many, some shards empty
```
### Merged report missing results
- **Blob reports collide**. Use unique names:
```yaml
# Each shard
- uses: actions/upload-artifact@v4
with:
name: blob-report-${{ strategy.job-index }}
path: blob-report/
# Merge step
- uses: actions/download-artifact@v4
with:
pattern: blob-report-*
merge-multiple: true
path: all-blob-reports
```
### Worker-scoped fixture not working
- **Missing `{ scope: 'worker' }`**. Fix:
```ts
export const test = base.extend({
resource: [
async ({}, use) => {
const r = await Resource.create();
await use(r);
await r.destroy();
},
{ scope: "worker" },
],
});
```
### More workers = Slower
- **Too many workers thrash**. Limit in CI:
```ts
export default defineConfig({
workers: process.env.CI ? 2 : undefined,
});
```

View file

@ -0,0 +1,453 @@
# Performance & Parallelization
## Table of Contents
1. [Parallel Execution](#parallel-execution)
2. [Sharding](#sharding)
3. [Test Optimization](#test-optimization)
4. [Network Optimization](#network-optimization)
5. [Isolation and Parallel Execution](#isolation-and-parallel-execution)
6. [Resource Management](#resource-management)
7. [Benchmarking](#benchmarking)
## Parallel Execution
### Configuration
```typescript
// playwright.config.ts
export default defineConfig({
// Run test files in parallel
fullyParallel: true,
// Number of worker processes
workers: process.env.CI ? 1 : undefined, // undefined = half CPU cores
// Or explicit count
// workers: 4,
// workers: '50%', // Percentage of CPU cores
});
```
### Serial Execution When Needed
```typescript
// Entire file serial
test.describe.configure({ mode: "serial" });
test.describe("Sequential Tests", () => {
test("first", async ({ page }) => {
// Runs first
});
test("second", async ({ page }) => {
// Runs after first
});
});
```
```typescript
// Single describe block serial
test.describe("Parallel Tests", () => {
test("a", async () => {}); // Parallel
test("b", async () => {}); // Parallel
});
test.describe.serial("Serial Tests", () => {
test("c", async () => {}); // Serial
test("d", async () => {}); // Serial
});
```
### Parallel Projects
```typescript
// playwright.config.ts
export default defineConfig({
projects: [
{ name: "chromium", use: { ...devices["Desktop Chrome"] } },
{ name: "firefox", use: { ...devices["Desktop Firefox"] } },
{ name: "webkit", use: { ...devices["Desktop Safari"] } },
],
});
```
```bash
# Run all projects in parallel
npx playwright test
# Run specific project
npx playwright test --project=chromium
```
## Sharding
### Basic Sharding
```bash
# Split tests across 4 machines
# Machine 1:
npx playwright test --shard=1/4
# Machine 2:
npx playwright test --shard=2/4
# Machine 3:
npx playwright test --shard=3/4
# Machine 4:
npx playwright test --shard=4/4
```
### Sharding Strategy
Tests are distributed evenly by file. For optimal sharding:
- Keep test files similar in size
- Use `fullyParallel: true` for even distribution
- Balance slow tests across files
### CI Sharding Pattern
```yaml
# GitHub Actions
jobs:
test:
strategy:
matrix:
shard: [1, 2, 3, 4]
steps:
- run: npx playwright test --shard=${{ matrix.shard }}/4
```
> **For comprehensive CI sharding** (blob reports, merging sharded results, full workflows), see [ci-cd.md](ci-cd.md#sharding).
## Test Optimization
### Reuse Authentication
Avoid logging in for every test. Use setup projects with storage state to authenticate once and reuse the session.
> **For authentication patterns** (storage state, multiple auth states, setup projects), see [fixtures-hooks.md](fixtures-hooks.md#authentication-patterns).
### Reuse Page State (serial only — trade-off with isolation)
Sharing a single page/context across tests with `beforeAll`/`afterAll` is **not recommended** for most suites: it breaks test isolation, causes state leak between tests, and makes failures harder to debug. Prefer a fresh `page` per test (Playwright default). Use shared page only when you explicitly need serial execution and accept no isolation.
```typescript
// ⚠️ Serial only, no isolation: state from one test leaks into the next.
// Prefer test.describe.configure({ mode: 'serial' }) + fresh page per test, or beforeEach + page.goto().
test.describe.configure({ mode: "serial" });
test.describe("Dashboard", () => {
let page: Page;
test.beforeAll(async ({ browser }) => {
const context = await browser.newContext({
storageState: ".auth/user.json",
});
page = await context.newPage();
await page.goto("/dashboard");
});
test.afterAll(async () => {
await page?.close();
});
test("shows stats", async () => {
await expect(page.getByTestId("stats")).toBeVisible();
});
test("shows chart", async () => {
await expect(page.getByTestId("chart")).toBeVisible();
});
});
```
### Lazy Navigation
```typescript
// Bad: Navigate in every test
test("check header", async ({ page }) => {
await page.goto("/products");
await expect(page.getByRole("heading")).toBeVisible();
});
test("check footer", async ({ page }) => {
await page.goto("/products");
await expect(page.getByRole("contentinfo")).toBeVisible();
});
// Good: Share navigation
test.describe("Products Page", () => {
test.beforeEach(async ({ page }) => {
await page.goto("/products");
});
test("check header", async ({ page }) => {
await expect(page.getByRole("heading")).toBeVisible();
});
test("check footer", async ({ page }) => {
await expect(page.getByRole("contentinfo")).toBeVisible();
});
});
```
### Skip Unnecessary Setup
```typescript
// Use test.skip for conditional execution
test("admin feature", async ({ page }) => {
test.skip(!process.env.ADMIN_ENABLED, "Admin features disabled");
// ...
});
// Use test.fixme for known broken tests
test.fixme("broken feature", async ({ page }) => {
// Skipped but tracked
});
```
## Network Optimization
### Mock APIs
```typescript
test.beforeEach(async ({ page }) => {
// Mock slow/heavy endpoints
await page.route("**/api/analytics", (route) =>
route.fulfill({ json: { views: 1000 } }),
);
await page.route("**/api/recommendations", (route) =>
route.fulfill({ json: [] }),
);
});
```
### Block Unnecessary Resources
```typescript
test.beforeEach(async ({ page }) => {
// Block analytics, ads, tracking
await page.route("**/*", (route) => {
const url = route.request().url();
if (
url.includes("google-analytics") ||
url.includes("facebook") ||
url.includes("hotjar")
) {
return route.abort();
}
return route.continue();
});
});
```
### Block Resource Types
```typescript
// Block images and fonts for faster tests
await page.route("**/*", (route) => {
const resourceType = route.request().resourceType();
if (["image", "font", "stylesheet"].includes(resourceType)) {
return route.abort();
}
return route.continue();
});
```
### Cache API Responses
```typescript
const apiCache = new Map<string, object>();
test.beforeEach(async ({ page }) => {
await page.route("**/api/**", async (route) => {
const url = route.request().url();
if (apiCache.has(url)) {
return route.fulfill({ json: apiCache.get(url) });
}
const response = await route.fetch();
const json = await response.json();
apiCache.set(url, json);
return route.fulfill({ json });
});
});
```
## Isolation and Parallel Execution
### Default: one context per test
Playwright gives each test its own browser context (and page). That gives isolation: no shared cookies, storage, or DOM between tests, so failures dont carry over and you can run tests in any order or in parallel. Keep this default unless you have a clear reason to share state.
### Avoiding state leak in parallel runs
- **Do not** rely on shared mutable state (e.g. a single `page` or `context` in `beforeAll`) when tests can run in parallel. State from one test can leak into another and cause flaky, order-dependent failures.
- Use **fixtures** for setup/teardown and **`beforeEach`** for per-test navigation so each test gets a fresh page or a clean slate.
- For **backend or DB state** shared across tests, isolate per worker so parallel workers dont collide. Use a worker-scoped fixture and `testInfo.workerIndex` (or `process.env.TEST_WORKER_INDEX`) to create unique data per worker (e.g. unique user or DB prefix). See [fixtures-hooks.md](../core/fixtures-hooks.md) for worker-scoped fixtures and [debugging.md](../debugging/debugging.md) for debugging flaky parallel runs.
### Debugging flaky parallel runs
If a test is flaky only with multiple workers:
1. **Reproduce**: Run with default workers and `--repeat-each=10` (or `--repeat-each=100 --max-failures=1`).
2. **Confirm parallel-specific**: Run with `--workers=1`. If the failure disappears, the cause is likely shared state or non-isolated backend/DB data.
3. **Fix**: Remove shared page/context; use per-test fixtures and `beforeEach`; isolate test data per worker with `workerIndex` in a worker-scoped fixture.
Workers are restarted after a test failure so subsequent tests in that worker get a clean environment; fixing isolation still prevents the initial flakiness.
## Resource Management
### Browser Contexts
```typescript
// Recommended: One context per test (default) — full isolation
test("isolated test", async ({ page }) => {
// Fresh context automatically
});
// Manual context for specific needs
test("multiple tabs", async ({ browser }) => {
const context = await browser.newContext();
const page1 = await context.newPage();
const page2 = await context.newPage();
// Clean up
await context.close();
});
```
### Memory Management
```typescript
// playwright.config.ts
export default defineConfig({
// Limit concurrent workers
workers: 2,
// Limit parallel tests per worker
use: {
// Lower memory usage
launchOptions: {
args: ["--disable-dev-shm-usage"],
},
},
});
```
### Timeouts
```typescript
// playwright.config.ts
export default defineConfig({
// Global test timeout
timeout: 30000,
// Assertion timeout
expect: {
timeout: 5000,
},
// Navigation timeout
use: {
navigationTimeout: 15000,
actionTimeout: 10000,
},
});
```
## Benchmarking
### Measure Test Duration
```typescript
test("performance test", async ({ page }, testInfo) => {
const startTime = Date.now();
await page.goto("/");
const loadTime = Date.now() - startTime;
console.log(`Page load: ${loadTime}ms`);
// Add to test report
testInfo.annotations.push({
type: "performance",
description: `Load time: ${loadTime}ms`,
});
});
```
### Performance Metrics
```typescript
test("collect metrics", async ({ page }) => {
await page.goto("/");
const metrics = await page.evaluate(() => ({
// Navigation timing
loadTime:
performance.timing.loadEventEnd - performance.timing.navigationStart,
domContentLoaded:
performance.timing.domContentLoadedEventEnd -
performance.timing.navigationStart,
// Performance entries
resources: performance.getEntriesByType("resource").length,
// Memory (Chrome only)
// @ts-ignore
memory: performance.memory?.usedJSHeapSize,
}));
console.log("Metrics:", metrics);
expect(metrics.loadTime).toBeLessThan(3000);
});
```
### Lighthouse Integration
```typescript
import { playAudit } from "playwright-lighthouse";
test("lighthouse audit", async ({ page }) => {
await page.goto("/");
const audit = await playAudit({
page,
thresholds: {
performance: 80,
accessibility: 90,
"best-practices": 80,
seo: 80,
},
port: 9222,
});
expect(audit.lhr.categories.performance.score * 100).toBeGreaterThanOrEqual(
80,
);
});
```
## Performance Checklist
| Optimization | Impact |
| ------------------------------ | ---------- |
| Enable `fullyParallel` | High |
| Reuse authentication | High |
| Mock heavy APIs | High |
| Block tracking scripts | Medium |
| Use sharding in CI | High |
| Reduce workers if memory-bound | Medium |
| Cache API responses | Medium |
| Skip unnecessary tests | Low-Medium |
## Related References
- **CI/CD sharding**: See [ci-cd.md](ci-cd.md) for CI configuration
- **Test organization**: See [test-suite-structure.md](../core/test-suite-structure.md) for structuring tests
- **Fixtures for reuse**: See [fixtures-hooks.md](../core/fixtures-hooks.md) for authentication patterns

View file

@ -0,0 +1,424 @@
# Test Reports & Artifacts
## Table of Contents
1. [CLI Commands](#cli-commands)
2. [Reporter Configuration](#reporter-configuration)
3. [Custom Reporter](#custom-reporter)
4. [Trace Configuration](#trace-configuration)
5. [Screenshot & Video Settings](#screenshot--video-settings)
6. [Artifact Directory Structure](#artifact-directory-structure)
7. [CI Artifact Upload](#ci-artifact-upload)
8. [Decision Guide](#decision-guide)
9. [Anti-Patterns](#anti-patterns)
10. [Troubleshooting](#troubleshooting)
> **When to use**: Configuring test output for debugging, CI dashboards, and team visibility.
## CLI Commands
```bash
# Display last HTML report
npx playwright show-report
# Specify reporter
npx playwright test --reporter=html
npx playwright test --reporter=dot # minimal CI output
npx playwright test --reporter=line # one line per test
npx playwright test --reporter=json # machine-readable
npx playwright test --reporter=junit # CI integration
# Combine reporters
npx playwright test --reporter=dot,html
# Merge sharded reports
npx playwright merge-reports --reporter=html ./blob-report
```
## Reporter Configuration
### Environment-Based Setup
```typescript
// playwright.config.ts
import { defineConfig } from '@playwright/test';
export default defineConfig({
reporter: process.env.CI
? [
['dot'],
['html', { open: 'never' }],
['junit', { outputFile: 'results/junit.xml' }],
['github'],
]
: [
['list'],
['html', { open: 'on-failure' }],
],
});
```
### Reporter Types
| Reporter | Output | Use Case |
|---|---|---|
| `list` | One line per test | Local development |
| `line` | Single updating line | Local, less verbose |
| `dot` | `.` pass, `F` fail | CI logs |
| `html` | Interactive HTML page | Post-run analysis |
| `json` | Machine-readable JSON | Custom tooling |
| `junit` | JUnit XML | CI platforms |
| `github` | PR annotations | GitHub Actions |
| `blob` | Binary archive | Shard merging |
### JSON Output to File
```typescript
import { defineConfig } from '@playwright/test';
export default defineConfig({
reporter: [
['json', { outputFile: 'results/output.json' }],
],
});
```
### JUnit Customization
```typescript
import { defineConfig } from '@playwright/test';
export default defineConfig({
reporter: [
['junit', {
outputFile: 'results/junit.xml',
stripANSIControlSequences: true,
includeProjectInTestName: true,
}],
],
});
```
## Custom Reporter
Build custom reporters for Slack notifications, database logging, or dashboards.
```typescript
// reporters/notification-reporter.ts
import type {
FullResult,
Reporter,
TestCase,
TestResult,
} from '@playwright/test/reporter';
class NotificationReporter implements Reporter {
private passed = 0;
private failed = 0;
private skipped = 0;
private failures: string[] = [];
onTestEnd(test: TestCase, result: TestResult) {
switch (result.status) {
case 'passed':
this.passed++;
break;
case 'failed':
case 'timedOut':
this.failed++;
this.failures.push(`${test.title}: ${result.error?.message?.split('\n')[0]}`);
break;
case 'skipped':
this.skipped++;
break;
}
}
async onEnd(result: FullResult) {
const total = this.passed + this.failed + this.skipped;
const status = this.failed > 0 ? 'FAILED' : 'PASSED';
const message = [
`Tests ${status}`,
`Passed: ${this.passed} | Failed: ${this.failed} | Skipped: ${this.skipped}`,
`Duration: ${(result.duration / 1000).toFixed(1)}s`,
];
if (this.failures.length > 0) {
message.push('', 'Failures:');
this.failures.slice(0, 5).forEach((f) => message.push(` - ${f}`));
if (this.failures.length > 5) {
message.push(` ...and ${this.failures.length - 5} more`);
}
}
const webhookUrl = process.env.NOTIFICATION_WEBHOOK;
if (webhookUrl) {
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), 5000);
try {
await fetch(webhookUrl, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ text: message.join('\n') }),
signal: controller.signal,
});
} catch (error) {
// Intentionally swallow notifier failures to avoid blocking test completion
console.warn('Webhook notification failed:', error.message);
} finally {
clearTimeout(timeout);
}
}
}
}
export default NotificationReporter;
```
**Register custom reporter:**
```typescript
import { defineConfig } from '@playwright/test';
export default defineConfig({
reporter: [
['dot'],
['html', { open: 'never' }],
['./reporters/notification-reporter.ts'],
],
});
```
## Trace Configuration
Traces capture actions, network requests, DOM snapshots, and console logs.
```typescript
import { defineConfig } from '@playwright/test';
export default defineConfig({
retries: process.env.CI ? 2 : 0,
use: {
trace: 'on-first-retry',
},
});
```
### Trace Options
| Value | Behavior | Overhead |
|---|---|---|
| `'off'` | Never records | None |
| `'on'` | Every test | High |
| `'on-first-retry'` | On first retry after failure | Minimal |
| `'retain-on-failure'` | Records all, keeps failures | Medium |
| `'retain-on-first-failure'` | Records all, keeps first failure | Medium |
### Viewing Traces
```bash
# Local trace viewer
npx playwright show-trace results/my-test/trace.zip
# From HTML report (click Traces tab)
npx playwright show-report
# Online viewer: https://trace.playwright.dev
```
## Screenshot & Video Settings
```typescript
import { defineConfig } from '@playwright/test';
export default defineConfig({
use: {
screenshot: 'only-on-failure',
video: 'retain-on-failure',
},
});
```
### Video with Custom Size
```typescript
use: {
video: {
mode: 'retain-on-failure',
size: { width: 1280, height: 720 },
},
},
```
### Screenshot Options
| Value | Captures | Disk Cost |
|---|---|---|
| `'off'` | Never | None |
| `'on'` | Every test | High |
| `'only-on-failure'` | Failed tests | Low |
### Video Options
| Value | Records | Keeps | Disk Cost |
|---|---|---|---|
| `'off'` | Never | — | None |
| `'on'` | Every test | All | Very high |
| `'on-first-retry'` | On retry | Retried | Low |
| `'retain-on-failure'` | Every test | Failed | Medium |
## Artifact Directory Structure
```text
test-results/
├── checkout-test-chromium/
│ ├── trace.zip
│ ├── test-failed-1.png
│ └── video.webm
├── login-test-firefox/
│ ├── trace.zip
│ └── test-failed-1.png
└── junit.xml
playwright-report/
├── index.html
└── data/
blob-report/
└── report-1.zip
```
## CI Artifact Upload
### GitHub Actions
```yaml
- uses: actions/upload-artifact@v4
if: ${{ !cancelled() }}
with:
name: playwright-report
path: playwright-report/
retention-days: 14
- uses: actions/upload-artifact@v4
if: failure()
with:
name: test-traces
path: |
test-results/**/trace.zip
test-results/**/*.png
test-results/**/*.webm
retention-days: 7
```
## Decision Guide
| Scenario | Reporter Configuration |
|---|---|
| Local development | `[['list'], ['html', { open: 'on-failure' }]]` |
| GitHub Actions | `[['dot'], ['html'], ['github']]` |
| GitLab CI | `[['dot'], ['html'], ['junit']]` |
| Azure DevOps / Jenkins | `[['dot'], ['html'], ['junit']]` |
| Sharded CI | `[['blob'], ['github']]` |
| Custom dashboard | `[['json', { outputFile: '...' }]]` + custom reporter |
| Artifact | When to Collect | Retention | Upload Condition |
|---|---|---|---|
| HTML report | Always | 14 days | `if: ${{ !cancelled() }}` |
| Traces | On failure | 7 days | `if: failure()` |
| Screenshots | On failure | 7 days | `if: failure()` |
| Videos | On failure | 7 days | `if: failure()` |
| JUnit XML | Always | 14 days | `if: ${{ !cancelled() }}` |
| Blob report | Always (sharded) | 1 day | `if: ${{ !cancelled() }}` |
## Anti-Patterns
| Anti-Pattern | Problem | Solution |
|---|---|---|
| No reporter configured | Default `list` only; no persistent report | Configure `html` + CI reporter |
| `trace: 'on'` in CI | Massive artifacts, slow uploads | Use `trace: 'on-first-retry'` |
| `video: 'on'` in CI | Enormous storage, slower tests | Use `video: 'retain-on-failure'` |
| Upload artifacts only on failure | No report when tests pass | Upload with `if: ${{ !cancelled() }}` |
| No retention limits | CI storage fills quickly | Set `retention-days: 7-14` |
| Only `dot` reporter | Cannot drill into failures | Pair `dot` with `html` |
| JUnit to stdout | Interferes with console output | Write to file |
| Blocking `onEnd` in custom reporter | Slow HTTP calls delay pipeline | Use `Promise.race` with timeout |
## Troubleshooting
### Empty HTML Report
Check reporter config. HTML report defaults to `playwright-report/`:
```typescript
import { defineConfig } from '@playwright/test';
export default defineConfig({
reporter: [['html', { outputFolder: 'playwright-report', open: 'never' }]],
});
```
### Traces Too Large
Switch from `trace: 'on'` to `'on-first-retry'` with retries enabled:
```typescript
import { defineConfig } from '@playwright/test';
export default defineConfig({
retries: process.env.CI ? 2 : 0,
use: {
trace: 'on-first-retry',
},
});
```
### JUnit XML Not Recognized
Ensure path matches CI configuration:
```typescript
reporter: [['junit', { outputFile: 'results/junit.xml' }]],
```
```yaml
# GitHub Actions
- uses: dorny/test-reporter@latest
with:
path: results/junit.xml
reporter: java-junit
# Azure DevOps
- task: PublishTestResults@latest
inputs:
testResultsFiles: 'results/junit.xml'
# Jenkins
junit 'results/junit.xml'
```
### Empty Merged Report
Use `blob` reporter for sharded runs (not `html`):
```typescript
import { defineConfig } from '@playwright/test';
export default defineConfig({
reporter: process.env.CI
? [['blob'], ['dot']]
: [['html', { open: 'on-failure' }]],
});
```
### Missing Screenshots in Report
Enable screenshots and keep both directories:
```typescript
use: {
screenshot: 'only-on-failure',
},
```
The HTML report embeds screenshots from `test-results/`. Deleting that directory removes screenshots from the report.

View file

@ -0,0 +1,497 @@
# Test Coverage
## Table of Contents
1. [Coverage Setup](#coverage-setup)
2. [Collecting Coverage](#collecting-coverage)
3. [Coverage Reports](#coverage-reports)
4. [Coverage Thresholds](#coverage-thresholds)
5. [Advanced Patterns](#advanced-patterns)
6. [CI Integration](#ci-integration)
## Coverage Setup
### Install Dependencies
```bash
# For V8 coverage (built into Playwright)
# No additional dependencies needed
# For Istanbul-based coverage (more features)
npm install -D nyc @istanbuljs/nyc-config-typescript
```
### Basic Configuration
```typescript
// playwright.config.ts
import { defineConfig } from "@playwright/test";
export default defineConfig({
use: {
// Enable coverage collection
contextOptions: {
// V8 coverage is automatic with the API below
},
},
});
```
### V8 Coverage Fixture
```typescript
// fixtures/coverage.ts
import { test as base, expect } from "@playwright/test";
import fs from "fs";
import path from "path";
import { randomUUID } from "crypto";
export const test = base.extend<{}, { collectCoverage: void }>({
collectCoverage: [
async ({ browser }, use) => {
// Start coverage for all pages
const context = await browser.newContext();
const page = await context.newPage();
await page.coverage.startJSCoverage();
await page.coverage.startCSSCoverage();
await use();
// Collect coverage
const [jsCoverage, cssCoverage] = await Promise.all([
page.coverage.stopJSCoverage(),
page.coverage.stopCSSCoverage(),
]);
// Save coverage data
const coverageDir = "./coverage";
if (!fs.existsSync(coverageDir)) {
fs.mkdirSync(coverageDir, { recursive: true });
}
fs.writeFileSync(
path.join(coverageDir, `coverage-${randomUUID()}.json`),
JSON.stringify([...jsCoverage, ...cssCoverage])
);
await context.close();
},
{ scope: "worker", auto: true },
],
});
```
## Collecting Coverage
### Per-Test Coverage
```typescript
test("collect coverage for single test", async ({ page }) => {
// Start coverage collection
await page.coverage.startJSCoverage({
resetOnNavigation: false,
});
// Run test
await page.goto("/app");
await page.getByRole("button", { name: "Submit" }).click();
await expect(page.getByText("Success")).toBeVisible();
// Stop and get coverage
const coverage = await page.coverage.stopJSCoverage();
// Filter to only your source files
const appCoverage = coverage.filter((entry) => entry.url.includes("/src/"));
console.log(`Covered ${appCoverage.length} source files`);
});
```
### Coverage for Specific Files
```typescript
test("track specific module coverage", async ({ page }) => {
await page.coverage.startJSCoverage();
await page.goto("/checkout");
await page.getByRole("button", { name: "Pay" }).click();
const coverage = await page.coverage.stopJSCoverage();
// Find coverage for checkout module
const checkoutCoverage = coverage.find((c) => c.url.includes("checkout.js"));
if (checkoutCoverage) {
const totalBytes = checkoutCoverage.text?.length || 0;
const coveredBytes = checkoutCoverage.ranges.reduce(
(sum, range) => sum + (range.end - range.start),
0
);
const percentage = (coveredBytes / totalBytes) * 100;
console.log(`Checkout module: ${percentage.toFixed(1)}% covered`);
expect(percentage).toBeGreaterThan(80);
}
});
```
### CSS Coverage
```typescript
test("collect CSS coverage", async ({ page }) => {
await page.coverage.startCSSCoverage();
await page.goto("/app");
// Interact to trigger different CSS states
await page.getByRole("button").hover();
await page.getByRole("dialog").waitFor();
const cssCoverage = await page.coverage.stopCSSCoverage();
// Find unused CSS
for (const entry of cssCoverage) {
const totalBytes = entry.text?.length || 0;
const usedBytes = entry.ranges.reduce(
(sum, range) => sum + (range.end - range.start),
0
);
const unusedPercentage = ((totalBytes - usedBytes) / totalBytes) * 100;
if (unusedPercentage > 50) {
console.warn(`${entry.url}: ${unusedPercentage.toFixed(1)}% unused CSS`);
}
}
});
```
## Coverage Reports
### Converting to Istanbul Format
```typescript
// scripts/convert-coverage.ts
import { execSync } from "child_process";
import fs from "fs";
import path from "path";
import v8ToIstanbul from "v8-to-istanbul";
async function convertCoverage() {
const coverageDir = "./coverage";
const files = fs.readdirSync(coverageDir).filter((f) => f.endsWith(".json"));
const istanbulCoverage: any = {};
for (const file of files) {
const coverageData = JSON.parse(
fs.readFileSync(path.join(coverageDir, file), "utf-8")
);
for (const entry of coverageData) {
if (!entry.url.startsWith("file://")) continue;
const filePath = entry.url.replace("file://", "");
const converter = v8ToIstanbul(filePath);
await converter.load();
converter.applyCoverage(entry.functions || []);
const istanbul = converter.toIstanbul();
Object.assign(istanbulCoverage, istanbul);
}
}
fs.writeFileSync(
path.join(coverageDir, "coverage-final.json"),
JSON.stringify(istanbulCoverage)
);
}
convertCoverage();
```
### Generating HTML Report
```bash
# Using nyc to generate report
npx nyc report --reporter=html --reporter=text --temp-dir=./coverage
```
```typescript
// package.json scripts
{
"scripts": {
"test": "playwright test",
"test:coverage": "playwright test && npm run coverage:report",
"coverage:report": "npx nyc report --reporter=html --reporter=lcov --temp-dir=./coverage"
}
}
```
### Custom Coverage Reporter
```typescript
// reporters/coverage-reporter.ts
import type { Reporter, FullResult } from "@playwright/test/reporter";
import fs from "fs";
import path from "path";
class CoverageReporter implements Reporter {
private coverageData: any[] = [];
onEnd(result: FullResult) {
// Aggregate all coverage files
const coverageDir = "./coverage";
const files = fs
.readdirSync(coverageDir)
.filter((f) => f.endsWith(".json"));
for (const file of files) {
const data = JSON.parse(
fs.readFileSync(path.join(coverageDir, file), "utf-8")
);
this.coverageData.push(...data);
}
// Generate summary
const summary = this.generateSummary();
console.log("\n📊 Coverage Summary:");
console.log(` Files: ${summary.totalFiles}`);
console.log(` Lines: ${summary.lineCoverage.toFixed(1)}%`);
console.log(` Bytes: ${summary.byteCoverage.toFixed(1)}%`);
if (summary.lineCoverage < 80) {
console.warn("⚠️ Coverage below 80% threshold!");
}
}
private generateSummary() {
let totalBytes = 0;
let coveredBytes = 0;
const files = new Set<string>();
for (const entry of this.coverageData) {
if (entry.url.includes("/src/")) {
files.add(entry.url);
totalBytes += entry.text?.length || 0;
coveredBytes += entry.ranges.reduce(
(sum: number, r: any) => sum + (r.end - r.start),
0
);
}
}
return {
totalFiles: files.size,
byteCoverage: (coveredBytes / totalBytes) * 100,
lineCoverage: (coveredBytes / totalBytes) * 100, // Simplified
};
}
}
export default CoverageReporter;
```
## Coverage Thresholds
### Enforcing Minimum Coverage
```typescript
// tests/coverage.spec.ts
import { test, expect } from "@playwright/test";
import fs from "fs";
import path from "path";
test.afterAll(async () => {
const coverageDir = "./coverage";
const files = fs.readdirSync(coverageDir).filter((f) => f.endsWith(".json"));
let totalBytes = 0;
let coveredBytes = 0;
for (const file of files) {
const coverage = JSON.parse(
fs.readFileSync(path.join(coverageDir, file), "utf-8")
);
for (const entry of coverage) {
if (!entry.url.includes("/src/")) continue;
totalBytes += entry.text?.length || 0;
coveredBytes += entry.ranges.reduce(
(sum: number, r: any) => sum + (r.end - r.start),
0
);
}
}
const coveragePercent = (coveredBytes / totalBytes) * 100;
// Enforce threshold
expect(coveragePercent).toBeGreaterThan(80);
});
```
### Per-Directory Thresholds
```typescript
// coverage-check.ts
interface CoverageThreshold {
pattern: RegExp;
minCoverage: number;
}
const thresholds: CoverageThreshold[] = [
{ pattern: /\/src\/core\//, minCoverage: 90 },
{ pattern: /\/src\/utils\//, minCoverage: 85 },
{ pattern: /\/src\/components\//, minCoverage: 70 },
{ pattern: /\/src\/pages\//, minCoverage: 60 },
];
function checkThresholds(coverage: any[]): string[] {
const violations: string[] = [];
for (const threshold of thresholds) {
const matchingFiles = coverage.filter((c) => threshold.pattern.test(c.url));
let total = 0;
let covered = 0;
for (const file of matchingFiles) {
total += file.text?.length || 0;
covered += file.ranges.reduce(
(sum: number, r: any) => sum + (r.end - r.start),
0
);
}
const percent = total > 0 ? (covered / total) * 100 : 0;
if (percent < threshold.minCoverage) {
violations.push(
`${threshold.pattern}: ${percent.toFixed(1)}% < ${
threshold.minCoverage
}%`
);
}
}
return violations;
}
```
## Advanced Patterns
### Merging Coverage Across Shards
```typescript
// scripts/merge-coverage.ts
import fs from "fs";
import { glob } from "glob";
async function mergeCoverage() {
const files = await glob("shard-*/coverage/*.json");
const merged = new Map<string, any>();
for (const file of files) {
const data = JSON.parse(fs.readFileSync(file, "utf-8"));
for (const entry of data) {
if (merged.has(entry.url)) {
const existing = merged.get(entry.url);
existing.ranges.push(...entry.ranges);
} else {
merged.set(entry.url, { ...entry });
}
}
}
fs.writeFileSync(
"./coverage/merged.json",
JSON.stringify([...merged.values()])
);
}
mergeCoverage();
```
### Incremental Coverage
```typescript
// Check coverage only for changed files in CI
import { execSync } from "child_process";
import fs from "fs";
const changedFiles = execSync("git diff --name-only HEAD~1")
.toString()
.split("\n")
.filter((f) => f.endsWith(".ts"));
const coverage = JSON.parse(fs.readFileSync("./coverage/merged.json", "utf-8"));
for (const file of changedFiles) {
const entry = coverage.find((c: any) => c.url.includes(file));
if (entry) {
const percent =
(entry.ranges.reduce((s: number, r: any) => s + r.end - r.start, 0) /
(entry.text?.length || 1)) *
100;
console.log(`${file}: ${percent.toFixed(1)}%`);
}
}
```
## CI Integration
### GitHub Actions
```yaml
# .github/workflows/test.yml
name: Tests with Coverage
on: [push, pull_request]
jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 22
- run: npm ci
- run: npx playwright install --with-deps
- name: Run tests with coverage
run: npm run test:coverage
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3
with:
files: ./coverage/lcov.info
fail_ci_if_error: true
- name: Check coverage threshold
run: |
COVERAGE=$(cat coverage/coverage-summary.json | jq '.total.lines.pct')
if (( $(echo "$COVERAGE < 80" | bc -l) )); then
echo "Coverage $COVERAGE% is below 80% threshold"
exit 1
fi
```
## Anti-Patterns to Avoid
| Anti-Pattern | Problem | Solution |
| ---------------------------- | -------------------------------------- | --------------------------- |
| Coverage for coverage's sake | Gaming metrics | Focus on critical paths |
| 100% coverage target | Diminishing returns, tests for getters | Set realistic thresholds |
| Ignoring coverage drops | Technical debt | Enforce thresholds in CI |
| No source map support | Wrong line numbers | Enable source maps in build |
| Coverage only in CI | Late feedback | Run locally too |
## Related References
- **CI/CD**: See [ci-cd.md](ci-cd.md) for pipeline configuration
- **Performance**: See [performance.md](performance.md) for optimizing coverage collection