mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-05-13 01:32:40 +02:00
chore: add playwright cursor skill
This commit is contained in:
Anish Sarkar
parent
25aad38ca4
commit
d52225c18d
57 changed files with 25244 additions and 0 deletions
420
.cursor/skills/playwright-testing/debugging/console-errors.md
Normal file
420
.cursor/skills/playwright-testing/debugging/console-errors.md
Normal file
|
|
@ -0,0 +1,420 @@
|
|||
# Browser Console & JavaScript Error Handling
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [Capturing Console Messages](#capturing-console-messages)
|
||||
2. [Failing on Console Errors](#failing-on-console-errors)
|
||||
3. [JavaScript Error Detection](#javascript-error-detection)
|
||||
4. [Monitoring Warnings](#monitoring-warnings)
|
||||
5. [Console Fixtures](#console-fixtures)
|
||||
|
||||
## Capturing Console Messages
|
||||
|
||||
### Basic Console Capture
|
||||
|
||||
```typescript
|
||||
test("capture console logs", async ({ page }) => {
|
||||
const logs: string[] = [];
|
||||
|
||||
page.on("console", (msg) => {
|
||||
logs.push(`${msg.type()}: ${msg.text()}`);
|
||||
});
|
||||
|
||||
await page.goto("/");
|
||||
|
||||
// Check what was logged
|
||||
console.log("Captured logs:", logs);
|
||||
});
|
||||
```
|
||||
|
||||
### Capture by Type
|
||||
|
||||
```typescript
|
||||
test("capture specific console types", async ({ page }) => {
|
||||
const errors: string[] = [];
|
||||
const warnings: string[] = [];
|
||||
const infos: string[] = [];
|
||||
|
||||
page.on("console", (msg) => {
|
||||
switch (msg.type()) {
|
||||
case "error":
|
||||
errors.push(msg.text());
|
||||
break;
|
||||
case "warning":
|
||||
warnings.push(msg.text());
|
||||
break;
|
||||
case "info":
|
||||
case "log":
|
||||
infos.push(msg.text());
|
||||
break;
|
||||
}
|
||||
});
|
||||
|
||||
await page.goto("/dashboard");
|
||||
|
||||
expect(errors).toHaveLength(0);
|
||||
console.log("Warnings:", warnings);
|
||||
});
|
||||
```
|
||||
|
||||
### Capture with Stack Trace
|
||||
|
||||
```typescript
|
||||
test("capture errors with location", async ({ page }) => {
|
||||
const errors: { message: string; location?: string }[] = [];
|
||||
|
||||
page.on("console", async (msg) => {
|
||||
if (msg.type() === "error") {
|
||||
const location = msg.location();
|
||||
errors.push({
|
||||
message: msg.text(),
|
||||
location: location
|
||||
? `${location.url}:${location.lineNumber}`
|
||||
: undefined,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
await page.goto("/buggy-page");
|
||||
|
||||
// Log errors with source location
|
||||
errors.forEach((e) => {
|
||||
console.log(`Error: ${e.message}`);
|
||||
if (e.location) console.log(` at ${e.location}`);
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
## Failing on Console Errors
|
||||
|
||||
### Fail Test on Any Error
|
||||
|
||||
```typescript
|
||||
test("no console errors allowed", async ({ page }) => {
|
||||
const errors: string[] = [];
|
||||
|
||||
page.on("console", (msg) => {
|
||||
if (msg.type() === "error") {
|
||||
errors.push(msg.text());
|
||||
}
|
||||
});
|
||||
|
||||
await page.goto("/");
|
||||
await page.getByRole("button", { name: "Load Data" }).click();
|
||||
|
||||
// Fail if any console errors
|
||||
expect(errors, `Console errors found:\n${errors.join("\n")}`).toHaveLength(0);
|
||||
});
|
||||
```
|
||||
|
||||
### Fail with Allowed Exceptions
|
||||
|
||||
```typescript
|
||||
test("no unexpected console errors", async ({ page }) => {
|
||||
const allowedErrors = [
|
||||
/Failed to load resource.*favicon/,
|
||||
/ResizeObserver loop/,
|
||||
];
|
||||
|
||||
const unexpectedErrors: string[] = [];
|
||||
|
||||
page.on("console", (msg) => {
|
||||
if (msg.type() === "error") {
|
||||
const text = msg.text();
|
||||
const isAllowed = allowedErrors.some((pattern) => pattern.test(text));
|
||||
if (!isAllowed) {
|
||||
unexpectedErrors.push(text);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
await page.goto("/");
|
||||
|
||||
expect(
|
||||
unexpectedErrors,
|
||||
`Unexpected console errors:\n${unexpectedErrors.join("\n")}`,
|
||||
).toHaveLength(0);
|
||||
});
|
||||
```
|
||||
|
||||
### Auto-Fail Fixture
|
||||
|
||||
```typescript
|
||||
// fixtures/console.fixture.ts
|
||||
type ConsoleFixtures = {
|
||||
failOnConsoleError: void;
|
||||
};
|
||||
|
||||
export const test = base.extend<ConsoleFixtures>({
|
||||
failOnConsoleError: [
|
||||
async ({ page }, use, testInfo) => {
|
||||
const errors: string[] = [];
|
||||
|
||||
page.on("console", (msg) => {
|
||||
if (msg.type() === "error") {
|
||||
errors.push(msg.text());
|
||||
}
|
||||
});
|
||||
|
||||
await use();
|
||||
|
||||
// After test, check for errors
|
||||
if (errors.length > 0) {
|
||||
testInfo.annotations.push({
|
||||
type: "console-errors",
|
||||
description: errors.join("\n"),
|
||||
});
|
||||
throw new Error(`Console errors detected:\n${errors.join("\n")}`);
|
||||
}
|
||||
},
|
||||
{ auto: true }, // Runs for every test
|
||||
],
|
||||
});
|
||||
```
|
||||
|
||||
## JavaScript Error Detection
|
||||
|
||||
### Catch Uncaught Exceptions
|
||||
|
||||
```typescript
|
||||
test("no uncaught exceptions", async ({ page }) => {
|
||||
const pageErrors: Error[] = [];
|
||||
|
||||
page.on("pageerror", (error) => {
|
||||
pageErrors.push(error);
|
||||
});
|
||||
|
||||
await page.goto("/");
|
||||
await page.getByRole("button", { name: "Trigger Action" }).click();
|
||||
|
||||
expect(
|
||||
pageErrors,
|
||||
`Uncaught exceptions:\n${pageErrors.map((e) => e.message).join("\n")}`,
|
||||
).toHaveLength(0);
|
||||
});
|
||||
```
|
||||
|
||||
### Capture Error Details
|
||||
|
||||
```typescript
|
||||
test("capture JS error details", async ({ page }) => {
|
||||
const errors: { message: string; stack?: string }[] = [];
|
||||
|
||||
page.on("pageerror", (error) => {
|
||||
errors.push({
|
||||
message: error.message,
|
||||
stack: error.stack,
|
||||
});
|
||||
});
|
||||
|
||||
await page.goto("/error-page");
|
||||
|
||||
if (errors.length > 0) {
|
||||
console.log("JavaScript errors:");
|
||||
errors.forEach((e) => {
|
||||
console.log(` Message: ${e.message}`);
|
||||
console.log(` Stack: ${e.stack}`);
|
||||
});
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
### Test Error Boundary Triggers
|
||||
|
||||
```typescript
|
||||
test("error boundary catches render error", async ({ page }) => {
|
||||
let errorCaught = false;
|
||||
|
||||
page.on("pageerror", () => {
|
||||
// Note: React error boundaries catch errors before they become pageerrors
|
||||
// This would only fire for unhandled errors
|
||||
errorCaught = true;
|
||||
});
|
||||
|
||||
// Trigger component error via props
|
||||
await page.route(
|
||||
"**/api/data",
|
||||
(route) => route.fulfill({ json: null }), // Will cause "cannot read property of null"
|
||||
);
|
||||
|
||||
await page.goto("/dashboard");
|
||||
|
||||
// Error boundary should show fallback, not crash
|
||||
await expect(page.getByText("Something went wrong")).toBeVisible();
|
||||
expect(errorCaught).toBe(false); // Error was caught by boundary
|
||||
});
|
||||
```
|
||||
|
||||
## Monitoring Warnings
|
||||
|
||||
### Capture Deprecation Warnings
|
||||
|
||||
```typescript
|
||||
test("no deprecation warnings", async ({ page }) => {
|
||||
const deprecations: string[] = [];
|
||||
|
||||
page.on("console", (msg) => {
|
||||
const text = msg.text();
|
||||
if (
|
||||
msg.type() === "warning" &&
|
||||
(text.includes("deprecated") || text.includes("Deprecation"))
|
||||
) {
|
||||
deprecations.push(text);
|
||||
}
|
||||
});
|
||||
|
||||
await page.goto("/");
|
||||
|
||||
if (deprecations.length > 0) {
|
||||
console.warn("Deprecation warnings found:");
|
||||
deprecations.forEach((d) => console.warn(` - ${d}`));
|
||||
}
|
||||
|
||||
// Optionally fail
|
||||
// expect(deprecations).toHaveLength(0);
|
||||
});
|
||||
```
|
||||
|
||||
### React Development Warnings
|
||||
|
||||
```typescript
|
||||
test("no React warnings", async ({ page }) => {
|
||||
const reactWarnings: string[] = [];
|
||||
|
||||
page.on("console", (msg) => {
|
||||
const text = msg.text();
|
||||
if (
|
||||
msg.type() === "warning" &&
|
||||
(text.includes("Warning:") || text.includes("React"))
|
||||
) {
|
||||
reactWarnings.push(text);
|
||||
}
|
||||
});
|
||||
|
||||
await page.goto("/");
|
||||
|
||||
// Common React warnings to check
|
||||
const criticalWarnings = reactWarnings.filter(
|
||||
(w) =>
|
||||
w.includes("Each child in a list should have a unique") ||
|
||||
w.includes("Cannot update a component") ||
|
||||
w.includes("Can't perform a React state update"),
|
||||
);
|
||||
|
||||
expect(
|
||||
criticalWarnings,
|
||||
`React warnings:\n${criticalWarnings.join("\n")}`,
|
||||
).toHaveLength(0);
|
||||
});
|
||||
```
|
||||
|
||||
## Console Fixtures
|
||||
|
||||
### Comprehensive Console Fixture
|
||||
|
||||
```typescript
|
||||
// fixtures/console.fixture.ts
|
||||
type ConsoleMessage = {
|
||||
type: string;
|
||||
text: string;
|
||||
location?: { url: string; line: number };
|
||||
timestamp: number;
|
||||
};
|
||||
|
||||
type ConsoleFixtures = {
|
||||
consoleMessages: ConsoleMessage[];
|
||||
getConsoleErrors: () => ConsoleMessage[];
|
||||
getConsoleWarnings: () => ConsoleMessage[];
|
||||
assertNoErrors: (allowedPatterns?: RegExp[]) => void;
|
||||
};
|
||||
|
||||
export const test = base.extend<ConsoleFixtures>({
|
||||
consoleMessages: async ({ page }, use) => {
|
||||
const messages: ConsoleMessage[] = [];
|
||||
|
||||
page.on("console", (msg) => {
|
||||
const location = msg.location();
|
||||
messages.push({
|
||||
type: msg.type(),
|
||||
text: msg.text(),
|
||||
location: location
|
||||
? { url: location.url, line: location.lineNumber }
|
||||
: undefined,
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
});
|
||||
|
||||
await use(messages);
|
||||
},
|
||||
|
||||
getConsoleErrors: async ({ consoleMessages }, use) => {
|
||||
await use(() => consoleMessages.filter((m) => m.type === "error"));
|
||||
},
|
||||
|
||||
getConsoleWarnings: async ({ consoleMessages }, use) => {
|
||||
await use(() => consoleMessages.filter((m) => m.type === "warning"));
|
||||
},
|
||||
|
||||
assertNoErrors: async ({ getConsoleErrors }, use) => {
|
||||
await use((allowedPatterns = []) => {
|
||||
const errors = getConsoleErrors();
|
||||
const unexpected = errors.filter(
|
||||
(e) => !allowedPatterns.some((p) => p.test(e.text)),
|
||||
);
|
||||
|
||||
if (unexpected.length > 0) {
|
||||
throw new Error(
|
||||
`Unexpected console errors:\n${unexpected.map((e) => e.text).join("\n")}`,
|
||||
);
|
||||
}
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
// Usage
|
||||
test("page loads without errors", async ({ page, assertNoErrors }) => {
|
||||
await page.goto("/dashboard");
|
||||
await page.getByRole("button", { name: "Load" }).click();
|
||||
|
||||
assertNoErrors([/favicon/]); // Allow favicon errors
|
||||
});
|
||||
```
|
||||
|
||||
### Attach Console to Report
|
||||
|
||||
```typescript
|
||||
test("capture console for debugging", async ({ page }, testInfo) => {
|
||||
const logs: string[] = [];
|
||||
|
||||
page.on("console", (msg) => {
|
||||
logs.push(`[${msg.type()}] ${msg.text()}`);
|
||||
});
|
||||
|
||||
page.on("pageerror", (error) => {
|
||||
logs.push(`[EXCEPTION] ${error.message}`);
|
||||
});
|
||||
|
||||
await page.goto("/");
|
||||
// ... test actions
|
||||
|
||||
// Attach console log to test report
|
||||
await testInfo.attach("console-log", {
|
||||
body: logs.join("\n"),
|
||||
contentType: "text/plain",
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
## Anti-Patterns to Avoid
|
||||
|
||||
| Anti-Pattern | Problem | Solution |
|
||||
| -------------------------- | -------------------------- | --------------------------- |
|
||||
| Ignoring console errors | Bugs go unnoticed | Check for errors in tests |
|
||||
| Too strict error checking | Tests fail on minor issues | Allow known/expected errors |
|
||||
| Not capturing stack traces | Hard to debug | Include location info |
|
||||
| Checking only at end | Miss errors during actions | Capture continuously |
|
||||
|
||||
## Related References
|
||||
|
||||
- **Debugging**: See [debugging.md](debugging.md) for troubleshooting
|
||||
- **Error Testing**: See [error-testing.md](error-testing.md) for error scenarios
|
||||
504
.cursor/skills/playwright-testing/debugging/debugging.md
Normal file
504
.cursor/skills/playwright-testing/debugging/debugging.md
Normal file
|
|
@ -0,0 +1,504 @@
|
|||
# Debugging & Troubleshooting
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [Debug Tools](#debug-tools)
|
||||
2. [Trace Viewer](#trace-viewer)
|
||||
3. [Identifying Flaky Tests](#identifying-flaky-tests)
|
||||
4. [Debugging Network Issues](#debugging-network-issues)
|
||||
5. [Debugging in CI](#debugging-in-ci)
|
||||
6. [Debugging Authentication](#debugging-authentication)
|
||||
7. [Debugging Screenshots](#debugging-screenshots)
|
||||
8. [Common Issues](#common-issues)
|
||||
9. [Logging](#logging)
|
||||
|
||||
## Debug Tools
|
||||
|
||||
### Playwright Inspector
|
||||
|
||||
```bash
|
||||
# Run with inspector
|
||||
PWDEBUG=1 npx playwright test
|
||||
# Or specific test
|
||||
PWDEBUG=1 npx playwright test login.spec.ts
|
||||
```
|
||||
|
||||
Features:
|
||||
|
||||
- Step through test actions
|
||||
- Pick locators visually
|
||||
- Inspect DOM state
|
||||
- Edit and re-run
|
||||
|
||||
### Headed Mode
|
||||
|
||||
```bash
|
||||
# Run with visible browser
|
||||
npx playwright test --headed
|
||||
|
||||
# Interactive debugging (headed, paused, step-through)
|
||||
npx playwright test --debug
|
||||
```
|
||||
|
||||
You can also set `slowMo` to add an `N` ms delay per action, making test execution easier to follow while debugging.
|
||||
|
||||
```typescript
|
||||
// playwright.config.ts
|
||||
export default defineConfig({
|
||||
use: {
|
||||
launchOptions: {
|
||||
slowMo: 500,
|
||||
},
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
### UI Mode
|
||||
|
||||
```bash
|
||||
# Interactive test runner
|
||||
npx playwright test --ui
|
||||
```
|
||||
|
||||
Features:
|
||||
|
||||
- Watch mode
|
||||
- Test timeline
|
||||
- DOM snapshots
|
||||
- Network logs
|
||||
- Console logs
|
||||
|
||||
### Debug in Code
|
||||
|
||||
```typescript
|
||||
test("debug example", async ({ page }) => {
|
||||
await page.goto("/");
|
||||
|
||||
// Pause and open inspector
|
||||
await page.pause();
|
||||
|
||||
// Continue test...
|
||||
await page.click("button");
|
||||
});
|
||||
```
|
||||
|
||||
## Trace Viewer
|
||||
|
||||
### Enable Traces
|
||||
|
||||
```typescript
|
||||
// playwright.config.ts
|
||||
export default defineConfig({
|
||||
use: {
|
||||
trace: "on-first-retry", // Record on retry
|
||||
// trace: 'on', // Always record
|
||||
// trace: 'retain-on-failure', // Keep only failures
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
### View Traces
|
||||
|
||||
```bash
|
||||
# Open trace file
|
||||
npx playwright show-trace trace.zip
|
||||
|
||||
# From test-results
|
||||
npx playwright show-trace test-results/test-name/trace.zip
|
||||
```
|
||||
|
||||
### Trace Contents
|
||||
|
||||
- Screenshots at each action
|
||||
- DOM snapshots
|
||||
- Network requests/responses
|
||||
- Console logs
|
||||
- Action timeline
|
||||
- Source code
|
||||
|
||||
### Programmatic Traces
|
||||
|
||||
```typescript
|
||||
test("manual trace", async ({ page, context }) => {
|
||||
await context.tracing.start({ screenshots: true, snapshots: true });
|
||||
|
||||
await page.goto("/");
|
||||
await page.click("button");
|
||||
|
||||
await context.tracing.stop({ path: "trace.zip" });
|
||||
});
|
||||
```
|
||||
|
||||
## Identifying Flaky Tests
|
||||
|
||||
If a test fails intermittently, it's likely flaky. Quick checks:
|
||||
|
||||
| Behavior | Likely Cause | Next Step |
|
||||
| -------------------------------------- | ----------------------------- | -------------------------------------- |
|
||||
| Fails sometimes, passes other times | Flaky - timing/race condition | [flaky-tests.md](flaky-tests.md) |
|
||||
| Fails only with multiple workers | Flaky - parallelism/isolation | [flaky-tests.md](flaky-tests.md) |
|
||||
| Fails only in CI | Environment difference | [CI Debugging](#debugging-in-ci) below |
|
||||
| Always fails | Bug in test or app | Debug with tools above |
|
||||
| Always passes locally, always fails CI | CI-specific issue | [ci-cd.md](../infrastructure-ci-cd/ci-cd.md) |
|
||||
|
||||
> **For flaky test detection commands, root cause analysis, and fixing strategies**, see [flaky-tests.md](flaky-tests.md).
|
||||
|
||||
## Debugging Network Issues
|
||||
|
||||
### Monitor All Requests
|
||||
|
||||
```typescript
|
||||
test("debug network", async ({ page }) => {
|
||||
const requests: string[] = [];
|
||||
const failures: string[] = [];
|
||||
|
||||
page.on("request", (req) => requests.push(`>> ${req.method()} ${req.url()}`));
|
||||
page.on("requestfinished", (req) => {
|
||||
const resp = req.response();
|
||||
requests.push(`<< ${resp?.status()} ${req.url()}`);
|
||||
});
|
||||
page.on("requestfailed", (req) => {
|
||||
failures.push(`FAILED: ${req.url()} - ${req.failure()?.errorText}`);
|
||||
});
|
||||
|
||||
await page.goto("/dashboard");
|
||||
|
||||
// Log summary
|
||||
console.log("Requests:", requests.length);
|
||||
if (failures.length) console.log("Failures:", failures);
|
||||
});
|
||||
```
|
||||
|
||||
### Wait for Specific API Response
|
||||
|
||||
When debugging network-dependent issues, wait for specific API responses instead of arbitrary timeouts.
|
||||
|
||||
```typescript
|
||||
// Start waiting BEFORE triggering the request
|
||||
const responsePromise = page.waitForResponse(
|
||||
(resp) => resp.url().includes("/api/data") && resp.status() === 200,
|
||||
);
|
||||
await page.getByRole("button", { name: "Load" }).click();
|
||||
const response = await responsePromise;
|
||||
console.log("Status:", response.status());
|
||||
```
|
||||
|
||||
> **For comprehensive waiting patterns** (navigation, element state, network, polling), see [assertions-waiting.md](../core/assertions-waiting.md#waiting-strategies).
|
||||
|
||||
### Debug Slow Requests
|
||||
|
||||
```typescript
|
||||
test("find slow requests", async ({ page }) => {
|
||||
page.on("requestfinished", (request) => {
|
||||
const timing = request.timing();
|
||||
const total = timing.responseEnd - timing.requestStart;
|
||||
if (total > 1000) {
|
||||
console.log(`SLOW (${total}ms): ${request.url()}`);
|
||||
}
|
||||
});
|
||||
|
||||
await page.goto("/");
|
||||
});
|
||||
```
|
||||
|
||||
## Debugging in CI
|
||||
|
||||
### Simulate CI Locally
|
||||
|
||||
```bash
|
||||
# Run in headless mode like CI
|
||||
CI=true npx playwright test
|
||||
|
||||
# Match CI browser versions
|
||||
npx playwright install --with-deps
|
||||
|
||||
# Run in Docker (same as CI)
|
||||
docker run --rm -v $(pwd):/work -w /work \
|
||||
mcr.microsoft.com/playwright:v1.40.0-jammy \
|
||||
npx playwright test
|
||||
```
|
||||
|
||||
### CI-Specific Configuration
|
||||
|
||||
```typescript
|
||||
// playwright.config.ts
|
||||
export default defineConfig({
|
||||
// More artifacts in CI for debugging
|
||||
use: {
|
||||
trace: process.env.CI ? "on-first-retry" : "off",
|
||||
video: process.env.CI ? "retain-on-failure" : "off",
|
||||
screenshot: process.env.CI ? "only-on-failure" : "off",
|
||||
},
|
||||
|
||||
// More retries in CI (but investigate failures!)
|
||||
retries: process.env.CI ? 2 : 0,
|
||||
});
|
||||
```
|
||||
|
||||
### Debug CI Environment
|
||||
|
||||
```typescript
|
||||
test("CI environment check", async ({ page }, testInfo) => {
|
||||
console.log("CI:", process.env.CI);
|
||||
console.log("Project:", testInfo.project.name);
|
||||
console.log("Worker:", testInfo.workerIndex);
|
||||
console.log("Retry:", testInfo.retry);
|
||||
console.log("Base URL:", testInfo.project.use.baseURL);
|
||||
|
||||
// Check viewport
|
||||
const viewport = page.viewportSize();
|
||||
console.log("Viewport:", viewport);
|
||||
});
|
||||
```
|
||||
|
||||
## Debugging Authentication
|
||||
|
||||
```typescript
|
||||
test("debug auth", async ({ page, context }) => {
|
||||
// Inspect current storage state
|
||||
const storage = await context.storageState();
|
||||
console.log(
|
||||
"Cookies:",
|
||||
storage.cookies.map((c) => c.name),
|
||||
);
|
||||
|
||||
// Check if auth cookies are present
|
||||
const cookies = await context.cookies();
|
||||
const authCookie = cookies.find((c) => c.name.includes("session"));
|
||||
console.log("Auth cookie:", authCookie ? "present" : "MISSING");
|
||||
|
||||
await page.goto("/protected");
|
||||
|
||||
// Check if redirected to login (auth failed)
|
||||
if (page.url().includes("/login")) {
|
||||
console.error("Auth failed - redirected to login");
|
||||
// Save state for inspection
|
||||
await context.storageState({ path: "debug-auth.json" });
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
## Debugging Screenshots
|
||||
|
||||
### Compare Visual State
|
||||
|
||||
```typescript
|
||||
test("visual debug", async ({ page }, testInfo) => {
|
||||
await page.goto("/");
|
||||
|
||||
// Screenshot before action
|
||||
await page.screenshot({
|
||||
path: testInfo.outputPath("before.png"),
|
||||
fullPage: true,
|
||||
});
|
||||
|
||||
await page.getByRole("button", { name: "Open Menu" }).click();
|
||||
|
||||
// Screenshot after action
|
||||
await page.screenshot({
|
||||
path: testInfo.outputPath("after.png"),
|
||||
fullPage: true,
|
||||
});
|
||||
|
||||
// Attach to report
|
||||
await testInfo.attach("before", {
|
||||
path: testInfo.outputPath("before.png"),
|
||||
contentType: "image/png",
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
### Screenshot Specific Element
|
||||
|
||||
```typescript
|
||||
test("element screenshot", async ({ page }) => {
|
||||
await page.goto("/");
|
||||
|
||||
const element = page.getByTestId("problem-area");
|
||||
|
||||
// Screenshot just the element
|
||||
await element.screenshot({ path: "element-debug.png" });
|
||||
|
||||
// Highlight element in full page screenshot
|
||||
await element.evaluate((el) => (el.style.border = "3px solid red"));
|
||||
await page.screenshot({ path: "highlighted.png" });
|
||||
});
|
||||
```
|
||||
|
||||
## Common Issues
|
||||
|
||||
### Element Not Found
|
||||
|
||||
```typescript
|
||||
// Debug: Check if element exists
|
||||
console.log(await page.getByRole("button").count());
|
||||
|
||||
// Debug: Log all buttons
|
||||
const buttons = await page.getByRole("button").all();
|
||||
for (const button of buttons) {
|
||||
console.log(await button.textContent());
|
||||
}
|
||||
|
||||
// Debug: Screenshot before action
|
||||
await page.screenshot({ path: "debug.png" });
|
||||
await page.getByRole("button").click();
|
||||
```
|
||||
|
||||
### Timeout Issues
|
||||
|
||||
```typescript
|
||||
// Increase timeout for slow operations
|
||||
await expect(page.getByText("Loaded")).toBeVisible({ timeout: 30000 });
|
||||
|
||||
// Global timeout increase
|
||||
test.setTimeout(60000);
|
||||
|
||||
// Check what's blocking
|
||||
test("debug timeout", async ({ page }) => {
|
||||
await page.goto("/slow-page");
|
||||
|
||||
// Log network activity
|
||||
page.on("request", (request) => console.log(">>", request.url()));
|
||||
page.on("response", (response) =>
|
||||
console.log("<<", response.url(), response.status()),
|
||||
);
|
||||
});
|
||||
```
|
||||
|
||||
### Selector Issues
|
||||
|
||||
```typescript
|
||||
// Debug: Highlight element
|
||||
await page.getByRole("button").highlight();
|
||||
|
||||
// Debug: Evaluate selector in browser console
|
||||
// Run in Inspector console:
|
||||
// playwright.locator('button').first().highlight()
|
||||
|
||||
// Debug: Get element info
|
||||
const element = page.getByRole("button");
|
||||
console.log("Count:", await element.count());
|
||||
console.log("Visible:", await element.isVisible());
|
||||
console.log("Enabled:", await element.isEnabled());
|
||||
```
|
||||
|
||||
### Frame Issues
|
||||
|
||||
```typescript
|
||||
// Debug: List all frames
|
||||
for (const frame of page.frames()) {
|
||||
console.log("Frame:", frame.url());
|
||||
}
|
||||
|
||||
// Debug: Check if element is in iframe
|
||||
const frame = page.frameLocator("iframe").first();
|
||||
console.log(await frame.getByRole("button").count());
|
||||
```
|
||||
|
||||
## Logging
|
||||
|
||||
### Capture Browser Console
|
||||
|
||||
```typescript
|
||||
test("with logging", async ({ page }) => {
|
||||
page.on("console", (msg) => console.log("Browser:", msg.text()));
|
||||
page.on("pageerror", (error) => console.log("Page error:", error.message));
|
||||
await page.goto("/");
|
||||
});
|
||||
```
|
||||
|
||||
> **For comprehensive console error handling** (fail on errors, allowed patterns, fixtures), see [console-errors.md](console-errors.md).
|
||||
|
||||
### Custom Test Attachments
|
||||
|
||||
```typescript
|
||||
test("with attachments", async ({ page }, testInfo) => {
|
||||
// Attach screenshot to report
|
||||
const screenshot = await page.screenshot();
|
||||
await testInfo.attach("screenshot", {
|
||||
body: screenshot,
|
||||
contentType: "image/png",
|
||||
});
|
||||
|
||||
// Attach logs or data
|
||||
await testInfo.attach("logs", {
|
||||
body: "Custom log data",
|
||||
contentType: "text/plain",
|
||||
});
|
||||
|
||||
// Use testInfo for output paths
|
||||
const outputPath = testInfo.outputPath("debug-file.json");
|
||||
});
|
||||
```
|
||||
|
||||
## Troubleshooting Checklist
|
||||
|
||||
### By Symptom
|
||||
|
||||
| Symptom | Common Causes | Quick Fixes | Reference |
|
||||
| --------------------------------------------- | ------------------------------------------------------------ | ------------------------------------------------------------------- | -------------------------------------------------------------------------- |
|
||||
| **Element not found** | Wrong selector, element not visible, in iframe, timing issue | Check locator with Inspector, wait for visibility, use frameLocator | [locators.md](../core/locators.md), [assertions-waiting.md](../core/assertions-waiting.md) |
|
||||
| **Timeout errors** | Slow network, heavy page load, waiting for wrong condition | Increase timeout, wait for specific response, check network tab | [assertions-waiting.md](../core/assertions-waiting.md) |
|
||||
| **Flaky tests** | Race conditions, shared state, timing dependencies | See comprehensive flaky test guide | [flaky-tests.md](flaky-tests.md) |
|
||||
| **Tests pass locally, fail in CI** | Environment differences, missing dependencies, timing | Simulate CI locally, check CI logs, verify environment vars | [ci-cd.md](../infrastructure-ci-cd/ci-cd.md), [flaky-tests.md](flaky-tests.md) |
|
||||
| **Slow test execution** | Not parallelized, heavy network calls, unnecessary waits | Enable parallelization, mock APIs, optimize waits | [performance.md](../infrastructure-ci-cd/performance.md) |
|
||||
| **Selector works in browser but not in test** | Element not attached, wrong context, dynamic content | Use auto-waiting, check iframe, verify element state | [locators.md](../core/locators.md) |
|
||||
| **Test fails on retry** | Non-deterministic data, external dependencies | Use test data fixtures, mock external services | [fixtures-hooks.md](../core/fixtures-hooks.md) |
|
||||
|
||||
### Step-by-Step Debugging Process
|
||||
|
||||
1. **Reproduce the issue**
|
||||
|
||||
```bash
|
||||
# Run with trace enabled
|
||||
npx playwright test tests/failing.spec.ts --trace on
|
||||
|
||||
# If intermittent, run multiple times
|
||||
npx playwright test --repeat-each=10
|
||||
```
|
||||
|
||||
2. **Inspect the failure**
|
||||
|
||||
```bash
|
||||
# View trace
|
||||
npx playwright show-trace test-results/path-to-trace.zip
|
||||
|
||||
# Run in headed mode to watch
|
||||
npx playwright test --headed
|
||||
|
||||
# Use inspector for step-by-step
|
||||
PWDEBUG=1 npx playwright test
|
||||
```
|
||||
|
||||
3. **Isolate the problem**
|
||||
|
||||
```typescript
|
||||
// Add debugging points
|
||||
await page.pause();
|
||||
|
||||
// Log element state
|
||||
console.log("Element count:", await page.getByRole("button").count());
|
||||
console.log("Element visible:", await page.getByRole("button").isVisible());
|
||||
|
||||
// Take screenshot at failure point
|
||||
await page.screenshot({ path: "debug.png" });
|
||||
```
|
||||
|
||||
4. **Check related areas**
|
||||
- Network requests: Are API calls completing? (see [Debugging Network Issues](#debugging-network-issues))
|
||||
- Timing: Is auto-waiting working correctly?
|
||||
- State: Is the test isolated? (see [flaky-tests.md](flaky-tests.md))
|
||||
- Environment: Does it work locally but fail in CI? (see [Debugging in CI](#debugging-in-ci))
|
||||
|
||||
5. **Apply fix and verify**
|
||||
- Fix the root cause (not just symptoms)
|
||||
- Run multiple times to confirm stability: `--repeat-each=10`
|
||||
- Check related tests aren't affected
|
||||
|
||||
## Related References
|
||||
|
||||
- **Flaky tests**: See [flaky-tests.md](flaky-tests.md) for comprehensive flaky test guide
|
||||
- **Locator issues**: See [locators.md](../core/locators.md) for selector strategies
|
||||
- **Waiting problems**: See [assertions-waiting.md](../core/assertions-waiting.md) for waiting patterns
|
||||
- **Test isolation**: See [fixtures-hooks.md](../core/fixtures-hooks.md) for fixtures and isolation
|
||||
- **CI issues**: See [ci-cd.md](../infrastructure-ci-cd/ci-cd.md) for CI configuration
|
||||
360
.cursor/skills/playwright-testing/debugging/error-testing.md
Normal file
360
.cursor/skills/playwright-testing/debugging/error-testing.md
Normal file
|
|
@ -0,0 +1,360 @@
|
|||
# Error & Edge Case Testing
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [Error Boundaries](#error-boundaries)
|
||||
2. [Network Failures](#network-failures)
|
||||
3. [Offline Testing](#offline-testing)
|
||||
4. [Loading States](#loading-states)
|
||||
5. [Form Validation](#form-validation)
|
||||
|
||||
## Error Boundaries
|
||||
|
||||
### Test Component Errors
|
||||
|
||||
```typescript
|
||||
test("error boundary catches component error", async ({ page }) => {
|
||||
// Trigger error via mock
|
||||
await page.route("**/api/user", (route) => {
|
||||
route.fulfill({
|
||||
json: null, // Will cause component to throw
|
||||
});
|
||||
});
|
||||
|
||||
await page.goto("/profile");
|
||||
|
||||
// Error boundary should render fallback
|
||||
await expect(page.getByText("Something went wrong")).toBeVisible();
|
||||
await expect(page.getByRole("button", { name: "Try Again" })).toBeVisible();
|
||||
});
|
||||
```
|
||||
|
||||
### Test Error Recovery
|
||||
|
||||
```typescript
|
||||
test("recover from error state", async ({ page }) => {
|
||||
let requestCount = 0;
|
||||
|
||||
await page.route("**/api/data", (route) => {
|
||||
requestCount++;
|
||||
if (requestCount === 1) {
|
||||
return route.fulfill({ status: 500 });
|
||||
}
|
||||
return route.fulfill({
|
||||
json: { data: "success" },
|
||||
});
|
||||
});
|
||||
|
||||
await page.goto("/dashboard");
|
||||
|
||||
// Error state
|
||||
await expect(page.getByText("Failed to load")).toBeVisible();
|
||||
|
||||
// Retry
|
||||
await page.getByRole("button", { name: "Retry" }).click();
|
||||
|
||||
// Success state
|
||||
await expect(page.getByText("success")).toBeVisible();
|
||||
});
|
||||
```
|
||||
|
||||
### Test JavaScript Errors
|
||||
|
||||
```typescript
|
||||
test("handles runtime error gracefully", async ({ page }) => {
|
||||
const errors: string[] = [];
|
||||
|
||||
page.on("pageerror", (error) => {
|
||||
errors.push(error.message);
|
||||
});
|
||||
|
||||
await page.goto("/buggy-page");
|
||||
|
||||
// App should still be functional despite error
|
||||
await expect(page.getByRole("navigation")).toBeVisible();
|
||||
|
||||
// Error was logged
|
||||
expect(errors.length).toBeGreaterThan(0);
|
||||
});
|
||||
```
|
||||
|
||||
## Network Failures
|
||||
|
||||
### Test API Errors
|
||||
|
||||
```typescript
|
||||
test.describe("API error handling", () => {
|
||||
const errorCodes = [400, 401, 403, 404, 500, 502, 503];
|
||||
|
||||
for (const status of errorCodes) {
|
||||
test(`handles ${status} error`, async ({ page }) => {
|
||||
await page.route("**/api/data", (route) =>
|
||||
route.fulfill({
|
||||
status,
|
||||
json: { error: `Error ${status}` },
|
||||
}),
|
||||
);
|
||||
|
||||
await page.goto("/dashboard");
|
||||
|
||||
// Appropriate error message shown
|
||||
await expect(page.getByRole("alert")).toBeVisible();
|
||||
});
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
### Test Timeout
|
||||
|
||||
```typescript
|
||||
test("handles request timeout", async ({ page }) => {
|
||||
await page.route("**/api/slow", async (route) => {
|
||||
// Never respond - simulates timeout
|
||||
await new Promise(() => {});
|
||||
});
|
||||
|
||||
await page.goto("/slow-page");
|
||||
|
||||
// Should show timeout message (app should have its own timeout)
|
||||
await expect(page.getByText("Request timed out")).toBeVisible({
|
||||
timeout: 15000,
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
### Test Connection Reset
|
||||
|
||||
```typescript
|
||||
test("handles connection failure", async ({ page }) => {
|
||||
await page.route("**/api/data", (route) => {
|
||||
route.abort("connectionfailed");
|
||||
});
|
||||
|
||||
await page.goto("/dashboard");
|
||||
|
||||
await expect(page.getByText("Connection failed")).toBeVisible();
|
||||
await expect(page.getByRole("button", { name: "Retry" })).toBeVisible();
|
||||
});
|
||||
```
|
||||
|
||||
### Test Mid-Request Failure
|
||||
|
||||
```typescript
|
||||
test("handles failure during request", async ({ page }) => {
|
||||
let requestStarted = false;
|
||||
|
||||
await page.route("**/api/upload", async (route) => {
|
||||
requestStarted = true;
|
||||
// Abort after small delay (mid-request)
|
||||
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||
route.abort("failed");
|
||||
});
|
||||
|
||||
await page.goto("/upload");
|
||||
await page.getByLabel("File").setInputFiles("./fixtures/large-file.pdf");
|
||||
await page.getByRole("button", { name: "Upload" }).click();
|
||||
|
||||
// Should show failure, not hang
|
||||
await expect(page.getByText("Upload failed")).toBeVisible();
|
||||
expect(requestStarted).toBe(true);
|
||||
});
|
||||
```
|
||||
|
||||
## Offline Testing
|
||||
|
||||
This section covers **unexpected network failures** and error recovery. For **offline-first apps (PWAs)** with service workers, caching, and background sync, see [service-workers.md](service-workers.md#offline-testing).
|
||||
|
||||
### Go Offline During Session
|
||||
|
||||
```typescript
|
||||
test("handles going offline", async ({ page, context }) => {
|
||||
await page.goto("/dashboard");
|
||||
await expect(page.getByTestId("data")).toBeVisible();
|
||||
|
||||
// Go offline unexpectedly
|
||||
await context.setOffline(true);
|
||||
|
||||
// Try to refresh data
|
||||
await page.getByRole("button", { name: "Refresh" }).click();
|
||||
|
||||
// Should show offline indicator
|
||||
await expect(page.getByText("You're offline")).toBeVisible();
|
||||
|
||||
// Go back online
|
||||
await context.setOffline(false);
|
||||
|
||||
// Should recover
|
||||
await page.getByRole("button", { name: "Refresh" }).click();
|
||||
await expect(page.getByText("You're offline")).toBeHidden();
|
||||
});
|
||||
```
|
||||
|
||||
### Test Network Recovery
|
||||
|
||||
```typescript
|
||||
test("recovers gracefully when connection returns", async ({
|
||||
page,
|
||||
context,
|
||||
}) => {
|
||||
await page.goto("/dashboard");
|
||||
|
||||
// Simulate connection drop
|
||||
await context.setOffline(true);
|
||||
|
||||
// App should show degraded state
|
||||
await expect(page.getByRole("alert")).toContainText(/offline|connection/i);
|
||||
|
||||
// Connection restored
|
||||
await context.setOffline(false);
|
||||
|
||||
// Retry should work
|
||||
await page.getByRole("button", { name: "Retry" }).click();
|
||||
await expect(page.getByTestId("data")).toBeVisible();
|
||||
});
|
||||
```
|
||||
|
||||
## Loading States
|
||||
|
||||
### Test Skeleton Loaders
|
||||
|
||||
```typescript
|
||||
test("shows skeleton during load", async ({ page }) => {
|
||||
// Add delay to API response
|
||||
await page.route("**/api/posts", async (route) => {
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
route.fulfill({
|
||||
json: [{ id: 1, title: "Post 1" }],
|
||||
});
|
||||
});
|
||||
|
||||
await page.goto("/posts");
|
||||
|
||||
// Skeleton should appear immediately
|
||||
await expect(page.getByTestId("skeleton")).toBeVisible();
|
||||
|
||||
// Then content replaces skeleton
|
||||
await expect(page.getByText("Post 1")).toBeVisible();
|
||||
await expect(page.getByTestId("skeleton")).toBeHidden();
|
||||
});
|
||||
```
|
||||
|
||||
### Test Loading Indicators
|
||||
|
||||
```typescript
|
||||
test("shows loading state for actions", async ({ page }) => {
|
||||
await page.route("**/api/save", async (route) => {
|
||||
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||
route.fulfill({ json: { success: true } });
|
||||
});
|
||||
|
||||
await page.goto("/editor");
|
||||
await page.getByLabel("Content").fill("New content");
|
||||
|
||||
const saveButton = page.getByRole("button", { name: "Save" });
|
||||
await saveButton.click();
|
||||
|
||||
// Button should show loading state
|
||||
await expect(saveButton).toBeDisabled();
|
||||
await expect(page.getByTestId("spinner")).toBeVisible();
|
||||
|
||||
// Then success state
|
||||
await expect(saveButton).toBeEnabled();
|
||||
await expect(page.getByText("Saved")).toBeVisible();
|
||||
});
|
||||
```
|
||||
|
||||
### Test Empty States
|
||||
|
||||
```typescript
|
||||
test("shows empty state when no data", async ({ page }) => {
|
||||
await page.route("**/api/items", (route) => route.fulfill({ json: [] }));
|
||||
|
||||
await page.goto("/items");
|
||||
|
||||
await expect(page.getByText("No items yet")).toBeVisible();
|
||||
await expect(
|
||||
page.getByRole("button", { name: "Create First Item" }),
|
||||
).toBeVisible();
|
||||
});
|
||||
```
|
||||
|
||||
## Form Validation
|
||||
|
||||
### Test Client-Side Validation
|
||||
|
||||
```typescript
|
||||
test("validates required fields", async ({ page }) => {
|
||||
await page.goto("/signup");
|
||||
|
||||
// Submit empty form
|
||||
await page.getByRole("button", { name: "Sign Up" }).click();
|
||||
|
||||
// Should show validation errors
|
||||
await expect(page.getByText("Email is required")).toBeVisible();
|
||||
await expect(page.getByText("Password is required")).toBeVisible();
|
||||
|
||||
// Form should not submit
|
||||
await expect(page).toHaveURL("/signup");
|
||||
});
|
||||
```
|
||||
|
||||
### Test Format Validation
|
||||
|
||||
```typescript
|
||||
test("validates email format", async ({ page }) => {
|
||||
await page.goto("/signup");
|
||||
|
||||
await page.getByLabel("Email").fill("invalid-email");
|
||||
await page.getByLabel("Email").blur();
|
||||
|
||||
await expect(page.getByText("Invalid email address")).toBeVisible();
|
||||
|
||||
// Fix the error
|
||||
await page.getByLabel("Email").fill("valid@email.com");
|
||||
await page.getByLabel("Email").blur();
|
||||
|
||||
await expect(page.getByText("Invalid email address")).toBeHidden();
|
||||
});
|
||||
```
|
||||
|
||||
### Test Server-Side Validation
|
||||
|
||||
```typescript
|
||||
test("handles server validation errors", async ({ page }) => {
|
||||
await page.route("**/api/register", (route) =>
|
||||
route.fulfill({
|
||||
status: 422,
|
||||
json: {
|
||||
errors: {
|
||||
email: "Email already exists",
|
||||
username: "Username is taken",
|
||||
},
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
await page.goto("/signup");
|
||||
await page.getByLabel("Email").fill("taken@email.com");
|
||||
await page.getByLabel("Username").fill("takenuser");
|
||||
await page.getByLabel("Password").fill("password123");
|
||||
await page.getByRole("button", { name: "Sign Up" }).click();
|
||||
|
||||
// Server errors should display
|
||||
await expect(page.getByText("Email already exists")).toBeVisible();
|
||||
await expect(page.getByText("Username is taken")).toBeVisible();
|
||||
});
|
||||
```
|
||||
|
||||
## Anti-Patterns to Avoid
|
||||
|
||||
| Anti-Pattern | Problem | Solution |
|
||||
| ------------------------ | ------------------------------ | -------------------------------------- |
|
||||
| Only testing happy path | Misses error handling bugs | Test all error scenarios |
|
||||
| No network failure tests | App crashes on poor connection | Test offline/slow/failed requests |
|
||||
| Skipping loading states | Janky UX not caught | Assert loading UI appears |
|
||||
| Ignoring validation | Form bugs slip through | Test both client and server validation |
|
||||
|
||||
## Related References
|
||||
|
||||
- **Network Mocking**: See [network-advanced.md](../advanced/network-advanced.md) for mock patterns
|
||||
- **Assertions**: See [assertions-waiting.md](../core/assertions-waiting.md) for error assertions
|
||||
496
.cursor/skills/playwright-testing/debugging/flaky-tests.md
Normal file
496
.cursor/skills/playwright-testing/debugging/flaky-tests.md
Normal file
|
|
@ -0,0 +1,496 @@
|
|||
# Debugging and Managing Flaky Tests
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [Understanding Flakiness Types](#understanding-flakiness-types)
|
||||
2. [Detection and Reproduction](#detection-and-reproduction)
|
||||
3. [Root Cause Analysis](#root-cause-analysis)
|
||||
4. [Fixing Strategies by Type](#fixing-strategies-by-type)
|
||||
5. [CI-Specific Flakiness](#ci-specific-flakiness)
|
||||
6. [Quarantine and Management](#quarantine-and-management)
|
||||
7. [Prevention Strategies](#prevention-strategies)
|
||||
|
||||
## Understanding Flakiness Types
|
||||
|
||||
### Categories of Flakiness
|
||||
|
||||
Most flaky tests fall into distinct categories requiring different remediation:
|
||||
|
||||
| Category | Symptoms | Common Causes |
|
||||
| --------------------------- | ------------------------------- | ------------------------------------------------------ |
|
||||
| **UI-driven** | Element not found, click missed | Missing waits, animations, dynamic rendering |
|
||||
| **Environment-driven** | CI-only failures | Slower CPU, memory limits, cold browser starts |
|
||||
| **Data/parallelism-driven** | Fails with multiple workers | Shared backend data, reused accounts, state collisions |
|
||||
| **Test-suite-driven** | Fails when run with other tests | Leaked state, shared fixtures, order dependencies |
|
||||
|
||||
### Flakiness Decision Tree
|
||||
|
||||
```
|
||||
Test fails intermittently
|
||||
├─ Fails locally too?
|
||||
│ ├─ YES → Timing/async issue → Check waits and assertions
|
||||
│ └─ NO → CI-specific → Check environment differences
|
||||
│
|
||||
├─ Fails only with multiple workers?
|
||||
│ └─ YES → Parallelism issue → Check data isolation
|
||||
│
|
||||
├─ Fails only when run after specific tests?
|
||||
│ └─ YES → State leak → Check fixtures and cleanup
|
||||
│
|
||||
└─ Fails randomly regardless of conditions?
|
||||
└─ External dependency → Check network/API stability
|
||||
```
|
||||
|
||||
## Detection and Reproduction
|
||||
|
||||
### Confirming Flakiness
|
||||
|
||||
```bash
|
||||
# Run test multiple times to confirm instability
|
||||
npx playwright test tests/checkout.spec.ts --repeat-each=20
|
||||
|
||||
# Run with single worker to isolate parallelism issues
|
||||
npx playwright test --workers=1
|
||||
|
||||
# Run in CI-like conditions locally
|
||||
CI=true npx playwright test --repeat-each=10
|
||||
```
|
||||
|
||||
### Reproduction Strategies
|
||||
|
||||
```typescript
|
||||
// playwright.config.ts - Enable artifacts for flaky test investigation
|
||||
export default defineConfig({
|
||||
retries: process.env.CI ? 2 : 0,
|
||||
use: {
|
||||
trace: "on-first-retry", // Capture trace on retry
|
||||
video: "retain-on-failure",
|
||||
screenshot: "only-on-failure",
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
### Identify Flaky Tests Programmatically
|
||||
|
||||
```typescript
|
||||
// Track test results across runs
|
||||
test.afterEach(async ({}, testInfo) => {
|
||||
if (testInfo.retry > 0 && testInfo.status === "passed") {
|
||||
console.warn(`FLAKY: ${testInfo.title} passed on retry ${testInfo.retry}`);
|
||||
// Log to your tracking system
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
## Root Cause Analysis
|
||||
|
||||
### Event Logging for Race Conditions
|
||||
|
||||
Add comprehensive event logging to expose timing issues:
|
||||
|
||||
```typescript
|
||||
test.beforeEach(async ({ page }) => {
|
||||
page.on("console", (msg) =>
|
||||
console.log(`CONSOLE [${msg.type()}]:`, msg.text()),
|
||||
);
|
||||
page.on("pageerror", (err) => console.error("PAGE ERROR:", err.message));
|
||||
page.on("requestfailed", (req) =>
|
||||
console.error(`REQUEST FAILED: ${req.url()}`),
|
||||
);
|
||||
});
|
||||
```
|
||||
|
||||
> **For comprehensive console error handling** (fail on errors, allowed patterns, fixtures), see [console-errors.md](console-errors.md).
|
||||
|
||||
### Network Timing Analysis
|
||||
|
||||
```typescript
|
||||
// Capture slow or failed requests
|
||||
test.beforeEach(async ({ page }) => {
|
||||
const slowRequests: string[] = [];
|
||||
|
||||
page.on("requestfinished", (request) => {
|
||||
const timing = request.timing();
|
||||
const duration = timing.responseEnd - timing.requestStart;
|
||||
if (duration > 2000) {
|
||||
slowRequests.push(`${request.url()} took ${duration}ms`);
|
||||
}
|
||||
});
|
||||
|
||||
page.on("requestfailed", (request) => {
|
||||
console.error(`Failed: ${request.url()} - ${request.failure()?.errorText}`);
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
### Trace Analysis
|
||||
|
||||
```bash
|
||||
# View trace from failed CI run
|
||||
npx playwright show-trace path/to/trace.zip
|
||||
|
||||
# Generate trace for specific test
|
||||
npx playwright test tests/flaky.spec.ts --trace on
|
||||
```
|
||||
|
||||
## Fixing Strategies by Type
|
||||
|
||||
### UI-Driven Flakiness
|
||||
|
||||
**Problem: Element not ready when action executes**
|
||||
|
||||
```typescript
|
||||
// ❌ BAD: No wait for element state
|
||||
await page.click("#submit");
|
||||
await page.fill("#username", "test"); // Element may not be ready
|
||||
|
||||
// ✅ GOOD: Actions + assertions pattern (auto-waiting built-in)
|
||||
await page.getByRole("button", { name: "Submit" }).click();
|
||||
await expect(page.getByRole("heading", { name: "Dashboard" })).toBeVisible();
|
||||
```
|
||||
|
||||
**Problem: Animations or transitions interfere**
|
||||
|
||||
```typescript
|
||||
// ❌ BAD: Click during animation
|
||||
await page.click(".menu-item");
|
||||
|
||||
// ✅ GOOD: Wait for animation to complete
|
||||
await page.getByRole("menuitem", { name: "Settings" }).click();
|
||||
await expect(page.getByRole("dialog")).toBeVisible();
|
||||
// Or disable animations entirely
|
||||
await page.emulateMedia({ reducedMotion: "reduce" });
|
||||
```
|
||||
|
||||
**Problem: Brittle selectors**
|
||||
|
||||
```typescript
|
||||
// ❌ BAD: Fragile CSS chain
|
||||
await page.click("div.container > div:nth-child(2) > button.btn-primary");
|
||||
|
||||
// ✅ GOOD: Semantic selectors
|
||||
await page.getByRole("button", { name: "Continue" }).click();
|
||||
await page.getByTestId("checkout-button").click();
|
||||
await page.getByLabel("Email address").fill("test@example.com");
|
||||
```
|
||||
|
||||
### Async/Timing Flakiness
|
||||
|
||||
**Problem: Race between test and application**
|
||||
|
||||
```typescript
|
||||
// ❌ BAD: Arbitrary sleep
|
||||
await page.click("#load-data");
|
||||
await page.waitForTimeout(3000); // Hope data loads in 3s
|
||||
|
||||
// ✅ GOOD: Wait for specific condition
|
||||
await page.click("#load-data");
|
||||
await expect(page.locator(".data-row")).toHaveCount(10, { timeout: 10000 });
|
||||
|
||||
// ✅ BETTER: Wait for network response, then assert
|
||||
const responsePromise = page.waitForResponse(
|
||||
(r) =>
|
||||
r.url().includes("/api/data") &&
|
||||
r.request().method() === "GET" &&
|
||||
r.ok(),
|
||||
);
|
||||
await page.click("#load-data");
|
||||
await responsePromise;
|
||||
await expect(page.locator(".data-row")).toHaveCount(10);
|
||||
```
|
||||
|
||||
> **For comprehensive waiting strategies** (navigation, element state, network, polling with `toPass()`), see [assertions-waiting.md](assertions-waiting.md#waiting-strategies).
|
||||
|
||||
**Problem: Complex async state**
|
||||
|
||||
```typescript
|
||||
// Custom wait for application-specific conditions
|
||||
await page.waitForFunction(() => {
|
||||
const app = (window as any).__APP_STATE__;
|
||||
return app?.isReady && !app?.isLoading;
|
||||
});
|
||||
|
||||
// Wait for multiple conditions
|
||||
await Promise.all([
|
||||
page.waitForResponse("**/api/user"),
|
||||
page.waitForResponse("**/api/settings"),
|
||||
page.getByRole("button", { name: "Load" }).click(),
|
||||
]);
|
||||
```
|
||||
|
||||
### Data/Parallelism-Driven Flakiness
|
||||
|
||||
**Problem: Tests share backend data**
|
||||
|
||||
```typescript
|
||||
// ❌ BAD: All workers use same user
|
||||
const testUser = { email: "test@example.com", password: "pass123" };
|
||||
|
||||
// ✅ GOOD: Unique data per worker
|
||||
import { test as base } from "@playwright/test";
|
||||
|
||||
export const test = base.extend<
|
||||
{},
|
||||
{ testUser: { email: string; id: string } }
|
||||
>({
|
||||
testUser: [
|
||||
async ({}, use, workerInfo) => {
|
||||
const email = `test-${workerInfo.workerIndex}-${Date.now()}@example.com`;
|
||||
const user = await createTestUser(email);
|
||||
await use(user);
|
||||
await deleteTestUser(user.id);
|
||||
},
|
||||
{ scope: "worker" },
|
||||
],
|
||||
});
|
||||
```
|
||||
|
||||
**Problem: Shared storageState across workers**
|
||||
|
||||
```typescript
|
||||
// ❌ BAD: All workers share same auth state
|
||||
use: {
|
||||
storageState: '.auth/user.json',
|
||||
}
|
||||
|
||||
// ✅ GOOD: Per-worker auth state
|
||||
export const test = base.extend<{}, { workerStorageState: string }>({
|
||||
workerStorageState: [
|
||||
async ({ browser }, use, workerInfo) => {
|
||||
const id = workerInfo.workerIndex;
|
||||
const fileName = `.auth/user-${id}.json`;
|
||||
|
||||
if (!fs.existsSync(fileName)) {
|
||||
const page = await browser.newPage({ storageState: undefined });
|
||||
await authenticateUser(page, `worker${id}@test.com`);
|
||||
await page.context().storageState({ path: fileName });
|
||||
await page.close();
|
||||
}
|
||||
|
||||
await use(fileName);
|
||||
},
|
||||
{ scope: "worker" },
|
||||
],
|
||||
});
|
||||
```
|
||||
|
||||
### Test-Suite-Driven Flakiness (State Leaks)
|
||||
|
||||
**Problem: Tests affect each other**
|
||||
|
||||
```typescript
|
||||
// ❌ BAD: Module-level state persists across tests
|
||||
let sharedPage: Page;
|
||||
|
||||
test.beforeAll(async ({ browser }) => {
|
||||
sharedPage = await browser.newPage(); // Shared across tests!
|
||||
});
|
||||
|
||||
// ✅ GOOD: Use Playwright's default isolation (fresh context per test)
|
||||
test("first test", async ({ page }) => {
|
||||
// Fresh page for this test
|
||||
});
|
||||
|
||||
test("second test", async ({ page }) => {
|
||||
// Fresh page for this test
|
||||
});
|
||||
```
|
||||
|
||||
**Problem: Fixture cleanup not happening**
|
||||
|
||||
```typescript
|
||||
// ✅ GOOD: Proper fixture with cleanup
|
||||
export const test = base.extend<{ tempFile: string }>({
|
||||
tempFile: async ({}, use) => {
|
||||
const file = `/tmp/test-${Date.now()}.json`;
|
||||
fs.writeFileSync(file, "{}");
|
||||
|
||||
await use(file);
|
||||
|
||||
// Cleanup always runs, even on failure
|
||||
if (fs.existsSync(file)) {
|
||||
fs.unlinkSync(file);
|
||||
}
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
## CI-Specific Flakiness
|
||||
|
||||
### Why Tests Fail Only in CI
|
||||
|
||||
| CI Condition | Impact | Solution |
|
||||
| ------------------ | ------------------------------------- | ---------------------------------------------------- |
|
||||
| Slower CPU | Actions complete later than expected | Use auto-waiting, not timeouts |
|
||||
| Cold browser start | No cached assets, slower initial load | Add explicit waits for first navigation |
|
||||
| Headless mode | Different rendering behavior | Test locally in headless mode |
|
||||
| Shared runners | Resource contention | Reduce parallelism or use dedicated runners |
|
||||
| Network latency | API calls slower | Mock external APIs, increase timeouts for real calls |
|
||||
|
||||
### Simulating CI Locally
|
||||
|
||||
```bash
|
||||
# Run headless with CI environment variable
|
||||
CI=true npx playwright test
|
||||
|
||||
# Limit CPU (Linux/Mac)
|
||||
cpulimit -l 50 -- npx playwright test
|
||||
|
||||
# Run in Docker matching CI environment
|
||||
docker run -it --rm \
|
||||
-v $(pwd):/work \
|
||||
-w /work \
|
||||
mcr.microsoft.com/playwright:v1.40.0-jammy \
|
||||
npx playwright test
|
||||
```
|
||||
|
||||
### Consistent Viewport and Scale
|
||||
|
||||
```typescript
|
||||
// playwright.config.ts - Match CI rendering exactly
|
||||
export default defineConfig({
|
||||
use: {
|
||||
viewport: { width: 1280, height: 720 },
|
||||
deviceScaleFactor: 1,
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
### Network Stubbing for External APIs
|
||||
|
||||
```typescript
|
||||
// Eliminate external API flakiness
|
||||
test.beforeEach(async ({ page }) => {
|
||||
// Stub unstable third-party APIs
|
||||
await page.route("**/api.analytics.com/**", (route) =>
|
||||
route.fulfill({ body: "" }),
|
||||
);
|
||||
await page.route("**/api.payment-provider.com/**", (route) =>
|
||||
route.fulfill({ json: { status: "ok" } }),
|
||||
);
|
||||
});
|
||||
|
||||
// Test-specific stub
|
||||
test("checkout with payment", async ({ page }) => {
|
||||
await page.route("**/api/payment", (route) =>
|
||||
route.fulfill({ json: { success: true, transactionId: "test-123" } }),
|
||||
);
|
||||
// Test proceeds with deterministic response
|
||||
});
|
||||
```
|
||||
|
||||
## Quarantine and Management
|
||||
|
||||
### Quarantine Pattern
|
||||
|
||||
```typescript
|
||||
// playwright.config.ts - Separate flaky tests
|
||||
export default defineConfig({
|
||||
projects: [
|
||||
{
|
||||
name: "stable",
|
||||
testIgnore: ["**/*.flaky.spec.ts"],
|
||||
},
|
||||
{
|
||||
name: "quarantine",
|
||||
testMatch: ["**/*.flaky.spec.ts"],
|
||||
retries: 3,
|
||||
},
|
||||
],
|
||||
});
|
||||
```
|
||||
|
||||
### Annotation-Based Quarantine
|
||||
|
||||
```typescript
|
||||
// Mark flaky tests with annotations
|
||||
test("intermittent checkout issue", async ({ page }, testInfo) => {
|
||||
testInfo.annotations.push({
|
||||
type: "flaky",
|
||||
description: "Investigating payment API timing - JIRA-1234",
|
||||
});
|
||||
|
||||
// Test implementation
|
||||
});
|
||||
|
||||
// Skip flaky test conditionally
|
||||
test("known CI flaky", async ({ page }) => {
|
||||
test.skip(!!process.env.CI, "Flaky in CI - investigating JIRA-5678");
|
||||
// Test implementation
|
||||
});
|
||||
```
|
||||
|
||||
## Prevention Strategies
|
||||
|
||||
### Test Burn-In
|
||||
|
||||
```bash
|
||||
# Run new tests many times before merging
|
||||
npx playwright test tests/new-feature.spec.ts --repeat-each=50
|
||||
|
||||
# Run in parallel to expose race conditions
|
||||
npx playwright test tests/new-feature.spec.ts --repeat-each=20 --workers=4
|
||||
```
|
||||
|
||||
### Isolation Checklist
|
||||
|
||||
```typescript
|
||||
// ✅ Each test should be self-contained
|
||||
test.describe("User profile", () => {
|
||||
test("can update name", async ({ page, testUser }) => {
|
||||
// Uses unique testUser fixture
|
||||
// No dependency on other tests
|
||||
// Cleanup handled by fixture
|
||||
});
|
||||
|
||||
test("can update email", async ({ page, testUser }) => {
|
||||
// Independent of "can update name"
|
||||
// Own testUser, own state
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
### Defensive Assertions
|
||||
|
||||
```typescript
|
||||
// ❌ BAD: Single point of failure
|
||||
await expect(page.locator(".items")).toHaveCount(5);
|
||||
|
||||
// ✅ GOOD: Progressive assertions that help diagnose
|
||||
await expect(page.locator(".items-container")).toBeVisible();
|
||||
await expect(page.locator(".loading")).not.toBeVisible();
|
||||
await expect(page.locator(".items")).toHaveCount(5);
|
||||
```
|
||||
|
||||
### Retry Budget
|
||||
|
||||
```typescript
|
||||
// playwright.config.ts - Limit retries to avoid masking issues
|
||||
export default defineConfig({
|
||||
retries: process.env.CI ? 2 : 0, // Only retry in CI
|
||||
expect: {
|
||||
timeout: 10000, // Reasonable assertion timeout
|
||||
},
|
||||
timeout: 60000, // Test timeout
|
||||
});
|
||||
```
|
||||
|
||||
## Anti-Patterns to Avoid
|
||||
|
||||
| Anti-Pattern | Problem | Solution |
|
||||
| ----------------------------------------- | ----------------------------------- | ---------------------------------------------- |
|
||||
| `waitForTimeout()` as primary wait | Arbitrary, hides real timing issues | Use auto-waiting assertions |
|
||||
| Increasing global timeout to "fix" flakes | Masks root cause, slows all tests | Find and fix actual timing issue |
|
||||
| Retrying until pass | Hides systemic problems | Fix root cause, use retries for diagnosis only |
|
||||
| Shared test data across workers | Race conditions, collisions | Isolate data per worker |
|
||||
| Testing real external APIs | Network variability | Mock external dependencies |
|
||||
| Module-level mutable state | Leaks between tests | Use fixtures with proper cleanup |
|
||||
| Ignoring flaky tests | Problem compounds over time | Quarantine and track for fixing |
|
||||
|
||||
## Related References
|
||||
|
||||
- **Debugging**: See [debugging.md](debugging.md) for trace viewer and inspector
|
||||
- **Fixtures**: See [fixtures-hooks.md](../core/fixtures-hooks.md) for worker-scoped isolation
|
||||
- **Performance**: See [performance.md](../infrastructure-ci-cd/performance.md) for parallel execution patterns
|
||||
- **Assertions**: See [assertions-waiting.md](../core/assertions-waiting.md) for auto-waiting patterns
|
||||
- **Global Setup**: See [global-setup.md](../core/global-setup.md) for setup vs fixtures decision
|
||||
Loading…
Add table
Add a link
Reference in a new issue