feat(junit): add includeRetries option to emit each retry as separate testcase (microsoft#39464)

vazidmansuri005 · web-flow · commit 0154add954e0 · 2026-03-10T13:29:43.000Z
diff --git a/packages/playwright/src/reporters/junit.ts b/packages/playwright/src/reporters/junit.ts
@@ -24,7 +24,7 @@ import { stripAnsiEscapes } from '../util';
 
 import type { ReporterV2 } from './reporterV2';
 import type { JUnitReporterOptions } from '../../types/test';
-import type { FullConfig, FullResult, Suite, TestCase } from '../../types/testReporter';
+import type { FullConfig, FullResult, Suite, TestCase, TestResult } from '../../types/testReporter';
 
 class JUnitReporter implements ReporterV2 {
   private config!: FullConfig;
@@ -38,10 +38,12 @@ class JUnitReporter implements ReporterV2 {
   private resolvedOutputFile: string | undefined;
   private stripANSIControlSequences = false;
   private includeProjectInTestName = false;
+  private includeRetries = false;
 
   constructor(options: JUnitReporterOptions & CommonReporterOptions) {
     this.stripANSIControlSequences = getAsBooleanFromENV('PLAYWRIGHT_JUNIT_STRIP_ANSI', !!options.stripANSIControlSequences);
     this.includeProjectInTestName = getAsBooleanFromENV('PLAYWRIGHT_JUNIT_INCLUDE_PROJECT_IN_TEST_NAME', !!options.includeProjectInTestName);
+    this.includeRetries = getAsBooleanFromENV('PLAYWRIGHT_JUNIT_INCLUDE_RETRIES', !!options.includeRetries);
     this.configDir = options.configDir;
     this.resolvedOutputFile = resolveOutputFile('JUNIT', options)?.outputFile;
   }
@@ -143,14 +145,24 @@ class JUnitReporter implements ReporterV2 {
   }
 
   private async _addTestCase(suiteName: string, namePrefix: string, test: TestCase, entries: XMLEntry[]): Promise<'failure' | 'error' | null> {
+    const isRetried = this.includeRetries && test.results.length > 1;
+    const isFlaky = isRetried && test.ok();
+
     const entry = {
       name: 'testcase',
       attributes: {
         // Skip root, project, file
         name: namePrefix + test.titlePath().slice(3).join(' › '),
         // filename
         classname: suiteName,
-        time: (test.results.reduce((acc, value) => acc + value.duration, 0)) / 1000
+        // For flaky tests, use the last (successful) result's duration.
+        // For permanent failures with retries, use the first result's duration.
+        // Otherwise, use total duration across all results.
+        time: isFlaky
+          ? test.results[test.results.length - 1].duration / 1000
+          : isRetried
+            ? test.results[0].duration / 1000
+            : (test.results.reduce((acc, value) => acc + value.duration, 0)) / 1000
 
       },
       children: [] as XMLEntry[]
@@ -185,34 +197,40 @@ class JUnitReporter implements ReporterV2 {
     }
 
     let classification: 'failure' | 'error' | null = null;
-    if (!test.ok()) {
-      const errorInfo = classifyError(test);
-      if (errorInfo) {
-        classification = errorInfo.elementName;
-        entry.children.push({
-          name: errorInfo.elementName,
-          attributes: {
-            message: errorInfo.message,
-            type: errorInfo.type,
-          },
-          text: stripAnsiEscapes(formatFailure(nonTerminalScreen, this.config, test))
-        });
-      } else {
-        classification = 'failure';
-        entry.children.push({
-          name: 'failure',
-          attributes: {
-            message: `${path.basename(test.location.file)}:${test.location.line}:${test.location.column} ${test.title}`,
-            type: 'FAILURE',
-          },
-          text: stripAnsiEscapes(formatFailure(nonTerminalScreen, this.config, test))
-        });
+
+    if (isFlaky) {
+      // Flaky test (eventually passed): use Maven Surefire <flakyFailure>/<flakyError>.
+      // No <failure> element — flaky tests count as passed.
+      for (const result of test.results) {
+        if (result.status === 'passed' || result.status === 'skipped')
+          continue;
+        entry.children.push(buildSurefireRetryEntry(result, 'flaky'));
       }
+      // classification stays null — flaky tests are not counted as failures.
+    } else if (isRetried) {
+      // Permanent failure (failed all retries): use <failure> + Maven Surefire <rerunFailure>/<rerunError>.
+      classification = this._addFailureEntry(test, entry);
+      // Add <rerunFailure>/<rerunError> for each subsequent retry.
+      for (let i = 1; i < test.results.length; i++) {
+        const result = test.results[i];
+        if (result.status === 'passed' || result.status === 'skipped')
+          continue;
+        entry.children.push(buildSurefireRetryEntry(result, 'rerun'));
+      }
+    } else if (!test.ok()) {
+      // Standard failure (no retries, or includeRetries is false).
+      classification = this._addFailureEntry(test, entry);
     }
 
     const systemOut: string[] = [];
     const systemErr: string[] = [];
-    for (const result of test.results) {
+    // When retries are included, top-level output comes from the primary result only:
+    // flaky → last (successful) result; permanent failure → first result.
+    // Without retries: all results (original behavior).
+    const outputResults = isRetried
+      ? [isFlaky ? test.results[test.results.length - 1] : test.results[0]]
+      : test.results;
+    for (const result of outputResults) {
       for (const item of result.stdout)
         systemOut.push(item.toString());
       for (const item of result.stderr)
@@ -245,40 +263,92 @@ class JUnitReporter implements ReporterV2 {
       entry.children.push({ name: 'system-err', text: systemErr.join('') });
     return classification;
   }
-}
 
-function classifyError(test: TestCase): { elementName: 'failure' | 'error'; type: string; message: string } | null {
-  for (const result of test.results) {
-    const error = result.error;
-    if (!error)
-      continue;
-
-    const rawMessage = stripAnsiEscapes(error.message || error.value || '');
-
-    // Parse "ErrorName: message" format from serialized error.
-    const nameMatch = rawMessage.match(/^(\w+): /);
-    const errorName = nameMatch ? nameMatch[1] : '';
-    const messageBody = nameMatch ? rawMessage.slice(nameMatch[0].length) : rawMessage;
-    const firstLine = messageBody.split('\n')[0].trim();
-
-    // Check for expect/assertion failure pattern.
-    const matcherMatch = rawMessage.match(/expect\(.*?\)\.(not\.)?(\w+)/);
-    if (matcherMatch) {
-      const matcherName = `expect.${matcherMatch[1] || ''}${matcherMatch[2]}`;
-      return {
-        elementName: 'failure',
-        type: matcherName,
-        message: firstLine,
-      };
+  private _addFailureEntry(test: TestCase, entry: XMLEntry): 'failure' | 'error' {
+    const errorInfo = classifyError(test);
+    if (errorInfo) {
+      entry.children!.push({
+        name: errorInfo.elementName,
+        attributes: { message: errorInfo.message, type: errorInfo.type },
+        text: stripAnsiEscapes(formatFailure(nonTerminalScreen, this.config, test))
+      });
+      return errorInfo.elementName;
     }
+    entry.children!.push({
+      name: 'failure',
+      attributes: {
+        message: `${path.basename(test.location.file)}:${test.location.line}:${test.location.column} ${test.title}`,
+        type: 'FAILURE',
+      },
+      text: stripAnsiEscapes(formatFailure(nonTerminalScreen, this.config, test))
+    });
+    return 'failure';
+  }
+
+}
 
-    // Thrown error.
+/**
+ * Builds a Maven Surefire retry entry (<flakyFailure>/<flakyError> or <rerunFailure>/<rerunError>)
+ * with per-result stackTrace, system-out, and system-err as children.
+ */
+function buildSurefireRetryEntry(result: TestResult, prefix: 'flaky' | 'rerun'): XMLEntry {
+  const errorInfo = classifyResultError(result);
+  const baseName = errorInfo?.elementName === 'error' ? 'Error' : 'Failure';
+  const elementName = `${prefix}${baseName}`;
+  const children: XMLEntry[] = [];
+  const stackTrace = result.error?.stack || result.error?.message || result.error?.value || '';
+  children.push({ name: 'stackTrace', text: stripAnsiEscapes(stackTrace) });
+  const resultOut = result.stdout.map(s => s.toString()).join('');
+  const resultErr = result.stderr.map(s => s.toString()).join('');
+  if (resultOut)
+    children.push({ name: 'system-out', text: resultOut });
+  if (resultErr)
+    children.push({ name: 'system-err', text: resultErr });
+  return {
+    name: elementName,
+    attributes: { message: errorInfo?.message || '', type: errorInfo?.type || 'FAILURE', time: result.duration / 1000 },
+    children,
+  };
+}
+
+function classifyResultError(result: TestResult): { elementName: 'failure' | 'error'; type: string; message: string } | null {
+  const error = result.error;
+  if (!error)
+    return null;
+
+  const rawMessage = stripAnsiEscapes(error.message || error.value || '');
+
+  // Parse "ErrorName: message" format from serialized error.
+  const nameMatch = rawMessage.match(/^(\w+): /);
+  const errorName = nameMatch ? nameMatch[1] : '';
+  const messageBody = nameMatch ? rawMessage.slice(nameMatch[0].length) : rawMessage;
+  const firstLine = messageBody.split('\n')[0].trim();
+
+  // Check for expect/assertion failure pattern.
+  const matcherMatch = rawMessage.match(/expect\(.*?\)\.(not\.)?(\w+)/);
+  if (matcherMatch) {
+    const matcherName = `expect.${matcherMatch[1] || ''}${matcherMatch[2]}`;
     return {
-      elementName: 'error',
-      type: errorName || 'Error',
+      elementName: 'failure',
+      type: matcherName,
       message: firstLine,
     };
   }
+
+  // Thrown error.
+  return {
+    elementName: 'error',
+    type: errorName || 'Error',
+    message: firstLine,
+  };
+}
+
+function classifyError(test: TestCase): { elementName: 'failure' | 'error'; type: string; message: string } | null {
+  for (const result of test.results) {
+    const info = classifyResultError(result);
+    if (info)
+      return info;
+  }
   return null;
 }
 
diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts
@@ -20,7 +20,7 @@ export * from 'playwright-core';
 
 export type BlobReporterOptions = { outputDir?: string, fileName?: string };
 export type ListReporterOptions = { printSteps?: boolean };
-export type JUnitReporterOptions = { outputFile?: string, stripANSIControlSequences?: boolean, includeProjectInTestName?: boolean };
+export type JUnitReporterOptions = { outputFile?: string, stripANSIControlSequences?: boolean, includeProjectInTestName?: boolean, includeRetries?: boolean };
 export type JsonReporterOptions = { outputFile?: string };
 export type HtmlReporterOptions = {
   outputFolder?: string;
diff --git a/tests/playwright-test/reporter-junit.spec.ts b/tests/playwright-test/reporter-junit.spec.ts
@@ -633,5 +633,87 @@ for (const useIntermediateMergeReport of [false, true] as const) {
       expect(time).toBe(result.report.stats.duration / 1000);
       expect(time).toBeGreaterThan(1);
     });
+
+    test('should emit flakyFailure for flaky tests when includeRetries is enabled', async ({ runInlineTest }) => {
+      const result = await runInlineTest({
+        'playwright.config.ts': `
+          module.exports = {
+            retries: 2,
+            reporter: [['junit', { includeRetries: true }]]
+          };
+        `,
+        'a.test.js': `
+          import { test, expect } from '@playwright/test';
+          test('one', async ({}, testInfo) => {
+            expect(testInfo.retry).toBe(2);
+          });
+        `,
+      }, { reporter: '' });
+      const xml = parseXML(result.output);
+      // Single testcase for the test; flaky tests count as passed.
+      expect(xml['testsuites']['$']['tests']).toBe('1');
+      expect(xml['testsuites']['$']['failures']).toBe('0');
+      expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('1');
+      expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0');
+      const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];
+      expect(testcase['$']['name']).toBe('one');
+      // No <failure> element — test eventually passed.
+      expect(testcase['failure']).toBeFalsy();
+      // Two <flakyFailure> elements for the two failed attempts.
+      expect(testcase['flakyFailure'].length).toBe(2);
+      expect(testcase['flakyFailure'][0]['stackTrace']).toBeTruthy();
+      expect(testcase['flakyFailure'][1]['stackTrace']).toBeTruthy();
+      expect(result.exitCode).toBe(0);
+    });
+
+    test('should not include retries by default', async ({ runInlineTest }) => {
+      const result = await runInlineTest({
+        'a.test.js': `
+          import { test, expect } from '@playwright/test';
+          test('one', async ({}, testInfo) => {
+            expect(testInfo.retry).toBe(1);
+          });
+        `,
+      }, { retries: 1, reporter: 'junit' });
+      const xml = parseXML(result.output);
+      // Default behavior: single testcase, no flakyFailure elements.
+      expect(xml['testsuites']['$']['tests']).toBe('1');
+      const testcases = xml['testsuites']['testsuite'][0]['testcase'];
+      expect(testcases.length).toBe(1);
+      expect(testcases[0]['$']['name']).toBe('one');
+      expect(testcases[0]['flakyFailure']).toBeFalsy();
+      expect(result.exitCode).toBe(0);
+    });
+
+    test('should emit rerunFailure for permanent failures when includeRetries is enabled', async ({ runInlineTest }) => {
+      const result = await runInlineTest({
+        'playwright.config.ts': `
+          module.exports = {
+            retries: 1,
+            reporter: [['junit', { includeRetries: true }]]
+          };
+        `,
+        'a.test.js': `
+          import { test, expect } from '@playwright/test';
+          test('one', async ({}) => {
+            expect(1).toBe(0);
+          });
+        `,
+      }, { reporter: '' });
+      const xml = parseXML(result.output);
+      // Single testcase; permanent failure counts as 1 test with 1 failure.
+      expect(xml['testsuites']['$']['tests']).toBe('1');
+      expect(xml['testsuites']['$']['failures']).toBe('1');
+      expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('1');
+      expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('1');
+      const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];
+      expect(testcase['$']['name']).toBe('one');
+      // <failure> for the first attempt.
+      expect(testcase['failure']).toBeTruthy();
+      // <rerunFailure> for the retry.
+      expect(testcase['rerunFailure'].length).toBe(1);
+      expect(testcase['rerunFailure'][0]['stackTrace']).toBeTruthy();
+      expect(result.exitCode).toBe(1);
+    });
   });
 }
diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts
@@ -19,7 +19,7 @@ export * from 'playwright-core';
 
 export type BlobReporterOptions = { outputDir?: string, fileName?: string };
 export type ListReporterOptions = { printSteps?: boolean };
-export type JUnitReporterOptions = { outputFile?: string, stripANSIControlSequences?: boolean, includeProjectInTestName?: boolean };
+export type JUnitReporterOptions = { outputFile?: string, stripANSIControlSequences?: boolean, includeProjectInTestName?: boolean, includeRetries?: boolean };
 export type JsonReporterOptions = { outputFile?: string };
 export type HtmlReporterOptions = {
   outputFolder?: string;
