From 8fcac5dcf40600798acbd42e7bb10ebd8fe814f4 Mon Sep 17 00:00:00 2001 From: abose Date: Sat, 23 Aug 2025 16:12:35 +0530 Subject: [PATCH] ci: allow playwright tests to test all and then fail at end --- .../workflows/playwright-chromium-linux.yml | 32 +++ .../workflows/playwright-chromium-macos.yml | 32 +++ .../playwright-chromium-windows-prod.yml | 35 +++- .../workflows/playwright-chromium-windows.yml | 33 +++ .../workflows/playwright-firefox-linux.yml | 32 +++ .../workflows/playwright-firefox-windows.yml | 33 +++ .github/workflows/playwright-staging-prod.yml | 194 ++++++++++++++++++ 7 files changed, 390 insertions(+), 1 deletion(-) diff --git a/.github/workflows/playwright-chromium-linux.yml b/.github/workflows/playwright-chromium-linux.yml index d3cc937c75..3d2bb5419f 100644 --- a/.github/workflows/playwright-chromium-linux.yml +++ b/.github/workflows/playwright-chromium-linux.yml @@ -26,6 +26,8 @@ jobs: - name: Run Playwright unit tests in Chromium uses: nick-fields/retry@v2 + id: chromiumUnit + continue-on-error: true with: timeout_minutes: 12 max_attempts: 3 @@ -33,6 +35,8 @@ jobs: - name: Run Playwright integration tests in Chromium uses: nick-fields/retry@v2 + id: chromiumIntegration + continue-on-error: true with: timeout_minutes: 12 max_attempts: 3 @@ -40,6 +44,8 @@ jobs: - name: Run Playwright mainview tests in Chromium uses: nick-fields/retry@v2 + id: chromiumMainview + continue-on-error: true with: timeout_minutes: 12 max_attempts: 3 @@ -47,6 +53,8 @@ jobs: - name: Run Playwright livepreview tests in Chromium uses: nick-fields/retry@v2 + id: chromiumLivepreview + continue-on-error: true with: timeout_minutes: 15 max_attempts: 3 @@ -54,7 +62,31 @@ jobs: - name: Run Playwright LegacyInteg tests in Chromium uses: nick-fields/retry@v2 + id: chromiumLegacyInteg + continue-on-error: true with: timeout_minutes: 20 max_attempts: 3 command: npx cross-env TEST_ENV=LegacyInteg npm run testChromium + + - name: Fail on test runs failed in Chromium Linux + if: steps.chromiumUnit.outcome == 'failure' || steps.chromiumIntegration.outcome == 'failure' || steps.chromiumMainview.outcome == 'failure' || steps.chromiumLivepreview.outcome == 'failure' || steps.chromiumLegacyInteg.outcome == 'failure' + run: | + echo "Chromium Linux Playwright tests failed, marking step as failed" + echo "Failed tests:" + if [ "${{ steps.chromiumUnit.outcome }}" == "failure" ]; then + echo "- Run Playwright unit tests in Chromium" + fi + if [ "${{ steps.chromiumIntegration.outcome }}" == "failure" ]; then + echo "- Run Playwright integration tests in Chromium" + fi + if [ "${{ steps.chromiumMainview.outcome }}" == "failure" ]; then + echo "- Run Playwright mainview tests in Chromium" + fi + if [ "${{ steps.chromiumLivepreview.outcome }}" == "failure" ]; then + echo "- Run Playwright livepreview tests in Chromium" + fi + if [ "${{ steps.chromiumLegacyInteg.outcome }}" == "failure" ]; then + echo "- Run Playwright LegacyInteg tests in Chromium" + fi + exit 1 diff --git a/.github/workflows/playwright-chromium-macos.yml b/.github/workflows/playwright-chromium-macos.yml index de89c5f09d..26b3854fc2 100644 --- a/.github/workflows/playwright-chromium-macos.yml +++ b/.github/workflows/playwright-chromium-macos.yml @@ -26,6 +26,8 @@ jobs: - name: Run Playwright unit tests in Chromium uses: nick-fields/retry@v2 + id: chromiumUnit + continue-on-error: true with: timeout_minutes: 12 max_attempts: 3 @@ -33,6 +35,8 @@ jobs: - name: Run Playwright integration tests in Chromium uses: nick-fields/retry@v2 + id: chromiumIntegration + continue-on-error: true with: timeout_minutes: 12 max_attempts: 3 @@ -40,6 +44,8 @@ jobs: - name: Run Playwright mainview tests in Chromium uses: nick-fields/retry@v2 + id: chromiumMainview + continue-on-error: true with: timeout_minutes: 12 max_attempts: 3 @@ -47,6 +53,8 @@ jobs: - name: Run Playwright livepreview tests in Chromium uses: nick-fields/retry@v2 + id: chromiumLivepreview + continue-on-error: true with: timeout_minutes: 15 max_attempts: 3 @@ -54,7 +62,31 @@ jobs: - name: Run Playwright LegacyInteg tests in Chromium uses: nick-fields/retry@v2 + id: chromiumLegacyInteg + continue-on-error: true with: timeout_minutes: 20 max_attempts: 3 command: npx cross-env TEST_ENV=LegacyInteg npm run testChromium + + - name: Fail on test runs failed in Chromium macOS + if: steps.chromiumUnit.outcome == 'failure' || steps.chromiumIntegration.outcome == 'failure' || steps.chromiumMainview.outcome == 'failure' || steps.chromiumLivepreview.outcome == 'failure' || steps.chromiumLegacyInteg.outcome == 'failure' + run: | + echo "Chromium macOS Playwright tests failed, marking step as failed" + echo "Failed tests:" + if [ "${{ steps.chromiumUnit.outcome }}" == "failure" ]; then + echo "- Run Playwright unit tests in Chromium" + fi + if [ "${{ steps.chromiumIntegration.outcome }}" == "failure" ]; then + echo "- Run Playwright integration tests in Chromium" + fi + if [ "${{ steps.chromiumMainview.outcome }}" == "failure" ]; then + echo "- Run Playwright mainview tests in Chromium" + fi + if [ "${{ steps.chromiumLivepreview.outcome }}" == "failure" ]; then + echo "- Run Playwright livepreview tests in Chromium" + fi + if [ "${{ steps.chromiumLegacyInteg.outcome }}" == "failure" ]; then + echo "- Run Playwright LegacyInteg tests in Chromium" + fi + exit 1 diff --git a/.github/workflows/playwright-chromium-windows-prod.yml b/.github/workflows/playwright-chromium-windows-prod.yml index 99087f5bc8..0909eb6d60 100644 --- a/.github/workflows/playwright-chromium-windows-prod.yml +++ b/.github/workflows/playwright-chromium-windows-prod.yml @@ -29,6 +29,8 @@ jobs: - name: Run Playwright unit tests in Chromium for prod stage uses: nick-fields/retry@v2 + id: chromiumUnit + continue-on-error: true with: timeout_minutes: 12 max_attempts: 3 @@ -36,6 +38,8 @@ jobs: - name: Run Playwright integration tests in Chromium for prod stage uses: nick-fields/retry@v2 + id: chromiumIntegration + continue-on-error: true with: timeout_minutes: 12 max_attempts: 3 @@ -43,6 +47,8 @@ jobs: - name: Run Playwright mainview tests in Chromium for prod stage uses: nick-fields/retry@v2 + id: chromiumMainview + continue-on-error: true with: timeout_minutes: 12 max_attempts: 3 @@ -50,6 +56,8 @@ jobs: - name: Run Playwright livepreview tests in Chromium for prod stage uses: nick-fields/retry@v2 + id: chromiumLivepreview + continue-on-error: true with: timeout_minutes: 15 max_attempts: 3 @@ -57,7 +65,32 @@ jobs: - name: Run Playwright LegacyInteg tests in Chromium for prod stage uses: nick-fields/retry@v2 + id: chromiumLegacyInteg + continue-on-error: true with: timeout_minutes: 30 max_attempts: 3 - command: npx cross-env TEST_ENV=LegacyInteg npm run testChromiumDist \ No newline at end of file + command: npx cross-env TEST_ENV=LegacyInteg npm run testChromiumDist + + - name: Fail on test runs failed in Chromium Windows (prod sanity) + if: steps.chromiumUnit.outcome == 'failure' || steps.chromiumIntegration.outcome == 'failure' || steps.chromiumMainview.outcome == 'failure' || steps.chromiumLivepreview.outcome == 'failure' || steps.chromiumLegacyInteg.outcome == 'failure' + shell: bash + run: | + echo "Chromium Windows (prod) Playwright tests failed, marking step as failed" + echo "Failed tests:" + if [ "${{ steps.chromiumUnit.outcome }}" == "failure" ]; then + echo "- Run Playwright unit tests in Chromium for prod stage" + fi + if [ "${{ steps.chromiumIntegration.outcome }}" == "failure" ]; then + echo "- Run Playwright integration tests in Chromium for prod stage" + fi + if [ "${{ steps.chromiumMainview.outcome }}" == "failure" ]; then + echo "- Run Playwright mainview tests in Chromium for prod stage" + fi + if [ "${{ steps.chromiumLivepreview.outcome }}" == "failure" ]; then + echo "- Run Playwright livepreview tests in Chromium for prod stage" + fi + if [ "${{ steps.chromiumLegacyInteg.outcome }}" == "failure" ]; then + echo "- Run Playwright LegacyInteg tests in Chromium for prod stage" + fi + exit 1 diff --git a/.github/workflows/playwright-chromium-windows.yml b/.github/workflows/playwright-chromium-windows.yml index f4538ca13d..c302710da6 100644 --- a/.github/workflows/playwright-chromium-windows.yml +++ b/.github/workflows/playwright-chromium-windows.yml @@ -26,6 +26,8 @@ jobs: - name: Run Playwright unit tests in Chromium uses: nick-fields/retry@v2 + id: chromiumUnit + continue-on-error: true with: timeout_minutes: 12 max_attempts: 3 @@ -33,6 +35,8 @@ jobs: - name: Run Playwright integration tests in Chromium uses: nick-fields/retry@v2 + id: chromiumIntegration + continue-on-error: true with: timeout_minutes: 12 max_attempts: 3 @@ -40,6 +44,8 @@ jobs: - name: Run Playwright mainview tests in Chromium uses: nick-fields/retry@v2 + id: chromiumMainview + continue-on-error: true with: timeout_minutes: 12 max_attempts: 3 @@ -47,6 +53,8 @@ jobs: - name: Run Playwright livepreview tests in Chromium uses: nick-fields/retry@v2 + id: chromiumLivepreview + continue-on-error: true with: timeout_minutes: 15 max_attempts: 3 @@ -54,7 +62,32 @@ jobs: - name: Run Playwright LegacyInteg tests in Chromium uses: nick-fields/retry@v2 + id: chromiumLegacyInteg + continue-on-error: true with: timeout_minutes: 30 max_attempts: 3 command: npx cross-env TEST_ENV=LegacyInteg npm run testChromium + + - name: Fail on test runs failed in Chromium Windows + if: steps.chromiumUnit.outcome == 'failure' || steps.chromiumIntegration.outcome == 'failure' || steps.chromiumMainview.outcome == 'failure' || steps.chromiumLivepreview.outcome == 'failure' || steps.chromiumLegacyInteg.outcome == 'failure' + shell: bash + run: | + echo "Chromium Windows Playwright tests failed, marking step as failed" + echo "Failed tests:" + if [ "${{ steps.chromiumUnit.outcome }}" == "failure" ]; then + echo "- Run Playwright unit tests in Chromium" + fi + if [ "${{ steps.chromiumIntegration.outcome }}" == "failure" ]; then + echo "- Run Playwright integration tests in Chromium" + fi + if [ "${{ steps.chromiumMainview.outcome }}" == "failure" ]; then + echo "- Run Playwright mainview tests in Chromium" + fi + if [ "${{ steps.chromiumLivepreview.outcome }}" == "failure" ]; then + echo "- Run Playwright livepreview tests in Chromium" + fi + if [ "${{ steps.chromiumLegacyInteg.outcome }}" == "failure" ]; then + echo "- Run Playwright LegacyInteg tests in Chromium" + fi + exit 1 diff --git a/.github/workflows/playwright-firefox-linux.yml b/.github/workflows/playwright-firefox-linux.yml index 602da564bc..f61e07ca6c 100644 --- a/.github/workflows/playwright-firefox-linux.yml +++ b/.github/workflows/playwright-firefox-linux.yml @@ -26,6 +26,8 @@ jobs: - name: Run Playwright unit tests in Firefox uses: nick-fields/retry@v2 + id: firefoxUnit + continue-on-error: true with: timeout_minutes: 12 max_attempts: 3 @@ -33,6 +35,8 @@ jobs: - name: Run Playwright integration tests in Firefox uses: nick-fields/retry@v2 + id: firefoxIntegration + continue-on-error: true with: timeout_minutes: 12 max_attempts: 3 @@ -40,6 +44,8 @@ jobs: - name: Run Playwright mainview tests in Firefox uses: nick-fields/retry@v2 + id: firefoxMainview + continue-on-error: true with: timeout_minutes: 12 max_attempts: 3 @@ -47,6 +53,8 @@ jobs: - name: Run Playwright livepreview tests in Firefox uses: nick-fields/retry@v2 + id: firefoxLivepreview + continue-on-error: true with: timeout_minutes: 15 max_attempts: 3 @@ -54,7 +62,31 @@ jobs: - name: Run Playwright LegacyInteg tests in Firefox uses: nick-fields/retry@v2 + id: firefoxLegacyInteg + continue-on-error: true with: timeout_minutes: 20 max_attempts: 3 command: npx cross-env TEST_ENV=LegacyInteg npm run testFirefox + + - name: Fail on test runs failed in Firefox Linux + if: steps.firefoxUnit.outcome == 'failure' || steps.firefoxIntegration.outcome == 'failure' || steps.firefoxMainview.outcome == 'failure' || steps.firefoxLivepreview.outcome == 'failure' || steps.firefoxLegacyInteg.outcome == 'failure' + run: | + echo "Firefox Linux Playwright tests failed, marking step as failed" + echo "Failed tests:" + if [ "${{ steps.firefoxUnit.outcome }}" == "failure" ]; then + echo "- Run Playwright unit tests in Firefox" + fi + if [ "${{ steps.firefoxIntegration.outcome }}" == "failure" ]; then + echo "- Run Playwright integration tests in Firefox" + fi + if [ "${{ steps.firefoxMainview.outcome }}" == "failure" ]; then + echo "- Run Playwright mainview tests in Firefox" + fi + if [ "${{ steps.firefoxLivepreview.outcome }}" == "failure" ]; then + echo "- Run Playwright livepreview tests in Firefox" + fi + if [ "${{ steps.firefoxLegacyInteg.outcome }}" == "failure" ]; then + echo "- Run Playwright LegacyInteg tests in Firefox" + fi + exit 1 diff --git a/.github/workflows/playwright-firefox-windows.yml b/.github/workflows/playwright-firefox-windows.yml index db07475ab3..69b75852a3 100644 --- a/.github/workflows/playwright-firefox-windows.yml +++ b/.github/workflows/playwright-firefox-windows.yml @@ -26,6 +26,8 @@ jobs: - name: Run Playwright unit tests in Firefox uses: nick-fields/retry@v2 + id: firefoxUnit + continue-on-error: true with: timeout_minutes: 12 max_attempts: 3 @@ -33,6 +35,8 @@ jobs: - name: Run Playwright integration tests in Firefox uses: nick-fields/retry@v2 + id: firefoxIntegration + continue-on-error: true with: timeout_minutes: 12 max_attempts: 3 @@ -40,6 +44,8 @@ jobs: - name: Run Playwright mainview tests in Firefox uses: nick-fields/retry@v2 + id: firefoxMainview + continue-on-error: true with: timeout_minutes: 12 max_attempts: 3 @@ -47,6 +53,8 @@ jobs: - name: Run Playwright livepreview tests in Firefox uses: nick-fields/retry@v2 + id: firefoxLivepreview + continue-on-error: true with: timeout_minutes: 15 max_attempts: 3 @@ -54,7 +62,32 @@ jobs: - name: Run Playwright LegacyInteg tests in Firefox uses: nick-fields/retry@v2 + id: firefoxLegacyInteg + continue-on-error: true with: timeout_minutes: 20 max_attempts: 3 command: npx cross-env TEST_ENV=LegacyInteg npm run testFirefox + + - name: Fail on test runs failed in Firefox Windows + if: steps.firefoxUnit.outcome == 'failure' || steps.firefoxIntegration.outcome == 'failure' || steps.firefoxMainview.outcome == 'failure' || steps.firefoxLivepreview.outcome == 'failure' || steps.firefoxLegacyInteg.outcome == 'failure' + shell: bash + run: | + echo "Firefox Windows Playwright tests failed, marking step as failed" + echo "Failed tests:" + if [ "${{ steps.firefoxUnit.outcome }}" == "failure" ]; then + echo "- Run Playwright unit tests in Firefox" + fi + if [ "${{ steps.firefoxIntegration.outcome }}" == "failure" ]; then + echo "- Run Playwright integration tests in Firefox" + fi + if [ "${{ steps.firefoxMainview.outcome }}" == "failure" ]; then + echo "- Run Playwright mainview tests in Firefox" + fi + if [ "${{ steps.firefoxLivepreview.outcome }}" == "failure" ]; then + echo "- Run Playwright livepreview tests in Firefox" + fi + if [ "${{ steps.firefoxLegacyInteg.outcome }}" == "failure" ]; then + echo "- Run Playwright LegacyInteg tests in Firefox" + fi + exit 1 diff --git a/.github/workflows/playwright-staging-prod.yml b/.github/workflows/playwright-staging-prod.yml index 1dd4436849..e8e10a76e5 100644 --- a/.github/workflows/playwright-staging-prod.yml +++ b/.github/workflows/playwright-staging-prod.yml @@ -25,6 +25,8 @@ jobs: - name: Run Playwright unit tests in Chromium for prod stage uses: nick-fields/retry@v2 + id: chromiumUnit + continue-on-error: true with: timeout_minutes: 12 max_attempts: 3 @@ -32,6 +34,8 @@ jobs: - name: Run Playwright integration tests in Chromium for prod stage uses: nick-fields/retry@v2 + id: chromiumIntegration + continue-on-error: true with: timeout_minutes: 12 max_attempts: 3 @@ -39,6 +43,8 @@ jobs: - name: Run Playwright mainview tests in Chromium for prod stage uses: nick-fields/retry@v2 + id: chromiumMainview + continue-on-error: true with: timeout_minutes: 12 max_attempts: 3 @@ -46,6 +52,8 @@ jobs: - name: Run Playwright livepreview tests in Chromium for prod stage uses: nick-fields/retry@v2 + id: chromiumLivepreview + continue-on-error: true with: timeout_minutes: 15 max_attempts: 3 @@ -53,11 +61,35 @@ jobs: - name: Run Playwright LegacyInteg tests in Chromium for prod stage uses: nick-fields/retry@v2 + id: chromiumLegacyInteg + continue-on-error: true with: timeout_minutes: 20 max_attempts: 3 command: npx cross-env TEST_ENV=LegacyInteg npm run testChromiumDist + - name: Fail on test runs failed in Chromium Linux (staging/prod) + if: steps.chromiumUnit.outcome == 'failure' || steps.chromiumIntegration.outcome == 'failure' || steps.chromiumMainview.outcome == 'failure' || steps.chromiumLivepreview.outcome == 'failure' || steps.chromiumLegacyInteg.outcome == 'failure' + run: | + echo "Chromium Linux (staging/prod) Playwright tests failed, marking step as failed" + echo "Failed tests:" + if [ "${{ steps.chromiumUnit.outcome }}" == "failure" ]; then + echo "- Run Playwright unit tests in Chromium for prod stage" + fi + if [ "${{ steps.chromiumIntegration.outcome }}" == "failure" ]; then + echo "- Run Playwright integration tests in Chromium for prod stage" + fi + if [ "${{ steps.chromiumMainview.outcome }}" == "failure" ]; then + echo "- Run Playwright mainview tests in Chromium for prod stage" + fi + if [ "${{ steps.chromiumLivepreview.outcome }}" == "failure" ]; then + echo "- Run Playwright livepreview tests in Chromium for prod stage" + fi + if [ "${{ steps.chromiumLegacyInteg.outcome }}" == "failure" ]; then + echo "- Run Playwright LegacyInteg tests in Chromium for prod stage" + fi + exit 1 + playwrightChromiumWindows: timeout-minutes: 90 runs-on: windows-latest @@ -78,6 +110,8 @@ jobs: - name: Run Playwright unit tests in Chromium for prod stage uses: nick-fields/retry@v2 + id: chromiumUnit + continue-on-error: true with: timeout_minutes: 12 max_attempts: 3 @@ -85,6 +119,8 @@ jobs: - name: Run Playwright integration tests in Chromium for prod stage uses: nick-fields/retry@v2 + id: chromiumIntegration + continue-on-error: true with: timeout_minutes: 12 max_attempts: 3 @@ -92,6 +128,8 @@ jobs: - name: Run Playwright mainview tests in Chromium for prod stage uses: nick-fields/retry@v2 + id: chromiumMainview + continue-on-error: true with: timeout_minutes: 12 max_attempts: 3 @@ -99,6 +137,8 @@ jobs: - name: Run Playwright livepreview tests in Chromium for prod stage uses: nick-fields/retry@v2 + id: chromiumLivepreview + continue-on-error: true with: timeout_minutes: 15 max_attempts: 3 @@ -106,11 +146,36 @@ jobs: - name: Run Playwright LegacyInteg tests in Chromium for prod stage uses: nick-fields/retry@v2 + id: chromiumLegacyInteg + continue-on-error: true with: timeout_minutes: 30 max_attempts: 3 command: npx cross-env TEST_ENV=LegacyInteg npm run testChromiumDist + - name: Fail on test runs failed in Chromium Windows (staging/prod) + if: steps.chromiumUnit.outcome == 'failure' || steps.chromiumIntegration.outcome == 'failure' || steps.chromiumMainview.outcome == 'failure' || steps.chromiumLivepreview.outcome == 'failure' || steps.chromiumLegacyInteg.outcome == 'failure' + shell: bash + run: | + echo "Chromium Windows (staging/prod) Playwright tests failed, marking step as failed" + echo "Failed tests:" + if [ "${{ steps.chromiumUnit.outcome }}" == "failure" ]; then + echo "- Run Playwright unit tests in Chromium for prod stage" + fi + if [ "${{ steps.chromiumIntegration.outcome }}" == "failure" ]; then + echo "- Run Playwright integration tests in Chromium for prod stage" + fi + if [ "${{ steps.chromiumMainview.outcome }}" == "failure" ]; then + echo "- Run Playwright mainview tests in Chromium for prod stage" + fi + if [ "${{ steps.chromiumLivepreview.outcome }}" == "failure" ]; then + echo "- Run Playwright livepreview tests in Chromium for prod stage" + fi + if [ "${{ steps.chromiumLegacyInteg.outcome }}" == "failure" ]; then + echo "- Run Playwright LegacyInteg tests in Chromium for prod stage" + fi + exit 1 + playwrightChromiumMacos: timeout-minutes: 90 runs-on: macos-14 @@ -131,6 +196,8 @@ jobs: - name: Run Playwright unit tests in Chromium uses: nick-fields/retry@v2 + id: chromiumUnit + continue-on-error: true with: timeout_minutes: 12 max_attempts: 3 @@ -138,6 +205,8 @@ jobs: - name: Run Playwright integration tests in Chromium uses: nick-fields/retry@v2 + id: chromiumIntegration + continue-on-error: true with: timeout_minutes: 12 max_attempts: 3 @@ -145,6 +214,8 @@ jobs: - name: Run Playwright mainview tests in Chromium uses: nick-fields/retry@v2 + id: chromiumMainview + continue-on-error: true with: timeout_minutes: 12 max_attempts: 3 @@ -152,6 +223,8 @@ jobs: - name: Run Playwright livepreview tests in Chromium uses: nick-fields/retry@v2 + id: chromiumLivepreview + continue-on-error: true with: timeout_minutes: 15 max_attempts: 3 @@ -159,11 +232,35 @@ jobs: - name: Run Playwright LegacyInteg tests in Chromium uses: nick-fields/retry@v2 + id: chromiumLegacyInteg + continue-on-error: true with: timeout_minutes: 20 max_attempts: 3 command: npx cross-env TEST_ENV=LegacyInteg npm run testChromiumDist + - name: Fail on test runs failed in Chromium macOS (staging/prod) + if: steps.chromiumUnit.outcome == 'failure' || steps.chromiumIntegration.outcome == 'failure' || steps.chromiumMainview.outcome == 'failure' || steps.chromiumLivepreview.outcome == 'failure' || steps.chromiumLegacyInteg.outcome == 'failure' + run: | + echo "Chromium macOS (staging/prod) Playwright tests failed, marking step as failed" + echo "Failed tests:" + if [ "${{ steps.chromiumUnit.outcome }}" == "failure" ]; then + echo "- Run Playwright unit tests in Chromium" + fi + if [ "${{ steps.chromiumIntegration.outcome }}" == "failure" ]; then + echo "- Run Playwright integration tests in Chromium" + fi + if [ "${{ steps.chromiumMainview.outcome }}" == "failure" ]; then + echo "- Run Playwright mainview tests in Chromium" + fi + if [ "${{ steps.chromiumLivepreview.outcome }}" == "failure" ]; then + echo "- Run Playwright livepreview tests in Chromium" + fi + if [ "${{ steps.chromiumLegacyInteg.outcome }}" == "failure" ]; then + echo "- Run Playwright LegacyInteg tests in Chromium" + fi + exit 1 + playwrightFirefoxLinux: timeout-minutes: 90 runs-on: ubuntu-22.04 @@ -184,6 +281,8 @@ jobs: - name: Run Playwright unit tests in Firefox uses: nick-fields/retry@v2 + id: firefoxUnit + continue-on-error: true with: timeout_minutes: 12 max_attempts: 3 @@ -191,6 +290,8 @@ jobs: - name: Run Playwright integration tests in Firefox uses: nick-fields/retry@v2 + id: firefoxIntegration + continue-on-error: true with: timeout_minutes: 12 max_attempts: 3 @@ -198,6 +299,8 @@ jobs: - name: Run Playwright mainview tests in Firefox uses: nick-fields/retry@v2 + id: firefoxMainview + continue-on-error: true with: timeout_minutes: 12 max_attempts: 3 @@ -205,6 +308,8 @@ jobs: - name: Run Playwright livepreview tests in Firefox uses: nick-fields/retry@v2 + id: firefoxLivepreview + continue-on-error: true with: timeout_minutes: 15 max_attempts: 3 @@ -212,11 +317,35 @@ jobs: - name: Run Playwright LegacyInteg tests in Firefox uses: nick-fields/retry@v2 + id: firefoxLegacyInteg + continue-on-error: true with: timeout_minutes: 20 max_attempts: 3 command: npx cross-env TEST_ENV=LegacyInteg npm run testFirefoxDist + - name: Fail on test runs failed in Firefox Linux (staging/prod) + if: steps.firefoxUnit.outcome == 'failure' || steps.firefoxIntegration.outcome == 'failure' || steps.firefoxMainview.outcome == 'failure' || steps.firefoxLivepreview.outcome == 'failure' || steps.firefoxLegacyInteg.outcome == 'failure' + run: | + echo "Firefox Linux (staging/prod) Playwright tests failed, marking step as failed" + echo "Failed tests:" + if [ "${{ steps.firefoxUnit.outcome }}" == "failure" ]; then + echo "- Run Playwright unit tests in Firefox" + fi + if [ "${{ steps.firefoxIntegration.outcome }}" == "failure" ]; then + echo "- Run Playwright integration tests in Firefox" + fi + if [ "${{ steps.firefoxMainview.outcome }}" == "failure" ]; then + echo "- Run Playwright mainview tests in Firefox" + fi + if [ "${{ steps.firefoxLivepreview.outcome }}" == "failure" ]; then + echo "- Run Playwright livepreview tests in Firefox" + fi + if [ "${{ steps.firefoxLegacyInteg.outcome }}" == "failure" ]; then + echo "- Run Playwright LegacyInteg tests in Firefox" + fi + exit 1 + playwrightFirefoxWindows: timeout-minutes: 90 runs-on: windows-latest @@ -237,6 +366,8 @@ jobs: - name: Run Playwright unit tests in Firefox uses: nick-fields/retry@v2 + id: firefoxUnit + continue-on-error: true with: timeout_minutes: 12 max_attempts: 3 @@ -244,6 +375,8 @@ jobs: - name: Run Playwright integration tests in Firefox uses: nick-fields/retry@v2 + id: firefoxIntegration + continue-on-error: true with: timeout_minutes: 12 max_attempts: 3 @@ -251,6 +384,8 @@ jobs: - name: Run Playwright mainview tests in Firefox uses: nick-fields/retry@v2 + id: firefoxMainview + continue-on-error: true with: timeout_minutes: 12 max_attempts: 3 @@ -258,6 +393,8 @@ jobs: - name: Run Playwright livepreview tests in Firefox uses: nick-fields/retry@v2 + id: firefoxLivepreview + continue-on-error: true with: timeout_minutes: 15 max_attempts: 3 @@ -265,11 +402,36 @@ jobs: - name: Run Playwright LegacyInteg tests in Firefox uses: nick-fields/retry@v2 + id: firefoxLegacyInteg + continue-on-error: true with: timeout_minutes: 20 max_attempts: 3 command: npx cross-env TEST_ENV=LegacyInteg npm run testFirefoxDist + - name: Fail on test runs failed in Firefox Windows (staging/prod) + if: steps.firefoxUnit.outcome == 'failure' || steps.firefoxIntegration.outcome == 'failure' || steps.firefoxMainview.outcome == 'failure' || steps.firefoxLivepreview.outcome == 'failure' || steps.firefoxLegacyInteg.outcome == 'failure' + shell: bash + run: | + echo "Firefox Windows (staging/prod) Playwright tests failed, marking step as failed" + echo "Failed tests:" + if [ "${{ steps.firefoxUnit.outcome }}" == "failure" ]; then + echo "- Run Playwright unit tests in Firefox" + fi + if [ "${{ steps.firefoxIntegration.outcome }}" == "failure" ]; then + echo "- Run Playwright integration tests in Firefox" + fi + if [ "${{ steps.firefoxMainview.outcome }}" == "failure" ]; then + echo "- Run Playwright mainview tests in Firefox" + fi + if [ "${{ steps.firefoxLivepreview.outcome }}" == "failure" ]; then + echo "- Run Playwright livepreview tests in Firefox" + fi + if [ "${{ steps.firefoxLegacyInteg.outcome }}" == "failure" ]; then + echo "- Run Playwright LegacyInteg tests in Firefox" + fi + exit 1 + playwrightFirefoxMacos: timeout-minutes: 90 runs-on: macos-14 @@ -290,6 +452,8 @@ jobs: - name: Run Playwright unit tests in Firefox uses: nick-fields/retry@v2 + id: firefoxUnit + continue-on-error: true with: timeout_minutes: 12 max_attempts: 3 @@ -297,6 +461,8 @@ jobs: - name: Run Playwright integration tests in Firefox uses: nick-fields/retry@v2 + id: firefoxIntegration + continue-on-error: true with: timeout_minutes: 12 max_attempts: 3 @@ -304,6 +470,8 @@ jobs: - name: Run Playwright mainview tests in Firefox uses: nick-fields/retry@v2 + id: firefoxMainview + continue-on-error: true with: timeout_minutes: 12 max_attempts: 3 @@ -311,6 +479,8 @@ jobs: - name: Run Playwright livepreview tests in Firefox uses: nick-fields/retry@v2 + id: firefoxLivepreview + continue-on-error: true with: timeout_minutes: 15 max_attempts: 3 @@ -318,7 +488,31 @@ jobs: - name: Run Playwright LegacyInteg tests in Firefox uses: nick-fields/retry@v2 + id: firefoxLegacyInteg + continue-on-error: true with: timeout_minutes: 20 max_attempts: 3 command: npx cross-env TEST_ENV=LegacyInteg npm run testFirefoxDist + + - name: Fail on test runs failed in Firefox macOS (staging/prod) + if: steps.firefoxUnit.outcome == 'failure' || steps.firefoxIntegration.outcome == 'failure' || steps.firefoxMainview.outcome == 'failure' || steps.firefoxLivepreview.outcome == 'failure' || steps.firefoxLegacyInteg.outcome == 'failure' + run: | + echo "Firefox macOS (staging/prod) Playwright tests failed, marking step as failed" + echo "Failed tests:" + if [ "${{ steps.firefoxUnit.outcome }}" == "failure" ]; then + echo "- Run Playwright unit tests in Firefox" + fi + if [ "${{ steps.firefoxIntegration.outcome }}" == "failure" ]; then + echo "- Run Playwright integration tests in Firefox" + fi + if [ "${{ steps.firefoxMainview.outcome }}" == "failure" ]; then + echo "- Run Playwright mainview tests in Firefox" + fi + if [ "${{ steps.firefoxLivepreview.outcome }}" == "failure" ]; then + echo "- Run Playwright livepreview tests in Firefox" + fi + if [ "${{ steps.firefoxLegacyInteg.outcome }}" == "failure" ]; then + echo "- Run Playwright LegacyInteg tests in Firefox" + fi + exit 1