Skip to content

ci: allow playwright tests to test all and then fail at end #2359

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
abose merged 1 commit into from
Aug 24, 2025
Merged

ci: allow playwright tests to test all and then fail at end #2359

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 32 additions & 0 deletions .github/workflows/playwright-chromium-linux.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,35 +26,67 @@ jobs:

- name: Run Playwright unit tests in Chromium
uses: nick-fields/retry@v2
id: chromiumUnit
continue-on-error: true
with:
timeout_minutes: 12
max_attempts: 3
command: npm run testChromium

- name: Run Playwright integration tests in Chromium
uses: nick-fields/retry@v2
id: chromiumIntegration
continue-on-error: true
with:
timeout_minutes: 12
max_attempts: 3
command: npx cross-env TEST_ENV=integration npm run testChromium

- name: Run Playwright mainview tests in Chromium
uses: nick-fields/retry@v2
id: chromiumMainview
continue-on-error: true
with:
timeout_minutes: 12
max_attempts: 3
command: npx cross-env TEST_ENV=mainview npm run testChromium

- name: Run Playwright livepreview tests in Chromium
uses: nick-fields/retry@v2
id: chromiumLivepreview
continue-on-error: true
with:
timeout_minutes: 15
max_attempts: 3
command: npx cross-env TEST_ENV=livepreview npm run testChromium

- name: Run Playwright LegacyInteg tests in Chromium
uses: nick-fields/retry@v2
id: chromiumLegacyInteg
continue-on-error: true
with:
timeout_minutes: 20
max_attempts: 3
command: npx cross-env TEST_ENV=LegacyInteg npm run testChromium

- name: Fail on test runs failed in Chromium Linux
if: steps.chromiumUnit.outcome == 'failure' || steps.chromiumIntegration.outcome == 'failure' || steps.chromiumMainview.outcome == 'failure' || steps.chromiumLivepreview.outcome == 'failure' || steps.chromiumLegacyInteg.outcome == 'failure'
run: |
echo "Chromium Linux Playwright tests failed, marking step as failed"
echo "Failed tests:"
if [ "${{ steps.chromiumUnit.outcome }}" == "failure" ]; then
echo "- Run Playwright unit tests in Chromium"
fi
if [ "${{ steps.chromiumIntegration.outcome }}" == "failure" ]; then
echo "- Run Playwright integration tests in Chromium"
fi
if [ "${{ steps.chromiumMainview.outcome }}" == "failure" ]; then
echo "- Run Playwright mainview tests in Chromium"
fi
if [ "${{ steps.chromiumLivepreview.outcome }}" == "failure" ]; then
echo "- Run Playwright livepreview tests in Chromium"
fi
if [ "${{ steps.chromiumLegacyInteg.outcome }}" == "failure" ]; then
echo "- Run Playwright LegacyInteg tests in Chromium"
fi
exit 1
32 changes: 32 additions & 0 deletions .github/workflows/playwright-chromium-macos.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,35 +26,67 @@ jobs:

- name: Run Playwright unit tests in Chromium
uses: nick-fields/retry@v2
id: chromiumUnit
continue-on-error: true
with:
timeout_minutes: 12
max_attempts: 3
command: npm run testChromium

- name: Run Playwright integration tests in Chromium
uses: nick-fields/retry@v2
id: chromiumIntegration
continue-on-error: true
with:
timeout_minutes: 12
max_attempts: 3
command: npx cross-env TEST_ENV=integration npm run testChromium

- name: Run Playwright mainview tests in Chromium
uses: nick-fields/retry@v2
id: chromiumMainview
continue-on-error: true
with:
timeout_minutes: 12
max_attempts: 3
command: npx cross-env TEST_ENV=mainview npm run testChromium

- name: Run Playwright livepreview tests in Chromium
uses: nick-fields/retry@v2
id: chromiumLivepreview
continue-on-error: true
with:
timeout_minutes: 15
max_attempts: 3
command: npx cross-env TEST_ENV=livepreview npm run testChromium

- name: Run Playwright LegacyInteg tests in Chromium
uses: nick-fields/retry@v2
id: chromiumLegacyInteg
continue-on-error: true
with:
timeout_minutes: 20
max_attempts: 3
command: npx cross-env TEST_ENV=LegacyInteg npm run testChromium

- name: Fail on test runs failed in Chromium macOS
if: steps.chromiumUnit.outcome == 'failure' || steps.chromiumIntegration.outcome == 'failure' || steps.chromiumMainview.outcome == 'failure' || steps.chromiumLivepreview.outcome == 'failure' || steps.chromiumLegacyInteg.outcome == 'failure'
run: |
echo "Chromium macOS Playwright tests failed, marking step as failed"
echo "Failed tests:"
if [ "${{ steps.chromiumUnit.outcome }}" == "failure" ]; then
echo "- Run Playwright unit tests in Chromium"
fi
if [ "${{ steps.chromiumIntegration.outcome }}" == "failure" ]; then
echo "- Run Playwright integration tests in Chromium"
fi
if [ "${{ steps.chromiumMainview.outcome }}" == "failure" ]; then
echo "- Run Playwright mainview tests in Chromium"
fi
if [ "${{ steps.chromiumLivepreview.outcome }}" == "failure" ]; then
echo "- Run Playwright livepreview tests in Chromium"
fi
if [ "${{ steps.chromiumLegacyInteg.outcome }}" == "failure" ]; then
echo "- Run Playwright LegacyInteg tests in Chromium"
fi
exit 1
35 changes: 34 additions & 1 deletion .github/workflows/playwright-chromium-windows-prod.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,35 +29,68 @@ jobs:

- name: Run Playwright unit tests in Chromium for prod stage
uses: nick-fields/retry@v2
id: chromiumUnit
continue-on-error: true
with:
timeout_minutes: 12
max_attempts: 3
command: npm run testChromiumDist

- name: Run Playwright integration tests in Chromium for prod stage
uses: nick-fields/retry@v2
id: chromiumIntegration
continue-on-error: true
with:
timeout_minutes: 12
max_attempts: 3
command: npx cross-env TEST_ENV=integration npm run testChromiumDist

- name: Run Playwright mainview tests in Chromium for prod stage
uses: nick-fields/retry@v2
id: chromiumMainview
continue-on-error: true
with:
timeout_minutes: 12
max_attempts: 3
command: npx cross-env TEST_ENV=mainview npm run testChromiumDist

- name: Run Playwright livepreview tests in Chromium for prod stage
uses: nick-fields/retry@v2
id: chromiumLivepreview
continue-on-error: true
with:
timeout_minutes: 15
max_attempts: 3
command: npx cross-env TEST_ENV=livepreview npm run testChromiumDist

- name: Run Playwright LegacyInteg tests in Chromium for prod stage
uses: nick-fields/retry@v2
id: chromiumLegacyInteg
continue-on-error: true
with:
timeout_minutes: 30
max_attempts: 3
command: npx cross-env TEST_ENV=LegacyInteg npm run testChromiumDist
command: npx cross-env TEST_ENV=LegacyInteg npm run testChromiumDist

- name: Fail on test runs failed in Chromium Windows (prod sanity)
if: steps.chromiumUnit.outcome == 'failure' || steps.chromiumIntegration.outcome == 'failure' || steps.chromiumMainview.outcome == 'failure' || steps.chromiumLivepreview.outcome == 'failure' || steps.chromiumLegacyInteg.outcome == 'failure'
shell: bash
run: |
echo "Chromium Windows (prod) Playwright tests failed, marking step as failed"
echo "Failed tests:"
if [ "${{ steps.chromiumUnit.outcome }}" == "failure" ]; then
echo "- Run Playwright unit tests in Chromium for prod stage"
fi
if [ "${{ steps.chromiumIntegration.outcome }}" == "failure" ]; then
echo "- Run Playwright integration tests in Chromium for prod stage"
fi
if [ "${{ steps.chromiumMainview.outcome }}" == "failure" ]; then
echo "- Run Playwright mainview tests in Chromium for prod stage"
fi
if [ "${{ steps.chromiumLivepreview.outcome }}" == "failure" ]; then
echo "- Run Playwright livepreview tests in Chromium for prod stage"
fi
if [ "${{ steps.chromiumLegacyInteg.outcome }}" == "failure" ]; then
echo "- Run Playwright LegacyInteg tests in Chromium for prod stage"
fi
exit 1
33 changes: 33 additions & 0 deletions .github/workflows/playwright-chromium-windows.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,35 +26,68 @@ jobs:

- name: Run Playwright unit tests in Chromium
uses: nick-fields/retry@v2
id: chromiumUnit
continue-on-error: true
with:
timeout_minutes: 12
max_attempts: 3
command: npm run testChromium

- name: Run Playwright integration tests in Chromium
uses: nick-fields/retry@v2
id: chromiumIntegration
continue-on-error: true
with:
timeout_minutes: 12
max_attempts: 3
command: npx cross-env TEST_ENV=integration npm run testChromium

- name: Run Playwright mainview tests in Chromium
uses: nick-fields/retry@v2
id: chromiumMainview
continue-on-error: true
with:
timeout_minutes: 12
max_attempts: 3
command: npx cross-env TEST_ENV=mainview npm run testChromium

- name: Run Playwright livepreview tests in Chromium
uses: nick-fields/retry@v2
id: chromiumLivepreview
continue-on-error: true
with:
timeout_minutes: 15
max_attempts: 3
command: npx cross-env TEST_ENV=livepreview npm run testChromium

- name: Run Playwright LegacyInteg tests in Chromium
uses: nick-fields/retry@v2
id: chromiumLegacyInteg
continue-on-error: true
with:
timeout_minutes: 30
max_attempts: 3
command: npx cross-env TEST_ENV=LegacyInteg npm run testChromium

- name: Fail on test runs failed in Chromium Windows
if: steps.chromiumUnit.outcome == 'failure' || steps.chromiumIntegration.outcome == 'failure' || steps.chromiumMainview.outcome == 'failure' || steps.chromiumLivepreview.outcome == 'failure' || steps.chromiumLegacyInteg.outcome == 'failure'
shell: bash
run: |
echo "Chromium Windows Playwright tests failed, marking step as failed"
echo "Failed tests:"
if [ "${{ steps.chromiumUnit.outcome }}" == "failure" ]; then
echo "- Run Playwright unit tests in Chromium"
fi
if [ "${{ steps.chromiumIntegration.outcome }}" == "failure" ]; then
echo "- Run Playwright integration tests in Chromium"
fi
if [ "${{ steps.chromiumMainview.outcome }}" == "failure" ]; then
echo "- Run Playwright mainview tests in Chromium"
fi
if [ "${{ steps.chromiumLivepreview.outcome }}" == "failure" ]; then
echo "- Run Playwright livepreview tests in Chromium"
fi
if [ "${{ steps.chromiumLegacyInteg.outcome }}" == "failure" ]; then
echo "- Run Playwright LegacyInteg tests in Chromium"
fi
exit 1
32 changes: 32 additions & 0 deletions .github/workflows/playwright-firefox-linux.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,35 +26,67 @@ jobs:

- name: Run Playwright unit tests in Firefox
uses: nick-fields/retry@v2
id: firefoxUnit
continue-on-error: true
with:
timeout_minutes: 12
max_attempts: 3
command: npm run testFirefox

- name: Run Playwright integration tests in Firefox
uses: nick-fields/retry@v2
id: firefoxIntegration
continue-on-error: true
with:
timeout_minutes: 12
max_attempts: 3
command: npx cross-env TEST_ENV=integration npm run testFirefox

- name: Run Playwright mainview tests in Firefox
uses: nick-fields/retry@v2
id: firefoxMainview
continue-on-error: true
with:
timeout_minutes: 12
max_attempts: 3
command: npx cross-env TEST_ENV=mainview npm run testFirefox

- name: Run Playwright livepreview tests in Firefox
uses: nick-fields/retry@v2
id: firefoxLivepreview
continue-on-error: true
with:
timeout_minutes: 15
max_attempts: 3
command: npx cross-env TEST_ENV=livepreview npm run testFirefox

- name: Run Playwright LegacyInteg tests in Firefox
uses: nick-fields/retry@v2
id: firefoxLegacyInteg
continue-on-error: true
with:
timeout_minutes: 20
max_attempts: 3
command: npx cross-env TEST_ENV=LegacyInteg npm run testFirefox

- name: Fail on test runs failed in Firefox Linux
if: steps.firefoxUnit.outcome == 'failure' || steps.firefoxIntegration.outcome == 'failure' || steps.firefoxMainview.outcome == 'failure' || steps.firefoxLivepreview.outcome == 'failure' || steps.firefoxLegacyInteg.outcome == 'failure'
run: |
echo "Firefox Linux Playwright tests failed, marking step as failed"
echo "Failed tests:"
if [ "${{ steps.firefoxUnit.outcome }}" == "failure" ]; then
echo "- Run Playwright unit tests in Firefox"
fi
if [ "${{ steps.firefoxIntegration.outcome }}" == "failure" ]; then
echo "- Run Playwright integration tests in Firefox"
fi
if [ "${{ steps.firefoxMainview.outcome }}" == "failure" ]; then
echo "- Run Playwright mainview tests in Firefox"
fi
if [ "${{ steps.firefoxLivepreview.outcome }}" == "failure" ]; then
echo "- Run Playwright livepreview tests in Firefox"
fi
if [ "${{ steps.firefoxLegacyInteg.outcome }}" == "failure" ]; then
echo "- Run Playwright LegacyInteg tests in Firefox"
fi
exit 1
Loading
Loading