From dc4793f389f96bc4df477e829e23eaea2eec5377 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 3 Mar 2026 02:08:53 +0000 Subject: [PATCH 01/10] Initial plan From 1e0291c580f40325c1fb9055eabbf471f26fb2a4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 3 Mar 2026 02:21:36 +0000 Subject: [PATCH 02/10] Run screenshot tests on macOS and Windows in CI - Add ScreenshotTestsMacOS job (macos-latest) to main.yml - Add ScreenshotTestsWindows job (windows-latest) with Mesa3D setup to main.yml - Update build.gradle to add -XstartOnFirstThread on macOS and Mesa3D lib path support - Update TestDriver.java to run app on main thread on macOS (GLFW requirement) - Update screenshot-test-comment.yml to monitor all three platform jobs Co-authored-by: riccardobl <4943530+riccardobl@users.noreply.github.com> --- .github/workflows/main.yml | 75 +++++++++++++++++++ .github/workflows/screenshot-test-comment.yml | 57 +++++++++----- jme3-screenshot-tests/build.gradle | 17 +++++ .../testframework/TestDriver.java | 26 +++++-- 4 files changed, 150 insertions(+), 25 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a46920fb0b..9ea0db0f2f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -96,6 +96,81 @@ jobs: **/build/changed-images/** **/build/test-results/** + ScreenshotTestsMacOS: + name: Run Screenshot Tests (macOS) + runs-on: macos-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + - name: Setup the java environment + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '21' + - name: Validate the Gradle wrapper + uses: gradle/actions/wrapper-validation@v3 + - name: Test with Gradle Wrapper + run: | + ./gradlew :jme3-screenshot-tests:screenshotTest + - name: Upload Test Reports + uses: actions/upload-artifact@master + if: always() + with: + name: screenshot-test-report-macos + retention-days: 30 + path: | + **/build/reports/** + **/build/changed-images/** + **/build/test-results/** + + ScreenshotTestsWindows: + name: Run Screenshot Tests (Windows) + runs-on: windows-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + - name: Setup the java environment + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '21' + - name: Set up Mesa3D for software OpenGL rendering + shell: pwsh + run: | + $version = "24.2.4" + $url = "https://github.com/pal1000/mesa-dist-win/releases/download/$version/mesa3d-$version-release-msvc.7z" + Invoke-WebRequest -Uri $url -OutFile mesa3d.7z + 7z x mesa3d.7z -omesa3d + $mesaLibsPath = "${{ github.workspace }}\mesa-libs" + New-Item -ItemType Directory -Force -Path $mesaLibsPath + # Copy all Mesa DLLs (opengl32.dll and its dependencies) + Copy-Item "mesa3d\x64\*.dll" "$mesaLibsPath\" + echo "MESA3D_LIBS_PATH=$mesaLibsPath" | Out-File -FilePath $env:GITHUB_ENV -Append + - name: Report OpenGL info + shell: pwsh + run: | + $env:MESA_GL_VERSION_OVERRIDE = "4.5" + echo "Mesa3D libs path: $env:MESA3D_LIBS_PATH" + Get-ChildItem "$env:MESA3D_LIBS_PATH" | Select-Object Name + - name: Validate the Gradle wrapper + uses: gradle/actions/wrapper-validation@v3 + - name: Test with Gradle Wrapper + shell: bash + run: | + ./gradlew :jme3-screenshot-tests:screenshotTest + - name: Upload Test Reports + uses: actions/upload-artifact@master + if: always() + with: + name: screenshot-test-report-windows + retention-days: 30 + path: | + **/build/reports/** + **/build/changed-images/** + **/build/test-results/** + # Build iOS natives BuildIosNatives: name: Build natives for iOS diff --git a/.github/workflows/screenshot-test-comment.yml b/.github/workflows/screenshot-test-comment.yml index 5b4ae992e9..1027c27da0 100644 --- a/.github/workflows/screenshot-test-comment.yml +++ b/.github/workflows/screenshot-test-comment.yml @@ -15,7 +15,7 @@ jobs: monitor-screenshot-tests: name: Monitor Screenshot Tests and Comment runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 90 permissions: pull-requests: write contents: read @@ -23,7 +23,7 @@ jobs: - name: Wait for GitHub to register the workflow run run: sleep 120 - - name: Wait for Screenshot Tests to complete + - name: Wait for Screenshot Tests (Linux) to complete uses: lewagon/wait-on-check-action@v1.3.1 with: ref: ${{ github.event.pull_request.head.sha }} @@ -31,6 +31,25 @@ jobs: repo-token: ${{ secrets.GITHUB_TOKEN }} wait-interval: 10 allowed-conclusions: success,skipped,failure + + - name: Wait for Screenshot Tests (macOS) to complete + uses: lewagon/wait-on-check-action@v1.3.1 + with: + ref: ${{ github.event.pull_request.head.sha }} + check-name: 'Run Screenshot Tests (macOS)' + repo-token: ${{ secrets.GITHUB_TOKEN }} + wait-interval: 10 + allowed-conclusions: success,skipped,failure + + - name: Wait for Screenshot Tests (Windows) to complete + uses: lewagon/wait-on-check-action@v1.3.1 + with: + ref: ${{ github.event.pull_request.head.sha }} + check-name: 'Run Screenshot Tests (Windows)' + repo-token: ${{ secrets.GITHUB_TOKEN }} + wait-interval: 10 + allowed-conclusions: success,skipped,failure + - name: Check Screenshot Tests status id: check-status uses: actions/github-script@v6 @@ -47,8 +66,14 @@ jobs: head_sha: ref }); - // Find the ScreenshotTests job - let screenshotTestRun = null; + // Job names to monitor across all platforms + const screenshotJobNames = new Set([ + 'Run Screenshot Tests', + 'Run Screenshot Tests (macOS)', + 'Run Screenshot Tests (Windows)' + ]); + + const failedJobs = []; for (const run of runs.data.workflow_runs) { if (run.name === 'Build jMonkeyEngine') { const jobs = await github.rest.actions.listJobsForWorkflowRun({ @@ -58,26 +83,20 @@ jobs: }); for (const job of jobs.data.jobs) { - if (job.name === 'Run Screenshot Tests') { - screenshotTestRun = job; - break; + if (screenshotJobNames.has(job.name) && job.conclusion === 'failure') { + failedJobs.push(job.name); } } - - if (screenshotTestRun) break; + break; } } - if (!screenshotTestRun) { - console.log('Screenshot test job not found'); - return; - } - - // Check if the job failed - if (screenshotTestRun.conclusion === 'failure') { + if (failedJobs.length > 0) { core.setOutput('failed', 'true'); + core.setOutput('failedJobs', failedJobs.join(', ')); } else { core.setOutput('failed', 'false'); + core.setOutput('failedJobs', ''); } - name: Find Existing Comment uses: peter-evans/find-comment@v3 @@ -95,10 +114,12 @@ jobs: body: | 🖼️ **Screenshot tests have failed.** + Failed jobs: ${{ steps.check-status.outputs.failedJobs }} + The purpose of these tests is to ensure that changes introduced in this PR don't break visual features. They are visual unit tests. 📄 **Where to find the report:** - - Go to the (failed run) > Summary > Artifacts > screenshot-test-report + - Go to the (failed run) > Summary > Artifacts > screenshot-test-report (Linux), screenshot-test-report-macos (macOS), or screenshot-test-report-windows (Windows) - Download the zip and open jme3-screenshot-tests/build/reports/ScreenshotDiffReport.html ⚠️ **If you didn't expect to change anything visual:** @@ -110,7 +131,7 @@ jobs: ✨ **If you are creating entirely new tests:** Find the new images in jme3-screenshot-tests/build/changed-images and commit the new images at jme3-screenshot-tests/src/test/resources. - **Note;** it is very important that the committed reference images are created on the build pipeline, locally created images are not reliable. Similarly tests will fail locally but you can look at the report to check they are "visually similar". + **Note:** The reference images are generated on the Linux CI pipeline. macOS and Windows may produce visually similar but not pixel-identical images due to different GPU drivers or rendering backends. If only macOS or Windows screenshot tests fail but Linux passes, the differences may be platform-specific rendering variations rather than regressions. See https://github.com/jMonkeyEngine/jmonkeyengine/blob/master/jme3-screenshot-tests/README.md for more information diff --git a/jme3-screenshot-tests/build.gradle b/jme3-screenshot-tests/build.gradle index b0f7bd5c1d..d9998f0674 100644 --- a/jme3-screenshot-tests/build.gradle +++ b/jme3-screenshot-tests/build.gradle @@ -2,6 +2,8 @@ plugins { id 'java' } +import org.gradle.internal.os.OperatingSystem + repositories { mavenCentral() } @@ -26,6 +28,21 @@ tasks.register("screenshotTest", Test) { includeTags 'integration' } } + // On macOS, GLFW (used by LWJGL3) requires all window operations on the main OS thread. + // -XstartOnFirstThread makes the JVM main thread the macOS main thread so tests can + // start the JME application directly without going through an executor thread. + if (OperatingSystem.current().isMacOsX()) { + jvmArgs '-XstartOnFirstThread' + } + // On Windows CI, Mesa3D software rendering is needed for OpenGL support. + // Set MESA3D_LIBS_PATH to the directory containing Mesa's opengl32.dll and its + // dependencies before running the tests. LWJGL3 will extract its own natives to the + // same directory, and Windows' DLL search order will then prefer Mesa's opengl32.dll + // over the system one. + def mesaLibsPath = System.getenv("MESA3D_LIBS_PATH") + if (mesaLibsPath) { + systemProperty 'org.lwjgl.librarypath', mesaLibsPath + } } diff --git a/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/TestDriver.java b/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/TestDriver.java index 2d7220de14..4e0801fa28 100644 --- a/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/TestDriver.java +++ b/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/TestDriver.java @@ -153,6 +153,12 @@ public static void bootAppForTest(TestType testType, AppSettings appSettings, St Collections.sort(framesToTakeScreenshotsOn); + // On macOS, GLFW requires all window operations to happen on the main OS thread. + // With -XstartOnFirstThread, the JVM main thread is the macOS main thread, and + // JUnit runs test methods on this thread, so we start the app directly (blocking) + // rather than on a background executor thread. + boolean isMacOs = System.getProperty("os.name", "").toLowerCase().contains("mac"); + List tempFolders = new ArrayList<>(); Map> imageFilesPerScenario = new HashMap<>(); @@ -181,16 +187,22 @@ public static void bootAppForTest(TestType testType, AppSettings appSettings, St app.setShowSettings(false); testDriver.waitLatch = new CountDownLatch(1); - executor.execute(() -> app.start(JmeContext.Type.Display)); - - int maxWaitTimeMilliseconds = 45000; try { - boolean exitedProperly = testDriver.waitLatch.await(maxWaitTimeMilliseconds, TimeUnit.MILLISECONDS); + if (isMacOs) { + // On macOS the app must run on the main thread (GLFW requirement). + // app.start() blocks until the app stops, so no latch wait is needed. + app.start(JmeContext.Type.Display); + } else { + executor.execute(() -> app.start(JmeContext.Type.Display)); - if (!exitedProperly) { - logger.warning("Test driver did not exit in " + maxWaitTimeMilliseconds + "ms. Timed out"); - app.stop(true); + int maxWaitTimeMilliseconds = 45000; + boolean exitedProperly = testDriver.waitLatch.await(maxWaitTimeMilliseconds, TimeUnit.MILLISECONDS); + + if (!exitedProperly) { + logger.warning("Test driver did not exit in " + maxWaitTimeMilliseconds + "ms. Timed out"); + app.stop(true); + } } Thread.sleep(1000); //give time for openGL is fully released before starting a new test (get random JVM crashes without this) From 8f774d39e3ee0aeb81f225d83aac2bc954526c46 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 3 Mar 2026 02:58:13 +0000 Subject: [PATCH 03/10] Fix macOS and Windows screenshot test issues - macOS: change runner to macos-13 (Intel) to avoid NSGL pixel format error on Apple Silicon CI with GLFW 3.4; the Apple OpenGL compat layer on macOS 14+ can fail to find a suitable pixel format in virtualised CI environments - Windows: pre-set onError before app.start() so context-creation failures count down the latch immediately; wrap executor lambda in try/catch for UnsatisfiedLinkError; use app.stop(false) in timeout path to avoid deadlocking if glfwTerminate hangs inside Mesa3D; fix Mesa3D env vars (GALLIUM_DRIVER, MESA_GL_VERSION_OVERRIDE) to persist via GITHUB_ENV; use softpipe driver to avoid slow LLVM JIT startup from llvmpipe - Cross-platform threshold: add jme.screenshotTest.pixelTolerance system property; imagesAreTheSame() now allows per-pixel differences within the tolerance (using existing getMaximumComponentDifference); set tolerance=5 for macOS/Windows in build.gradle so Linux stays the ground truth Co-authored-by: riccardobl <4943530+riccardobl@users.noreply.github.com> --- .github/workflows/main.yml | 19 +++++-- jme3-screenshot-tests/build.gradle | 7 +++ .../testframework/TestDriver.java | 51 ++++++++++++++++--- 3 files changed, 66 insertions(+), 11 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9ea0db0f2f..4604bafd24 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -98,7 +98,11 @@ jobs: ScreenshotTestsMacOS: name: Run Screenshot Tests (macOS) - runs-on: macos-latest + # macos-13 is Intel x86_64 where GLFW/NSOpenGL context creation is well-supported on CI. + # macos-latest (Apple Silicon) can fail with "NSGL: Failed to find a suitable pixel format" + # when running GLFW 3.4 because Apple's OpenGL compatibility layer has restrictions in + # virtualized CI environments. + runs-on: macos-13 permissions: contents: read steps: @@ -145,14 +149,21 @@ jobs: 7z x mesa3d.7z -omesa3d $mesaLibsPath = "${{ github.workspace }}\mesa-libs" New-Item -ItemType Directory -Force -Path $mesaLibsPath - # Copy all Mesa DLLs (opengl32.dll and its dependencies) + # Copy all Mesa DLLs (opengl32.dll and its dependencies) from the x64 directory Copy-Item "mesa3d\x64\*.dll" "$mesaLibsPath\" + # Persist env vars so they are available in subsequent steps echo "MESA3D_LIBS_PATH=$mesaLibsPath" | Out-File -FilePath $env:GITHUB_ENV -Append - - name: Report OpenGL info + # Use softpipe (pure-software renderer) to avoid slow LLVM JIT initialisation + # that llvmpipe requires; softpipe starts up reliably on CI without GPU hardware. + echo "GALLIUM_DRIVER=softpipe" | Out-File -FilePath $env:GITHUB_ENV -Append + # Tell Mesa to advertise OpenGL 3.3 so that LWJGL3's default renderer is satisfied + echo "MESA_GL_VERSION_OVERRIDE=3.3" | Out-File -FilePath $env:GITHUB_ENV -Append + echo "MESA_GLSL_VERSION_OVERRIDE=330" | Out-File -FilePath $env:GITHUB_ENV -Append + - name: Report Mesa3D setup shell: pwsh run: | - $env:MESA_GL_VERSION_OVERRIDE = "4.5" echo "Mesa3D libs path: $env:MESA3D_LIBS_PATH" + echo "GALLIUM_DRIVER: $env:GALLIUM_DRIVER" Get-ChildItem "$env:MESA3D_LIBS_PATH" | Select-Object Name - name: Validate the Gradle wrapper uses: gradle/actions/wrapper-validation@v3 diff --git a/jme3-screenshot-tests/build.gradle b/jme3-screenshot-tests/build.gradle index d9998f0674..fa7c2ba517 100644 --- a/jme3-screenshot-tests/build.gradle +++ b/jme3-screenshot-tests/build.gradle @@ -43,6 +43,13 @@ tasks.register("screenshotTest", Test) { if (mesaLibsPath) { systemProperty 'org.lwjgl.librarypath', mesaLibsPath } + // Reference images are generated on the Linux CI pipeline. On macOS and Windows the + // GPU driver may produce slightly different pixel values for the same scene (e.g. minor + // rounding differences). Allow a small per-channel tolerance so that tests pass despite + // these platform-level rendering variations while still catching real visual regressions. + if (OperatingSystem.current().isMacOsX() || OperatingSystem.current().isWindows()) { + systemProperty 'jme.screenshotTest.pixelTolerance', '5' + } } diff --git a/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/TestDriver.java b/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/TestDriver.java index 4e0801fa28..4fe229d7f5 100644 --- a/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/TestDriver.java +++ b/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/TestDriver.java @@ -80,7 +80,18 @@ public class TestDriver extends BaseAppState{ private static final Logger logger = Logger.getLogger(TestDriver.class.getName()); - public static final String IMAGES_ARE_DIFFERENT = "Generated images is different from committed image. (If you are running the test locally this is expected, images only reproducible on github CI infrastructure)"; + public static final String IMAGES_ARE_DIFFERENT = "Generated image is different from committed image. (If you are running the test locally this is expected, images only reproducible on github CI infrastructure)"; + + /** + * Per-channel pixel tolerance used when comparing screenshots. A value of 0 requires + * exact pixel matching (the default). Set the system property + * {@code jme.screenshotTest.pixelTolerance} to a positive integer to allow minor + * per-pixel differences, which is useful when comparing against Linux-generated + * reference images on other platforms (macOS, Windows) where rendering may differ + * slightly due to different GPU drivers. + */ + private static final int PIXEL_TOLERANCE = Integer.parseInt( + System.getProperty("jme.screenshotTest.pixelTolerance", "0")); public static final String IMAGES_ARE_DIFFERENT_BETWEEN_SCENARIOS = "Images are different between scenarios."; @@ -188,20 +199,41 @@ public static void bootAppForTest(TestType testType, AppSettings appSettings, St testDriver.waitLatch = new CountDownLatch(1); + // Pre-set onError so that context-creation failures (which occur before + // TestDriver.initialize() runs) also count down the latch. + // TestDriver.initialize() will overwrite this with the same behaviour once + // the app has successfully started. + final CountDownLatch capturedLatch = testDriver.waitLatch; + ((App)app).onError = error -> { + logger.log(Level.WARNING, "Error in test application (during startup)", error); + capturedLatch.countDown(); + }; + try { if (isMacOs) { // On macOS the app must run on the main thread (GLFW requirement). // app.start() blocks until the app stops, so no latch wait is needed. app.start(JmeContext.Type.Display); } else { - executor.execute(() -> app.start(JmeContext.Type.Display)); + executor.execute(() -> { + try { + app.start(JmeContext.Type.Display); + } catch (Throwable t) { + // Catch any unexpected exception during app start (e.g. UnsatisfiedLinkError + // if native libraries cannot be loaded) and unblock the waiting test thread. + logger.log(Level.WARNING, "Exception while starting test application", t); + capturedLatch.countDown(); + } + }); int maxWaitTimeMilliseconds = 45000; boolean exitedProperly = testDriver.waitLatch.await(maxWaitTimeMilliseconds, TimeUnit.MILLISECONDS); if (!exitedProperly) { logger.warning("Test driver did not exit in " + maxWaitTimeMilliseconds + "ms. Timed out"); - app.stop(true); + // Use non-blocking stop so this thread doesn't deadlock if the app + // thread is itself stuck (e.g. inside glfwTerminate with Mesa3D). + app.stop(false); } } @@ -421,8 +453,9 @@ private static void attachImage(String title, String fileName, BufferedImage ori } /** - * Tests that the images are the same. If they are not the same it will return false (which may fail the test - * depending on the test type). Different sizes are so fatal that they will immediately fail the test. + * Tests that the images are the same (or within {@link #PIXEL_TOLERANCE}). If they are not + * the same it will return false (which may fail the test depending on the test type). + * Different sizes are so fatal that they will immediately fail the test. */ private static boolean imagesAreTheSame(BufferedImage img1, BufferedImage img2) { if (img1.getWidth() != img2.getWidth() || img1.getHeight() != img2.getHeight()) { @@ -433,8 +466,12 @@ private static boolean imagesAreTheSame(BufferedImage img1, BufferedImage img2) for (int y = 0; y < img1.getHeight(); y++) { for (int x = 0; x < img1.getWidth(); x++) { - if (img1.getRGB(x, y) != img2.getRGB(x, y)){ - return false; + int rgb1 = img1.getRGB(x, y); + int rgb2 = img2.getRGB(x, y); + if (rgb1 != rgb2) { + if (PIXEL_TOLERANCE == 0 || getMaximumComponentDifference(rgb1, rgb2) > PIXEL_TOLERANCE) { + return false; + } } } } From b3b4557598fc580526cfa7240f383fdda7ecd235 Mon Sep 17 00:00:00 2001 From: Riccardo Balbo Date: Tue, 3 Mar 2026 11:01:43 +0100 Subject: [PATCH 04/10] fix window software rendering --- .github/workflows/main.yml | 42 ++++++++++++++++-------------- jme3-screenshot-tests/build.gradle | 10 +------ 2 files changed, 23 insertions(+), 29 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4604bafd24..c1f0cb75df 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -135,11 +135,13 @@ jobs: contents: read steps: - uses: actions/checkout@v4 + - name: Setup the java environment uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '21' + - name: Set up Mesa3D for software OpenGL rendering shell: pwsh run: | @@ -147,32 +149,32 @@ jobs: $url = "https://github.com/pal1000/mesa-dist-win/releases/download/$version/mesa3d-$version-release-msvc.7z" Invoke-WebRequest -Uri $url -OutFile mesa3d.7z 7z x mesa3d.7z -omesa3d - $mesaLibsPath = "${{ github.workspace }}\mesa-libs" - New-Item -ItemType Directory -Force -Path $mesaLibsPath - # Copy all Mesa DLLs (opengl32.dll and its dependencies) from the x64 directory - Copy-Item "mesa3d\x64\*.dll" "$mesaLibsPath\" - # Persist env vars so they are available in subsequent steps - echo "MESA3D_LIBS_PATH=$mesaLibsPath" | Out-File -FilePath $env:GITHUB_ENV -Append - # Use softpipe (pure-software renderer) to avoid slow LLVM JIT initialisation - # that llvmpipe requires; softpipe starts up reliably on CI without GPU hardware. - echo "GALLIUM_DRIVER=softpipe" | Out-File -FilePath $env:GITHUB_ENV -Append - # Tell Mesa to advertise OpenGL 3.3 so that LWJGL3's default renderer is satisfied - echo "MESA_GL_VERSION_OVERRIDE=3.3" | Out-File -FilePath $env:GITHUB_ENV -Append - echo "MESA_GLSL_VERSION_OVERRIDE=330" | Out-File -FilePath $env:GITHUB_ENV -Append - - name: Report Mesa3D setup - shell: pwsh - run: | - echo "Mesa3D libs path: $env:MESA3D_LIBS_PATH" - echo "GALLIUM_DRIVER: $env:GALLIUM_DRIVER" - Get-ChildItem "$env:MESA3D_LIBS_PATH" | Select-Object Name + + $javaBin = Join-Path $env:JAVA_HOME "bin" + + # Deploy Mesa next to java.exe, because java.exe is the actual process executable. + Copy-Item "mesa3d\x64\*.dll" "$javaBin\" -Force + + # Force app-local DLL resolution for java.exe if the process would otherwise bypass it. + New-Item -ItemType File -Force -Path (Join-Path $javaBin "java.exe.local") | Out-Null + + # Start with llvmpipe first; it is Mesa's intended software fallback. + "GALLIUM_DRIVER=llvmpipe" | Out-File -FilePath $env:GITHUB_ENV -Append + + # Remove these until base loading works; add back only if you really need them. + # "MESA_GL_VERSION_OVERRIDE=3.3" | Out-File -FilePath $env:GITHUB_ENV -Append + # "MESA_GLSL_VERSION_OVERRIDE=330" | Out-File -FilePath $env:GITHUB_ENV -Append + - name: Validate the Gradle wrapper uses: gradle/actions/wrapper-validation@v3 + - name: Test with Gradle Wrapper shell: bash run: | - ./gradlew :jme3-screenshot-tests:screenshotTest + ./gradlew --no-daemon :jme3-screenshot-tests:screenshotTest + - name: Upload Test Reports - uses: actions/upload-artifact@master + uses: actions/upload-artifact@v4 if: always() with: name: screenshot-test-report-windows diff --git a/jme3-screenshot-tests/build.gradle b/jme3-screenshot-tests/build.gradle index fa7c2ba517..79fb01454c 100644 --- a/jme3-screenshot-tests/build.gradle +++ b/jme3-screenshot-tests/build.gradle @@ -34,15 +34,7 @@ tasks.register("screenshotTest", Test) { if (OperatingSystem.current().isMacOsX()) { jvmArgs '-XstartOnFirstThread' } - // On Windows CI, Mesa3D software rendering is needed for OpenGL support. - // Set MESA3D_LIBS_PATH to the directory containing Mesa's opengl32.dll and its - // dependencies before running the tests. LWJGL3 will extract its own natives to the - // same directory, and Windows' DLL search order will then prefer Mesa's opengl32.dll - // over the system one. - def mesaLibsPath = System.getenv("MESA3D_LIBS_PATH") - if (mesaLibsPath) { - systemProperty 'org.lwjgl.librarypath', mesaLibsPath - } + // Reference images are generated on the Linux CI pipeline. On macOS and Windows the // GPU driver may produce slightly different pixel values for the same scene (e.g. minor // rounding differences). Allow a small per-channel tolerance so that tests pass despite From ddb6c2e6fb21122cb11c53a1227151e996110e4f Mon Sep 17 00:00:00 2001 From: Riccardo Balbo Date: Tue, 3 Mar 2026 11:19:09 +0100 Subject: [PATCH 05/10] use macos 15 intel runner --- .github/workflows/main.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c1f0cb75df..d3991553b3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -98,27 +98,26 @@ jobs: ScreenshotTestsMacOS: name: Run Screenshot Tests (macOS) - # macos-13 is Intel x86_64 where GLFW/NSOpenGL context creation is well-supported on CI. - # macos-latest (Apple Silicon) can fail with "NSGL: Failed to find a suitable pixel format" - # when running GLFW 3.4 because Apple's OpenGL compatibility layer has restrictions in - # virtualized CI environments. - runs-on: macos-13 + runs-on: macos-15-intel permissions: contents: read steps: - uses: actions/checkout@v4 + - name: Setup the java environment uses: actions/setup-java@v4 with: - distribution: 'temurin' + distribution: temurin java-version: '21' + - name: Validate the Gradle wrapper uses: gradle/actions/wrapper-validation@v3 + - name: Test with Gradle Wrapper - run: | - ./gradlew :jme3-screenshot-tests:screenshotTest + run: ./gradlew --no-daemon :jme3-screenshot-tests:screenshotTest + - name: Upload Test Reports - uses: actions/upload-artifact@master + uses: actions/upload-artifact@v4 if: always() with: name: screenshot-test-report-macos From 3fa18f41272bd9e087e4b70b3825f5dea5909733 Mon Sep 17 00:00:00 2001 From: Riccardo Balbo Date: Tue, 3 Mar 2026 13:59:27 +0100 Subject: [PATCH 06/10] relax settings for mac --- .../screenshottests/testframework/TestDriver.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/TestDriver.java b/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/TestDriver.java index 4fe229d7f5..522c9b7ff0 100644 --- a/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/TestDriver.java +++ b/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/TestDriver.java @@ -153,7 +153,19 @@ public void update(float tpf){ @Override protected void onEnable(){} @Override protected void onDisable(){} - + + private static AppSettings applyPlatformSettings(AppSettings settings) { + settings = new AppSettings(settings); + String os = System.getProperty("os.name", "").toLowerCase(); + if (os.contains("mac")) { + settings.setSamples(0); + settings.setGammaCorrection(false); + settings.setStencilBits(0); + settings.setUseRetinaFrameBuffer(false); + } + return settings; + } + /** * Boots up the application on a separate thread (blocks this thread) and then does the following: * - Takes screenshots on the requested frames @@ -194,6 +206,7 @@ public static void bootAppForTest(TestType testType, AppSettings appSettings, St states.add(testDriver); SimpleApplication app = new App(states.toArray(new AppState[0])); + appSettings = applyPlatformSettings(appSettings); app.setSettings(appSettings); app.setShowSettings(false); From 0ebc16b0c23047770374c540b081c8eca3e687e5 Mon Sep 17 00:00:00 2001 From: Riccardo Balbo Date: Tue, 3 Mar 2026 14:14:32 +0100 Subject: [PATCH 07/10] fix --- .../jmonkeyengine/screenshottests/testframework/TestDriver.java | 1 - 1 file changed, 1 deletion(-) diff --git a/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/TestDriver.java b/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/TestDriver.java index 522c9b7ff0..9b194fe40f 100644 --- a/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/TestDriver.java +++ b/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/TestDriver.java @@ -155,7 +155,6 @@ public void update(float tpf){ @Override protected void onDisable(){} private static AppSettings applyPlatformSettings(AppSettings settings) { - settings = new AppSettings(settings); String os = System.getProperty("os.name", "").toLowerCase(); if (os.contains("mac")) { settings.setSamples(0); From d955cbd33cae30e82147d90f272e1ae31044a28a Mon Sep 17 00:00:00 2001 From: Riccardo Balbo Date: Tue, 3 Mar 2026 18:30:41 +0100 Subject: [PATCH 08/10] move screenshot tests into a separate workflow and attempt at fixing for macos --- .github/workflows/integration.yml | 207 ++++++++++++++++++ .github/workflows/main.yml | 128 +---------- .github/workflows/screenshot-test-comment.yml | 39 ++-- 3 files changed, 230 insertions(+), 144 deletions(-) create mode 100644 .github/workflows/integration.yml diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 0000000000..3b1f093a73 --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,207 @@ +name: Integration + +on: + push: + branches: + - master + - v3.7 + - v3.6 + - v3.5 + - v3.4 + - v3.3 + - ios-2024_2 + pull_request: + workflow_dispatch: + +jobs: + ScreenshotTests: + name: Run Screenshot Tests + runs-on: ubuntu-latest + container: + image: ghcr.io/onemillionworlds/opengl-docker-image:v1 + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + + - name: Setup the java environment + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '21' + + - name: Start xvfb + run: | + Xvfb :99 -ac -screen 0 1024x768x16 & + export DISPLAY=:99 + echo "DISPLAY=:99" >> $GITHUB_ENV + + - name: Report GL/Vulkan + run: | + set -x + echo "DISPLAY=$DISPLAY" + glxinfo | grep -E "OpenGL version|OpenGL renderer|OpenGL vendor" || true + vulkaninfo --summary || true + echo "VK_ICD_FILENAMES=$VK_ICD_FILENAMES" + echo "MESA_LOADER_DRIVER_OVERRIDE=$MESA_LOADER_DRIVER_OVERRIDE" + echo "GALLIUM_DRIVER=$GALLIUM_DRIVER" + + - name: Validate the Gradle wrapper + uses: gradle/actions/wrapper-validation@v3 + + - name: Test with Gradle Wrapper + run: | + ./gradlew --no-daemon :jme3-screenshot-tests:screenshotTest + + - name: Upload Test Reports + uses: actions/upload-artifact@v4 + if: always() + with: + name: screenshot-test-report + retention-days: 30 + path: | + **/build/reports/** + **/build/changed-images/** + **/build/test-results/** + + ScreenshotTestsMacOS: + name: Run Screenshot Tests (macOS) + runs-on: macos-15-intel + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + + - name: Setup the java environment + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '21' + + - name: Validate the Gradle wrapper + uses: gradle/actions/wrapper-validation@v3 + + - name: Prepare patched GLFW for macOS software renderer + shell: bash + run: | + set -euo pipefail + + if ! command -v cmake >/dev/null 2>&1; then + brew install cmake + fi + + curl -L -o glfw-3.4.tar.gz https://github.com/glfw/glfw/archive/refs/tags/3.4.tar.gz + tar -xzf glfw-3.4.tar.gz + mv glfw-3.4 glfw-src + + python3 - <<'PY' + from pathlib import Path + + p = Path("glfw-src/src/nsgl_context.m") + text = p.read_text() + + old = " ADD_ATTRIB(NSOpenGLPFAAccelerated);" + new = ( + " ADD_ATTRIB(NSOpenGLPFARendererID);\n" + " ADD_ATTRIB(kCGLRendererGenericFloatID);" + ) + + if old not in text: + raise SystemExit("Patch anchor not found in src/nsgl_context.m") + + p.write_text(text.replace(old, new, 1)) + print("Patched src/nsgl_context.m") + PY + + cmake -S glfw-src -B glfw-build \ + -D BUILD_SHARED_LIBS=ON \ + -D GLFW_BUILD_EXAMPLES=OFF \ + -D GLFW_BUILD_TESTS=OFF \ + -D GLFW_BUILD_DOCS=OFF \ + -D CMAKE_BUILD_TYPE=Release \ + -D CMAKE_OSX_ARCHITECTURES=x86_64 + + cmake --build glfw-build --config Release --parallel 3 + + mkdir -p ci-glfw + if [ -f glfw-build/src/libglfw.3.dylib ]; then + cp glfw-build/src/libglfw.3.dylib ci-glfw/libglfw-ci.dylib + elif [ -f glfw-build/src/libglfw.dylib ]; then + cp glfw-build/src/libglfw.dylib ci-glfw/libglfw-ci.dylib + else + echo "Could not find built GLFW dylib" + find glfw-build -name '*.dylib' -print + exit 1 + fi + + file ci-glfw/libglfw-ci.dylib + otool -L ci-glfw/libglfw-ci.dylib + + - name: Test with Gradle Wrapper + env: + JAVA_TOOL_OPTIONS: -Dorg.lwjgl.glfw.libname=${{ github.workspace }}/ci-glfw/libglfw-ci.dylib + run: | + ./gradlew --no-daemon :jme3-screenshot-tests:screenshotTest + + - name: Upload Test Reports + uses: actions/upload-artifact@v4 + if: always() + with: + name: screenshot-test-report-macos + retention-days: 30 + path: | + **/build/reports/** + **/build/changed-images/** + **/build/test-results/** + + ScreenshotTestsWindows: + name: Run Screenshot Tests (Windows) + runs-on: windows-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + + - name: Setup the java environment + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '21' + + - name: Set up Mesa3D for software OpenGL rendering + shell: pwsh + run: | + $version = "24.2.4" + $url = "https://github.com/pal1000/mesa-dist-win/releases/download/$version/mesa3d-$version-release-msvc.7z" + Invoke-WebRequest -Uri $url -OutFile mesa3d.7z + 7z x mesa3d.7z -omesa3d + + $javaBin = Join-Path $env:JAVA_HOME "bin" + + # Deploy Mesa next to java.exe, because java.exe is the actual process executable. + Copy-Item "mesa3d\x64\*.dll" "$javaBin\" -Force + + # Force app-local DLL resolution for java.exe if the process would otherwise bypass it. + New-Item -ItemType File -Force -Path (Join-Path $javaBin "java.exe.local") | Out-Null + + # Start with llvmpipe first; it is Mesa's intended software fallback. + "GALLIUM_DRIVER=llvmpipe" | Out-File -FilePath $env:GITHUB_ENV -Append + + - name: Validate the Gradle wrapper + uses: gradle/actions/wrapper-validation@v3 + + - name: Test with Gradle Wrapper + shell: bash + run: | + ./gradlew --no-daemon :jme3-screenshot-tests:screenshotTest + + - name: Upload Test Reports + uses: actions/upload-artifact@v4 + if: always() + with: + name: screenshot-test-report-windows + retention-days: 30 + path: | + **/build/reports/** + **/build/changed-images/** + **/build/test-results/** \ No newline at end of file diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d3991553b3..27d584e674 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -57,132 +57,6 @@ on: types: [published] jobs: - ScreenshotTests: - name: Run Screenshot Tests - runs-on: ubuntu-latest - container: - image: ghcr.io/onemillionworlds/opengl-docker-image:v1 - permissions: - contents: read - steps: - - uses: actions/checkout@v4 - - name: Start xvfb - run: | - Xvfb :99 -ac -screen 0 1024x768x16 & - export DISPLAY=:99 - echo "DISPLAY=:99" >> $GITHUB_ENV - - name: Report GL/Vulkan - run: | - set -x - echo "DISPLAY=$DISPLAY" - glxinfo | grep -E "OpenGL version|OpenGL renderer|OpenGL vendor" || true - vulkaninfo --summary || true - echo "VK_ICD_FILENAMES=$VK_ICD_FILENAMES" - echo "MESA_LOADER_DRIVER_OVERRIDE=$MESA_LOADER_DRIVER_OVERRIDE" - echo "GALLIUM_DRIVER=$GALLIUM_DRIVER" - - name: Validate the Gradle wrapper - uses: gradle/actions/wrapper-validation@v3 - - name: Test with Gradle Wrapper - run: | - ./gradlew :jme3-screenshot-test:screenshotTest - - name: Upload Test Reports - uses: actions/upload-artifact@master - if: always() - with: - name: screenshot-test-report - retention-days: 30 - path: | - **/build/reports/** - **/build/changed-images/** - **/build/test-results/** - - ScreenshotTestsMacOS: - name: Run Screenshot Tests (macOS) - runs-on: macos-15-intel - permissions: - contents: read - steps: - - uses: actions/checkout@v4 - - - name: Setup the java environment - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: '21' - - - name: Validate the Gradle wrapper - uses: gradle/actions/wrapper-validation@v3 - - - name: Test with Gradle Wrapper - run: ./gradlew --no-daemon :jme3-screenshot-tests:screenshotTest - - - name: Upload Test Reports - uses: actions/upload-artifact@v4 - if: always() - with: - name: screenshot-test-report-macos - retention-days: 30 - path: | - **/build/reports/** - **/build/changed-images/** - **/build/test-results/** - - ScreenshotTestsWindows: - name: Run Screenshot Tests (Windows) - runs-on: windows-latest - permissions: - contents: read - steps: - - uses: actions/checkout@v4 - - - name: Setup the java environment - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: '21' - - - name: Set up Mesa3D for software OpenGL rendering - shell: pwsh - run: | - $version = "24.2.4" - $url = "https://github.com/pal1000/mesa-dist-win/releases/download/$version/mesa3d-$version-release-msvc.7z" - Invoke-WebRequest -Uri $url -OutFile mesa3d.7z - 7z x mesa3d.7z -omesa3d - - $javaBin = Join-Path $env:JAVA_HOME "bin" - - # Deploy Mesa next to java.exe, because java.exe is the actual process executable. - Copy-Item "mesa3d\x64\*.dll" "$javaBin\" -Force - - # Force app-local DLL resolution for java.exe if the process would otherwise bypass it. - New-Item -ItemType File -Force -Path (Join-Path $javaBin "java.exe.local") | Out-Null - - # Start with llvmpipe first; it is Mesa's intended software fallback. - "GALLIUM_DRIVER=llvmpipe" | Out-File -FilePath $env:GITHUB_ENV -Append - - # Remove these until base loading works; add back only if you really need them. - # "MESA_GL_VERSION_OVERRIDE=3.3" | Out-File -FilePath $env:GITHUB_ENV -Append - # "MESA_GLSL_VERSION_OVERRIDE=330" | Out-File -FilePath $env:GITHUB_ENV -Append - - - name: Validate the Gradle wrapper - uses: gradle/actions/wrapper-validation@v3 - - - name: Test with Gradle Wrapper - shell: bash - run: | - ./gradlew --no-daemon :jme3-screenshot-tests:screenshotTest - - - name: Upload Test Reports - uses: actions/upload-artifact@v4 - if: always() - with: - name: screenshot-test-report-windows - retention-days: 30 - path: | - **/build/reports/** - **/build/changed-images/** - **/build/test-results/** - # Build iOS natives BuildIosNatives: name: Build natives for iOS @@ -709,4 +583,4 @@ jobs: branch="gh-pages" git push origin "$branch" --force || true - fi + fi \ No newline at end of file diff --git a/.github/workflows/screenshot-test-comment.yml b/.github/workflows/screenshot-test-comment.yml index 1027c27da0..805957fdfc 100644 --- a/.github/workflows/screenshot-test-comment.yml +++ b/.github/workflows/screenshot-test-comment.yml @@ -73,31 +73,36 @@ jobs: 'Run Screenshot Tests (Windows)' ]); - const failedJobs = []; + const failedJobs = new Set(); + for (const run of runs.data.workflow_runs) { - if (run.name === 'Build jMonkeyEngine') { - const jobs = await github.rest.actions.listJobsForWorkflowRun({ - owner, - repo, - run_id: run.id - }); - - for (const job of jobs.data.jobs) { - if (screenshotJobNames.has(job.name) && job.conclusion === 'failure') { - failedJobs.push(job.name); - } + if (run.name !== 'Integration') { + continue; + } + + const jobs = await github.rest.actions.listJobsForWorkflowRun({ + owner, + repo, + run_id: run.id + }); + + for (const job of jobs.data.jobs) { + if (screenshotJobNames.has(job.name) && job.conclusion === 'failure') { + failedJobs.add(job.name); } - break; } } - if (failedJobs.length > 0) { + const failedList = Array.from(failedJobs); + + if (failedList.length > 0) { core.setOutput('failed', 'true'); - core.setOutput('failedJobs', failedJobs.join(', ')); + core.setOutput('failedJobs', failedList.join(', ')); } else { core.setOutput('failed', 'false'); core.setOutput('failedJobs', ''); } + - name: Find Existing Comment uses: peter-evans/find-comment@v3 id: existingCommentId @@ -119,7 +124,7 @@ jobs: The purpose of these tests is to ensure that changes introduced in this PR don't break visual features. They are visual unit tests. 📄 **Where to find the report:** - - Go to the (failed run) > Summary > Artifacts > screenshot-test-report (Linux), screenshot-test-report-macos (macOS), or screenshot-test-report-windows (Windows) + - Go to the failed run in the Integration workflow > Summary > Artifacts > screenshot-test-report (Linux), screenshot-test-report-macos (macOS), or screenshot-test-report-windows (Windows) - Download the zip and open jme3-screenshot-tests/build/reports/ScreenshotDiffReport.html ⚠️ **If you didn't expect to change anything visual:** @@ -137,4 +142,4 @@ jobs: Contact @richardTingle (aka richtea) for guidance if required edit-mode: replace - comment-id: ${{ steps.existingCommentId.outputs.comment-id }} + comment-id: ${{ steps.existingCommentId.outputs.comment-id }} \ No newline at end of file From 952ab7c078c5ac91b2a09ab5e20c76347f21addc Mon Sep 17 00:00:00 2001 From: Riccardo Balbo Date: Tue, 3 Mar 2026 21:38:01 +0100 Subject: [PATCH 09/10] fix macos getting stuck on tests --- .github/workflows/integration.yml | 3 ++- .../screenshottests/testframework/TestDriver.java | 10 ++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 3b1f093a73..ac14bbad61 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -138,10 +138,11 @@ jobs: otool -L ci-glfw/libglfw-ci.dylib - name: Test with Gradle Wrapper + timeout-minutes: 20 env: JAVA_TOOL_OPTIONS: -Dorg.lwjgl.glfw.libname=${{ github.workspace }}/ci-glfw/libglfw-ci.dylib run: | - ./gradlew --no-daemon :jme3-screenshot-tests:screenshotTest + ./gradlew --no-daemon --info :jme3-screenshot-tests:screenshotTest - name: Upload Test Reports uses: actions/upload-artifact@v4 diff --git a/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/TestDriver.java b/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/TestDriver.java index 9b194fe40f..9554965e91 100644 --- a/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/TestDriver.java +++ b/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/TestDriver.java @@ -126,15 +126,17 @@ public TestDriver(ScreenshotNoInputAppState screenshotAppState, Collection= tickToTerminateApp){ - getApplication().stop(true); + + if (tick >= tickToTerminateApp) { waitLatch.countDown(); + getApplication().stop(false); + return; } tick++; From 8811e085bb3d9be1c7a89a56e2936fb0d47b2647 Mon Sep 17 00:00:00 2001 From: Riccardo Balbo Date: Wed, 4 Mar 2026 00:16:55 +0100 Subject: [PATCH 10/10] use apple silicon --- .github/workflows/integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index ac14bbad61..d74d86ea8e 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -66,7 +66,7 @@ jobs: ScreenshotTestsMacOS: name: Run Screenshot Tests (macOS) - runs-on: macos-15-intel + runs-on: macos-latest permissions: contents: read steps: