diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 0000000000..d74d86ea8e --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,208 @@ +name: Integration + +on: + push: + branches: + - master + - v3.7 + - v3.6 + - v3.5 + - v3.4 + - v3.3 + - ios-2024_2 + pull_request: + workflow_dispatch: + +jobs: + ScreenshotTests: + name: Run Screenshot Tests + runs-on: ubuntu-latest + container: + image: ghcr.io/onemillionworlds/opengl-docker-image:v1 + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + + - name: Setup the java environment + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '21' + + - name: Start xvfb + run: | + Xvfb :99 -ac -screen 0 1024x768x16 & + export DISPLAY=:99 + echo "DISPLAY=:99" >> $GITHUB_ENV + + - name: Report GL/Vulkan + run: | + set -x + echo "DISPLAY=$DISPLAY" + glxinfo | grep -E "OpenGL version|OpenGL renderer|OpenGL vendor" || true + vulkaninfo --summary || true + echo "VK_ICD_FILENAMES=$VK_ICD_FILENAMES" + echo "MESA_LOADER_DRIVER_OVERRIDE=$MESA_LOADER_DRIVER_OVERRIDE" + echo "GALLIUM_DRIVER=$GALLIUM_DRIVER" + + - name: Validate the Gradle wrapper + uses: gradle/actions/wrapper-validation@v3 + + - name: Test with Gradle Wrapper + run: | + ./gradlew --no-daemon :jme3-screenshot-tests:screenshotTest + + - name: Upload Test Reports + uses: actions/upload-artifact@v4 + if: always() + with: + name: screenshot-test-report + retention-days: 30 + path: | + **/build/reports/** + **/build/changed-images/** + **/build/test-results/** + + ScreenshotTestsMacOS: + name: Run Screenshot Tests (macOS) + runs-on: macos-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + + - name: Setup the java environment + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '21' + + - name: Validate the Gradle wrapper + uses: gradle/actions/wrapper-validation@v3 + + - name: Prepare patched GLFW for macOS software renderer + shell: bash + run: | + set -euo pipefail + + if ! command -v cmake >/dev/null 2>&1; then + brew install cmake + fi + + curl -L -o glfw-3.4.tar.gz https://github.com/glfw/glfw/archive/refs/tags/3.4.tar.gz + tar -xzf glfw-3.4.tar.gz + mv glfw-3.4 glfw-src + + python3 - <<'PY' + from pathlib import Path + + p = Path("glfw-src/src/nsgl_context.m") + text = p.read_text() + + old = " ADD_ATTRIB(NSOpenGLPFAAccelerated);" + new = ( + " ADD_ATTRIB(NSOpenGLPFARendererID);\n" + " ADD_ATTRIB(kCGLRendererGenericFloatID);" + ) + + if old not in text: + raise SystemExit("Patch anchor not found in src/nsgl_context.m") + + p.write_text(text.replace(old, new, 1)) + print("Patched src/nsgl_context.m") + PY + + cmake -S glfw-src -B glfw-build \ + -D BUILD_SHARED_LIBS=ON \ + -D GLFW_BUILD_EXAMPLES=OFF \ + -D GLFW_BUILD_TESTS=OFF \ + -D GLFW_BUILD_DOCS=OFF \ + -D CMAKE_BUILD_TYPE=Release \ + -D CMAKE_OSX_ARCHITECTURES=x86_64 + + cmake --build glfw-build --config Release --parallel 3 + + mkdir -p ci-glfw + if [ -f glfw-build/src/libglfw.3.dylib ]; then + cp glfw-build/src/libglfw.3.dylib ci-glfw/libglfw-ci.dylib + elif [ -f glfw-build/src/libglfw.dylib ]; then + cp glfw-build/src/libglfw.dylib ci-glfw/libglfw-ci.dylib + else + echo "Could not find built GLFW dylib" + find glfw-build -name '*.dylib' -print + exit 1 + fi + + file ci-glfw/libglfw-ci.dylib + otool -L ci-glfw/libglfw-ci.dylib + + - name: Test with Gradle Wrapper + timeout-minutes: 20 + env: + JAVA_TOOL_OPTIONS: -Dorg.lwjgl.glfw.libname=${{ github.workspace }}/ci-glfw/libglfw-ci.dylib + run: | + ./gradlew --no-daemon --info :jme3-screenshot-tests:screenshotTest + + - name: Upload Test Reports + uses: actions/upload-artifact@v4 + if: always() + with: + name: screenshot-test-report-macos + retention-days: 30 + path: | + **/build/reports/** + **/build/changed-images/** + **/build/test-results/** + + ScreenshotTestsWindows: + name: Run Screenshot Tests (Windows) + runs-on: windows-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + + - name: Setup the java environment + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '21' + + - name: Set up Mesa3D for software OpenGL rendering + shell: pwsh + run: | + $version = "24.2.4" + $url = "https://github.com/pal1000/mesa-dist-win/releases/download/$version/mesa3d-$version-release-msvc.7z" + Invoke-WebRequest -Uri $url -OutFile mesa3d.7z + 7z x mesa3d.7z -omesa3d + + $javaBin = Join-Path $env:JAVA_HOME "bin" + + # Deploy Mesa next to java.exe, because java.exe is the actual process executable. + Copy-Item "mesa3d\x64\*.dll" "$javaBin\" -Force + + # Force app-local DLL resolution for java.exe if the process would otherwise bypass it. + New-Item -ItemType File -Force -Path (Join-Path $javaBin "java.exe.local") | Out-Null + + # Start with llvmpipe first; it is Mesa's intended software fallback. + "GALLIUM_DRIVER=llvmpipe" | Out-File -FilePath $env:GITHUB_ENV -Append + + - name: Validate the Gradle wrapper + uses: gradle/actions/wrapper-validation@v3 + + - name: Test with Gradle Wrapper + shell: bash + run: | + ./gradlew --no-daemon :jme3-screenshot-tests:screenshotTest + + - name: Upload Test Reports + uses: actions/upload-artifact@v4 + if: always() + with: + name: screenshot-test-report-windows + retention-days: 30 + path: | + **/build/reports/** + **/build/changed-images/** + **/build/test-results/** \ No newline at end of file diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a46920fb0b..27d584e674 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -57,45 +57,6 @@ on: types: [published] jobs: - ScreenshotTests: - name: Run Screenshot Tests - runs-on: ubuntu-latest - container: - image: ghcr.io/onemillionworlds/opengl-docker-image:v1 - permissions: - contents: read - steps: - - uses: actions/checkout@v4 - - name: Start xvfb - run: | - Xvfb :99 -ac -screen 0 1024x768x16 & - export DISPLAY=:99 - echo "DISPLAY=:99" >> $GITHUB_ENV - - name: Report GL/Vulkan - run: | - set -x - echo "DISPLAY=$DISPLAY" - glxinfo | grep -E "OpenGL version|OpenGL renderer|OpenGL vendor" || true - vulkaninfo --summary || true - echo "VK_ICD_FILENAMES=$VK_ICD_FILENAMES" - echo "MESA_LOADER_DRIVER_OVERRIDE=$MESA_LOADER_DRIVER_OVERRIDE" - echo "GALLIUM_DRIVER=$GALLIUM_DRIVER" - - name: Validate the Gradle wrapper - uses: gradle/actions/wrapper-validation@v3 - - name: Test with Gradle Wrapper - run: | - ./gradlew :jme3-screenshot-test:screenshotTest - - name: Upload Test Reports - uses: actions/upload-artifact@master - if: always() - with: - name: screenshot-test-report - retention-days: 30 - path: | - **/build/reports/** - **/build/changed-images/** - **/build/test-results/** - # Build iOS natives BuildIosNatives: name: Build natives for iOS @@ -622,4 +583,4 @@ jobs: branch="gh-pages" git push origin "$branch" --force || true - fi + fi \ No newline at end of file diff --git a/.github/workflows/screenshot-test-comment.yml b/.github/workflows/screenshot-test-comment.yml index 5b4ae992e9..805957fdfc 100644 --- a/.github/workflows/screenshot-test-comment.yml +++ b/.github/workflows/screenshot-test-comment.yml @@ -15,7 +15,7 @@ jobs: monitor-screenshot-tests: name: Monitor Screenshot Tests and Comment runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 90 permissions: pull-requests: write contents: read @@ -23,7 +23,7 @@ jobs: - name: Wait for GitHub to register the workflow run run: sleep 120 - - name: Wait for Screenshot Tests to complete + - name: Wait for Screenshot Tests (Linux) to complete uses: lewagon/wait-on-check-action@v1.3.1 with: ref: ${{ github.event.pull_request.head.sha }} @@ -31,6 +31,25 @@ jobs: repo-token: ${{ secrets.GITHUB_TOKEN }} wait-interval: 10 allowed-conclusions: success,skipped,failure + + - name: Wait for Screenshot Tests (macOS) to complete + uses: lewagon/wait-on-check-action@v1.3.1 + with: + ref: ${{ github.event.pull_request.head.sha }} + check-name: 'Run Screenshot Tests (macOS)' + repo-token: ${{ secrets.GITHUB_TOKEN }} + wait-interval: 10 + allowed-conclusions: success,skipped,failure + + - name: Wait for Screenshot Tests (Windows) to complete + uses: lewagon/wait-on-check-action@v1.3.1 + with: + ref: ${{ github.event.pull_request.head.sha }} + check-name: 'Run Screenshot Tests (Windows)' + repo-token: ${{ secrets.GITHUB_TOKEN }} + wait-interval: 10 + allowed-conclusions: success,skipped,failure + - name: Check Screenshot Tests status id: check-status uses: actions/github-script@v6 @@ -47,38 +66,43 @@ jobs: head_sha: ref }); - // Find the ScreenshotTests job - let screenshotTestRun = null; + // Job names to monitor across all platforms + const screenshotJobNames = new Set([ + 'Run Screenshot Tests', + 'Run Screenshot Tests (macOS)', + 'Run Screenshot Tests (Windows)' + ]); + + const failedJobs = new Set(); + for (const run of runs.data.workflow_runs) { - if (run.name === 'Build jMonkeyEngine') { - const jobs = await github.rest.actions.listJobsForWorkflowRun({ - owner, - repo, - run_id: run.id - }); - - for (const job of jobs.data.jobs) { - if (job.name === 'Run Screenshot Tests') { - screenshotTestRun = job; - break; - } - } + if (run.name !== 'Integration') { + continue; + } + + const jobs = await github.rest.actions.listJobsForWorkflowRun({ + owner, + repo, + run_id: run.id + }); - if (screenshotTestRun) break; + for (const job of jobs.data.jobs) { + if (screenshotJobNames.has(job.name) && job.conclusion === 'failure') { + failedJobs.add(job.name); + } } } - if (!screenshotTestRun) { - console.log('Screenshot test job not found'); - return; - } + const failedList = Array.from(failedJobs); - // Check if the job failed - if (screenshotTestRun.conclusion === 'failure') { + if (failedList.length > 0) { core.setOutput('failed', 'true'); + core.setOutput('failedJobs', failedList.join(', ')); } else { core.setOutput('failed', 'false'); + core.setOutput('failedJobs', ''); } + - name: Find Existing Comment uses: peter-evans/find-comment@v3 id: existingCommentId @@ -95,10 +119,12 @@ jobs: body: | 🖼️ **Screenshot tests have failed.** + Failed jobs: ${{ steps.check-status.outputs.failedJobs }} + The purpose of these tests is to ensure that changes introduced in this PR don't break visual features. They are visual unit tests. 📄 **Where to find the report:** - - Go to the (failed run) > Summary > Artifacts > screenshot-test-report + - Go to the failed run in the Integration workflow > Summary > Artifacts > screenshot-test-report (Linux), screenshot-test-report-macos (macOS), or screenshot-test-report-windows (Windows) - Download the zip and open jme3-screenshot-tests/build/reports/ScreenshotDiffReport.html ⚠️ **If you didn't expect to change anything visual:** @@ -110,10 +136,10 @@ jobs: ✨ **If you are creating entirely new tests:** Find the new images in jme3-screenshot-tests/build/changed-images and commit the new images at jme3-screenshot-tests/src/test/resources. - **Note;** it is very important that the committed reference images are created on the build pipeline, locally created images are not reliable. Similarly tests will fail locally but you can look at the report to check they are "visually similar". + **Note:** The reference images are generated on the Linux CI pipeline. macOS and Windows may produce visually similar but not pixel-identical images due to different GPU drivers or rendering backends. If only macOS or Windows screenshot tests fail but Linux passes, the differences may be platform-specific rendering variations rather than regressions. See https://github.com/jMonkeyEngine/jmonkeyengine/blob/master/jme3-screenshot-tests/README.md for more information Contact @richardTingle (aka richtea) for guidance if required edit-mode: replace - comment-id: ${{ steps.existingCommentId.outputs.comment-id }} + comment-id: ${{ steps.existingCommentId.outputs.comment-id }} \ No newline at end of file diff --git a/jme3-screenshot-tests/build.gradle b/jme3-screenshot-tests/build.gradle index b0f7bd5c1d..79fb01454c 100644 --- a/jme3-screenshot-tests/build.gradle +++ b/jme3-screenshot-tests/build.gradle @@ -2,6 +2,8 @@ plugins { id 'java' } +import org.gradle.internal.os.OperatingSystem + repositories { mavenCentral() } @@ -26,6 +28,20 @@ tasks.register("screenshotTest", Test) { includeTags 'integration' } } + // On macOS, GLFW (used by LWJGL3) requires all window operations on the main OS thread. + // -XstartOnFirstThread makes the JVM main thread the macOS main thread so tests can + // start the JME application directly without going through an executor thread. + if (OperatingSystem.current().isMacOsX()) { + jvmArgs '-XstartOnFirstThread' + } + + // Reference images are generated on the Linux CI pipeline. On macOS and Windows the + // GPU driver may produce slightly different pixel values for the same scene (e.g. minor + // rounding differences). Allow a small per-channel tolerance so that tests pass despite + // these platform-level rendering variations while still catching real visual regressions. + if (OperatingSystem.current().isMacOsX() || OperatingSystem.current().isWindows()) { + systemProperty 'jme.screenshotTest.pixelTolerance', '5' + } } diff --git a/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/TestDriver.java b/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/TestDriver.java index 2d7220de14..9554965e91 100644 --- a/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/TestDriver.java +++ b/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/TestDriver.java @@ -80,7 +80,18 @@ public class TestDriver extends BaseAppState{ private static final Logger logger = Logger.getLogger(TestDriver.class.getName()); - public static final String IMAGES_ARE_DIFFERENT = "Generated images is different from committed image. (If you are running the test locally this is expected, images only reproducible on github CI infrastructure)"; + public static final String IMAGES_ARE_DIFFERENT = "Generated image is different from committed image. (If you are running the test locally this is expected, images only reproducible on github CI infrastructure)"; + + /** + * Per-channel pixel tolerance used when comparing screenshots. A value of 0 requires + * exact pixel matching (the default). Set the system property + * {@code jme.screenshotTest.pixelTolerance} to a positive integer to allow minor + * per-pixel differences, which is useful when comparing against Linux-generated + * reference images on other platforms (macOS, Windows) where rendering may differ + * slightly due to different GPU drivers. + */ + private static final int PIXEL_TOLERANCE = Integer.parseInt( + System.getProperty("jme.screenshotTest.pixelTolerance", "0")); public static final String IMAGES_ARE_DIFFERENT_BETWEEN_SCENARIOS = "Images are different between scenarios."; @@ -115,15 +126,17 @@ public TestDriver(ScreenshotNoInputAppState screenshotAppState, Collection= tickToTerminateApp){ - getApplication().stop(true); + + if (tick >= tickToTerminateApp) { waitLatch.countDown(); + getApplication().stop(false); + return; } tick++; @@ -142,7 +155,18 @@ public void update(float tpf){ @Override protected void onEnable(){} @Override protected void onDisable(){} - + + private static AppSettings applyPlatformSettings(AppSettings settings) { + String os = System.getProperty("os.name", "").toLowerCase(); + if (os.contains("mac")) { + settings.setSamples(0); + settings.setGammaCorrection(false); + settings.setStencilBits(0); + settings.setUseRetinaFrameBuffer(false); + } + return settings; + } + /** * Boots up the application on a separate thread (blocks this thread) and then does the following: * - Takes screenshots on the requested frames @@ -153,6 +177,12 @@ public static void bootAppForTest(TestType testType, AppSettings appSettings, St Collections.sort(framesToTakeScreenshotsOn); + // On macOS, GLFW requires all window operations to happen on the main OS thread. + // With -XstartOnFirstThread, the JVM main thread is the macOS main thread, and + // JUnit runs test methods on this thread, so we start the app directly (blocking) + // rather than on a background executor thread. + boolean isMacOs = System.getProperty("os.name", "").toLowerCase().contains("mac"); + List tempFolders = new ArrayList<>(); Map> imageFilesPerScenario = new HashMap<>(); @@ -177,20 +207,48 @@ public static void bootAppForTest(TestType testType, AppSettings appSettings, St states.add(testDriver); SimpleApplication app = new App(states.toArray(new AppState[0])); + appSettings = applyPlatformSettings(appSettings); app.setSettings(appSettings); app.setShowSettings(false); testDriver.waitLatch = new CountDownLatch(1); - executor.execute(() -> app.start(JmeContext.Type.Display)); - int maxWaitTimeMilliseconds = 45000; + // Pre-set onError so that context-creation failures (which occur before + // TestDriver.initialize() runs) also count down the latch. + // TestDriver.initialize() will overwrite this with the same behaviour once + // the app has successfully started. + final CountDownLatch capturedLatch = testDriver.waitLatch; + ((App)app).onError = error -> { + logger.log(Level.WARNING, "Error in test application (during startup)", error); + capturedLatch.countDown(); + }; try { - boolean exitedProperly = testDriver.waitLatch.await(maxWaitTimeMilliseconds, TimeUnit.MILLISECONDS); + if (isMacOs) { + // On macOS the app must run on the main thread (GLFW requirement). + // app.start() blocks until the app stops, so no latch wait is needed. + app.start(JmeContext.Type.Display); + } else { + executor.execute(() -> { + try { + app.start(JmeContext.Type.Display); + } catch (Throwable t) { + // Catch any unexpected exception during app start (e.g. UnsatisfiedLinkError + // if native libraries cannot be loaded) and unblock the waiting test thread. + logger.log(Level.WARNING, "Exception while starting test application", t); + capturedLatch.countDown(); + } + }); - if (!exitedProperly) { - logger.warning("Test driver did not exit in " + maxWaitTimeMilliseconds + "ms. Timed out"); - app.stop(true); + int maxWaitTimeMilliseconds = 45000; + boolean exitedProperly = testDriver.waitLatch.await(maxWaitTimeMilliseconds, TimeUnit.MILLISECONDS); + + if (!exitedProperly) { + logger.warning("Test driver did not exit in " + maxWaitTimeMilliseconds + "ms. Timed out"); + // Use non-blocking stop so this thread doesn't deadlock if the app + // thread is itself stuck (e.g. inside glfwTerminate with Mesa3D). + app.stop(false); + } } Thread.sleep(1000); //give time for openGL is fully released before starting a new test (get random JVM crashes without this) @@ -409,8 +467,9 @@ private static void attachImage(String title, String fileName, BufferedImage ori } /** - * Tests that the images are the same. If they are not the same it will return false (which may fail the test - * depending on the test type). Different sizes are so fatal that they will immediately fail the test. + * Tests that the images are the same (or within {@link #PIXEL_TOLERANCE}). If they are not + * the same it will return false (which may fail the test depending on the test type). + * Different sizes are so fatal that they will immediately fail the test. */ private static boolean imagesAreTheSame(BufferedImage img1, BufferedImage img2) { if (img1.getWidth() != img2.getWidth() || img1.getHeight() != img2.getHeight()) { @@ -421,8 +480,12 @@ private static boolean imagesAreTheSame(BufferedImage img1, BufferedImage img2) for (int y = 0; y < img1.getHeight(); y++) { for (int x = 0; x < img1.getWidth(); x++) { - if (img1.getRGB(x, y) != img2.getRGB(x, y)){ - return false; + int rgb1 = img1.getRGB(x, y); + int rgb2 = img2.getRGB(x, y); + if (rgb1 != rgb2) { + if (PIXEL_TOLERANCE == 0 || getMaximumComponentDifference(rgb1, rgb2) > PIXEL_TOLERANCE) { + return false; + } } } }