diff --git a/dd-java-agent/agent-crashtracking/src/main/resources/datadog/crashtracking/notify_oome.bat b/dd-java-agent/agent-crashtracking/src/main/resources/datadog/crashtracking/notify_oome.bat index b0d8b5bdbb4..9a1262da067 100644 --- a/dd-java-agent/agent-crashtracking/src/main/resources/datadog/crashtracking/notify_oome.bat +++ b/dd-java-agent/agent-crashtracking/src/main/resources/datadog/crashtracking/notify_oome.bat @@ -34,6 +34,12 @@ echo Tags: %tags% echo JAVA_HOME: %java_home% echo PID: %PID% +:: Clear environment variables that the parent JVM may have set so the child JVM +:: starts with a minimal configuration (avoids port conflicts, memory contention, etc.) +set JDK_JAVA_OPTIONS= +set JAVA_TOOL_OPTIONS= +set _JAVA_OPTIONS= + :: Execute the Java command with the loaded values "%java_home%\bin\java" -Ddd.dogstatsd.start-delay=0 -jar "%agent%" sendOomeEvent "%tags%" set RC=%ERRORLEVEL% diff --git a/dd-java-agent/agent-crashtracking/src/main/resources/datadog/crashtracking/notify_oome.sh b/dd-java-agent/agent-crashtracking/src/main/resources/datadog/crashtracking/notify_oome.sh index 5e75f6a7563..4623e157cb2 100644 --- a/dd-java-agent/agent-crashtracking/src/main/resources/datadog/crashtracking/notify_oome.sh +++ b/dd-java-agent/agent-crashtracking/src/main/resources/datadog/crashtracking/notify_oome.sh @@ -49,6 +49,12 @@ echo "Tags: $config_tags" echo "JAVA_HOME: $config_java_home" echo "PID: $PID" +# Clear environment variables that the parent JVM may have set so the child JVM +# starts with a minimal configuration (avoids port conflicts, memory contention, etc.) +unset JDK_JAVA_OPTIONS +unset JAVA_TOOL_OPTIONS +unset _JAVA_OPTIONS + # Execute the Java command with the loaded values "$config_java_home/bin/java" -Ddd.dogstatsd.start-delay=0 -jar "$config_agent" sendOomeEvent "$config_tags" RC=$? diff --git a/dd-java-agent/agent-crashtracking/src/main/resources/datadog/crashtracking/upload_crash.bat b/dd-java-agent/agent-crashtracking/src/main/resources/datadog/crashtracking/upload_crash.bat index 5b0e0b48eb4..7ee2064f734 100644 --- a/dd-java-agent/agent-crashtracking/src/main/resources/datadog/crashtracking/upload_crash.bat +++ b/dd-java-agent/agent-crashtracking/src/main/resources/datadog/crashtracking/upload_crash.bat @@ -4,6 +4,11 @@ setlocal enabledelayedexpansion :: Check if PID is provided if "%1"=="" ( echo "Error: No PID provided. Running in legacy mode." + :: Clear environment variables that the parent JVM may have set + set JDK_JAVA_OPTIONS= + set JAVA_TOOL_OPTIONS= + set _JAVA_OPTIONS= + "!JAVA_HOME!\bin\java" -jar "!AGENT_JAR!" uploadCrash "!JAVA_ERROR_FILE!" if %ERRORLEVEL% EQU 0 ( echo "Uploaded error file \"!JAVA_ERROR_FILE!\"" @@ -128,6 +133,12 @@ echo Error Log: %config_hs_err% echo JAVA_HOME: %config_java_home% echo PID: %PID% +:: Clear environment variables that the parent JVM may have set so the child JVM +:: starts with a minimal configuration (avoids port conflicts, memory contention, etc.) +set JDK_JAVA_OPTIONS= +set JAVA_TOOL_OPTIONS= +set _JAVA_OPTIONS= + :: Execute the Java command with the loaded values "%config_java_home%\bin\java" -jar "%config_agent%" uploadCrash -c "%configFile%" "%config_hs_err%" set RC=%ERRORLEVEL% diff --git a/dd-java-agent/agent-crashtracking/src/main/resources/datadog/crashtracking/upload_crash.sh b/dd-java-agent/agent-crashtracking/src/main/resources/datadog/crashtracking/upload_crash.sh index cb338aa9006..42b1fde286a 100644 --- a/dd-java-agent/agent-crashtracking/src/main/resources/datadog/crashtracking/upload_crash.sh +++ b/dd-java-agent/agent-crashtracking/src/main/resources/datadog/crashtracking/upload_crash.sh @@ -7,6 +7,11 @@ set +e if [ -z "$1" ]; then echo "Warn: No PID provided. Running in legacy mode." + # Clear environment variables that the parent JVM may have set + unset JDK_JAVA_OPTIONS + unset JAVA_TOOL_OPTIONS + unset _JAVA_OPTIONS + "!JAVA_HOME!/bin/java" -jar "!AGENT_JAR!" uploadCrash "!JAVA_ERROR_FILE!" if [ $? -eq 0 ]; then echo "Error file !JAVA_ERROR_FILE! was uploaded successfully" @@ -119,6 +124,12 @@ echo "Error Log: $config_hs_err" echo "JAVA_HOME: $config_java_home" echo "PID: $PID" +# Clear environment variables that the parent JVM may have set so the child JVM +# starts with a minimal configuration (avoids port conflicts, memory contention, etc.) +unset JDK_JAVA_OPTIONS +unset JAVA_TOOL_OPTIONS +unset _JAVA_OPTIONS + # Execute the Java command with the loaded values "$config_java_home/bin/java" -jar "$config_agent" uploadCrash -c "$configFile" "$config_hs_err" RC=$? diff --git a/dd-smoke-tests/crashtracking/src/test/java/datadog/smoketest/CrashtrackingSmokeTest.java b/dd-smoke-tests/crashtracking/src/test/java/datadog/smoketest/CrashtrackingSmokeTest.java index d832e5ef382..bbf554523e6 100644 --- a/dd-smoke-tests/crashtracking/src/test/java/datadog/smoketest/CrashtrackingSmokeTest.java +++ b/dd-smoke-tests/crashtracking/src/test/java/datadog/smoketest/CrashtrackingSmokeTest.java @@ -13,6 +13,7 @@ import datadog.environment.OperatingSystem; import java.io.File; import java.io.IOException; +import java.net.ServerSocket; import java.nio.charset.StandardCharsets; import java.nio.file.FileSystems; import java.nio.file.Files; @@ -319,6 +320,132 @@ void testCombineTracking() throws Exception { assertOOMEvent(); } + /** + * Verifies that the OOME notifier script correctly unsets inherited JVM environment variables. + * Without the fix, the child JVM spawned by the script would inherit JDK_JAVA_OPTIONS containing + * JMX port-binding flags, causing a BindException and losing the OOME event. + * + * @see #10766 + */ + @Test + void testOomeTrackingWithInheritedEnvVars() throws Exception { + int jmxPort = findFreePort(); + + Path script = tempDir.resolve("dd_oome_notifier." + getExtension()); + String onErrorValue = script + " %p"; + String errorFile = tempDir.resolve("hs_err_pid%p.log").toString(); + + String onOOMEArg = + !Platform.isLinux() + ? "-XX:OnOutOfMemoryError=" + onErrorValue + : "-Ddd.crashtracking.debug.autoconfig.enable=true"; + + List processArgs = new ArrayList<>(); + processArgs.add(javaPath()); + processArgs.add("-javaagent:" + agentShadowJar()); + processArgs.add("-Xmx96m"); + processArgs.add("-Xms96m"); + if (!onOOMEArg.isEmpty()) { + processArgs.add(onOOMEArg); + } + processArgs.add("-XX:ErrorFile=" + errorFile); + processArgs.add("-XX:+CrashOnOutOfMemoryError"); + processArgs.add("-Ddd.dogstatsd.start-delay=0"); + processArgs.add("-Ddd.trace.enabled=false"); + processArgs.add("-jar"); + processArgs.add(appShadowJar()); + + ProcessBuilder pb = new ProcessBuilder(processArgs); + pb.environment().put("DD_DOGSTATSD_PORT", String.valueOf(udpServer.getPort())); + // Simulate admission controller injecting JMX flags via JDK_JAVA_OPTIONS + pb.environment() + .put( + "JDK_JAVA_OPTIONS", + "-Dcom.sun.management.jmxremote" + + " -Dcom.sun.management.jmxremote.port=" + + jmxPort + + " -Dcom.sun.management.jmxremote.rmi.port=" + + jmxPort + + " -Dcom.sun.management.jmxremote.authenticate=false" + + " -Dcom.sun.management.jmxremote.ssl=false"); + + System.out.println("==> Process args: " + pb.command()); + System.out.println("==> JMX port: " + jmxPort); + + Process p = pb.start(); + OUTPUT.captureOutput( + p, LOG_FILE_DIR.resolve("testProcess.testOomeTrackingWithInheritedEnvVars.log").toFile()); + + assertExpectedCrash(p); + assertOOMEvent(); + } + + /** + * Verifies that the crash uploader script correctly unsets inherited JVM environment variables. + * Without the fix, the child JVM spawned by the script would inherit JDK_JAVA_OPTIONS containing + * JMX port-binding flags, causing a BindException and losing the crash data. + * + * @see #10766 + */ + @Test + void testCrashTrackingWithInheritedEnvVars() throws Exception { + int jmxPort = findFreePort(); + + Path script = tempDir.resolve("dd_crash_uploader." + getExtension()); + String onErrorValue = script + " %p"; + String errorFile = tempDir.resolve("hs_err.log").toString(); + + String onErrorArg = + !Platform.isLinux() + ? "-XX:OnError=" + onErrorValue + : "-Ddd.crashtracking.debug.autoconfig.enable=true"; + + List processArgs = new ArrayList<>(); + processArgs.add(javaPath()); + processArgs.add("-javaagent:" + agentShadowJar()); + processArgs.add("-Xmx96m"); + processArgs.add("-Xms96m"); + if (!onErrorArg.isEmpty()) { + processArgs.add(onErrorArg); + } + processArgs.add("-XX:ErrorFile=" + errorFile); + processArgs.add("-XX:+CrashOnOutOfMemoryError"); + processArgs.add("-Ddd.dogstatsd.start-delay=0"); + processArgs.add("-Ddd.trace.enabled=false"); + processArgs.add("-jar"); + processArgs.add(appShadowJar()); + + ProcessBuilder pb = new ProcessBuilder(processArgs); + pb.environment().put("DD_TRACE_AGENT_PORT", String.valueOf(tracingServer.getPort())); + // Simulate admission controller injecting JMX flags via JDK_JAVA_OPTIONS + pb.environment() + .put( + "JDK_JAVA_OPTIONS", + "-Dcom.sun.management.jmxremote" + + " -Dcom.sun.management.jmxremote.port=" + + jmxPort + + " -Dcom.sun.management.jmxremote.rmi.port=" + + jmxPort + + " -Dcom.sun.management.jmxremote.authenticate=false" + + " -Dcom.sun.management.jmxremote.ssl=false"); + + System.out.println("==> Process args: " + pb.command()); + System.out.println("==> JMX port: " + jmxPort); + + Process p = pb.start(); + OUTPUT.captureOutput( + p, LOG_FILE_DIR.resolve("testProcess.testCrashTrackingWithInheritedEnvVars.log").toFile()); + + assertExpectedCrash(p); + assertCrashData(assertCrashPing()); + } + + private static int findFreePort() throws IOException { + try (ServerSocket socket = new ServerSocket(0)) { + return socket.getLocalPort(); + } + } + private static void assertExpectedCrash(Process p) throws InterruptedException { // exit code -1 means the test application exited prematurely // exit code > 0 means the test application crashed, as expected