From 4bab59fd03821d7549bb3dca7a15ae07abc96caf Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Mon, 3 Aug 2026 17:56:34 +0200 Subject: [PATCH 1/6] Add experimental config for ddprof native memory profiling Wires dd.profiling.experimental.ddprof.nativemem.{enabled,interval} to the ddprof nativemem= start argument, gated behind a new NATIVEMEM ProfilingMode. Co-Authored-By: Claude Sonnet 5 --- .../profiling/ddprof/DatadogProfiler.java | 10 ++++++ .../ddprof/DatadogProfilerConfig.java | 26 ++++++++++++++ .../profiling/ddprof/DatadogProfilerTest.java | 35 ++++++++++++++++--- .../profiling/utils/ProfilingMode.java | 3 +- .../trace/api/config/ProfilingConfig.java | 7 ++++ 5 files changed, 76 insertions(+), 5 deletions(-) diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java index df1b75789db..223c96822f0 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java @@ -7,6 +7,7 @@ import static com.datadog.profiling.ddprof.DatadogProfilerConfig.getCpuInterval; import static com.datadog.profiling.ddprof.DatadogProfilerConfig.getLiveHeapSamplePercent; import static com.datadog.profiling.ddprof.DatadogProfilerConfig.getLogLevel; +import static com.datadog.profiling.ddprof.DatadogProfilerConfig.getNativeMemoryInterval; import static com.datadog.profiling.ddprof.DatadogProfilerConfig.getSafeMode; import static com.datadog.profiling.ddprof.DatadogProfilerConfig.getSchedulingEvent; import static com.datadog.profiling.ddprof.DatadogProfilerConfig.getSchedulingEventInterval; @@ -18,6 +19,7 @@ import static com.datadog.profiling.ddprof.DatadogProfilerConfig.isCpuProfilerEnabled; import static com.datadog.profiling.ddprof.DatadogProfilerConfig.isLiveHeapSizeTrackingEnabled; import static com.datadog.profiling.ddprof.DatadogProfilerConfig.isMemoryLeakProfilingEnabled; +import static com.datadog.profiling.ddprof.DatadogProfilerConfig.isNativeMemoryProfilingEnabled; import static com.datadog.profiling.ddprof.DatadogProfilerConfig.isResourceNameContextAttributeEnabled; import static com.datadog.profiling.ddprof.DatadogProfilerConfig.isSpanNameContextAttributeEnabled; import static com.datadog.profiling.ddprof.DatadogProfilerConfig.isTrackingGenerations; @@ -26,6 +28,7 @@ import static com.datadog.profiling.utils.ProfilingMode.ALLOCATION; import static com.datadog.profiling.utils.ProfilingMode.CPU; import static com.datadog.profiling.utils.ProfilingMode.MEMLEAK; +import static com.datadog.profiling.utils.ProfilingMode.NATIVEMEM; import static com.datadog.profiling.utils.ProfilingMode.WALL; import static datadog.trace.api.config.ProfilingConfig.PROFILING_DETAILED_DEBUG_LOGGING; import static datadog.trace.api.config.ProfilingConfig.PROFILING_DETAILED_DEBUG_LOGGING_DEFAULT; @@ -267,6 +270,9 @@ private DatadogProfiler(ConfigProvider configProvider) { if (isWallClockProfilerEnabled(configProvider)) { profilingModes.add(WALL); } + if (isNativeMemoryProfilingEnabled(configProvider)) { + profilingModes.add(NATIVEMEM); + } Set contextAttributes = getContextAttributes(configProvider); this.orderedContextAttributes = getOrderedContextAttributes(contextAttributes, configProvider); this.contextSetter = new ContextSetter(profiler, orderedContextAttributes); @@ -488,6 +494,10 @@ String cmdStartProfiling(Path file) throws IllegalStateException { .append(String.format("%.2f", getLiveHeapSamplePercent(configProvider) / 100.0d)); } } + if (profilingModes.contains(NATIVEMEM)) { + // native memory (malloc) profiling is enabled + cmd.append(",nativemem=").append(getNativeMemoryInterval(configProvider)); + } String cmdString = cmd.toString(); log.debug("Datadog profiler command line: {}", cmdString); return cmdString; diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilerConfig.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilerConfig.java index 8582e27c853..88fb0c023f7 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilerConfig.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilerConfig.java @@ -33,6 +33,10 @@ import static datadog.trace.api.config.ProfilingConfig.PROFILING_DATADOG_PROFILER_MEMLEAK_CAPACITY; import static datadog.trace.api.config.ProfilingConfig.PROFILING_DATADOG_PROFILER_MEMLEAK_ENABLED; import static datadog.trace.api.config.ProfilingConfig.PROFILING_DATADOG_PROFILER_MEMLEAK_INTERVAL; +import static datadog.trace.api.config.ProfilingConfig.PROFILING_DATADOG_PROFILER_NATIVEMEM_ENABLED; +import static datadog.trace.api.config.ProfilingConfig.PROFILING_DATADOG_PROFILER_NATIVEMEM_ENABLED_DEFAULT; +import static datadog.trace.api.config.ProfilingConfig.PROFILING_DATADOG_PROFILER_NATIVEMEM_INTERVAL; +import static datadog.trace.api.config.ProfilingConfig.PROFILING_DATADOG_PROFILER_NATIVEMEM_INTERVAL_DEFAULT; import static datadog.trace.api.config.ProfilingConfig.PROFILING_DATADOG_PROFILER_SAFEMODE; import static datadog.trace.api.config.ProfilingConfig.PROFILING_DATADOG_PROFILER_SAFEMODE_DEFAULT; import static datadog.trace.api.config.ProfilingConfig.PROFILING_DATADOG_PROFILER_SCHEDULING_EVENT; @@ -290,6 +294,28 @@ public static int getMemleakCapacity() { return getMemleakCapacity(ConfigProvider.getInstance()); } + public static boolean isNativeMemoryProfilingEnabled(ConfigProvider configProvider) { + return getBoolean( + configProvider, + PROFILING_DATADOG_PROFILER_NATIVEMEM_ENABLED, + PROFILING_DATADOG_PROFILER_NATIVEMEM_ENABLED_DEFAULT); + } + + public static boolean isNativeMemoryProfilingEnabled() { + return isNativeMemoryProfilingEnabled(ConfigProvider.getInstance()); + } + + public static int getNativeMemoryInterval(ConfigProvider configProvider) { + return getInteger( + configProvider, + PROFILING_DATADOG_PROFILER_NATIVEMEM_INTERVAL, + PROFILING_DATADOG_PROFILER_NATIVEMEM_INTERVAL_DEFAULT); + } + + public static int getNativeMemoryInterval() { + return getNativeMemoryInterval(ConfigProvider.getInstance()); + } + public static int getStackDepth(ConfigProvider configProvider) { return getInteger( configProvider, diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerTest.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerTest.java index 6dcb3e9d231..482017f5ac2 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerTest.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerTest.java @@ -118,11 +118,38 @@ void testStartCmdEnableJMethodIDOptim() throws Exception { DatadogProfiler profiler = DatadogProfiler.newInstance(ConfigProvider.withPropertiesOverride(props)); - Path dir = Paths.get("/tmp"); - Path targetFile = Files.createTempFile(dir, "target_", ".jfr"); - String cmd = profiler.cmdStartProfiling(targetFile); + assertTrue(startCmd(profiler).contains(",fjmethodid=false")); + } + + @Test + void testStartCmdNativeMemDisabledByDefault() throws Exception { + assertDoesNotThrow( + () -> DdprofLibraryLoader.jvmAccess().getReasonNotLoaded(), "Profiler not available"); + + DatadogProfiler profiler = DatadogProfiler.newInstance(ConfigProvider.getInstance()); + assertFalse(profiler.enabledModes().contains(ProfilingMode.NATIVEMEM)); + + assertFalse(startCmd(profiler).contains(",nativemem=")); + } + + @Test + void testStartCmdNativeMemEnabled() throws Exception { + assertDoesNotThrow( + () -> DdprofLibraryLoader.jvmAccess().getReasonNotLoaded(), "Profiler not available"); + + Properties props = new Properties(); + props.put(ProfilingConfig.PROFILING_DATADOG_PROFILER_NATIVEMEM_ENABLED, "true"); + props.put(ProfilingConfig.PROFILING_DATADOG_PROFILER_NATIVEMEM_INTERVAL, "131072"); + DatadogProfiler profiler = + DatadogProfiler.newInstance(ConfigProvider.withPropertiesOverride(props)); + assertTrue(profiler.enabledModes().contains(ProfilingMode.NATIVEMEM)); + + assertTrue(startCmd(profiler).contains(",nativemem=131072")); + } - assertTrue(cmd.contains(",fjmethodid=false"), cmd); + private static String startCmd(DatadogProfiler profiler) throws Exception { + Path targetFile = Files.createTempFile(Paths.get("/tmp"), "target_", ".jfr"); + return profiler.cmdStartProfiling(targetFile); } @ParameterizedTest diff --git a/dd-java-agent/agent-profiling/profiling-utils/src/main/java/com/datadog/profiling/utils/ProfilingMode.java b/dd-java-agent/agent-profiling/profiling-utils/src/main/java/com/datadog/profiling/utils/ProfilingMode.java index 9697edd74a1..e313c064445 100644 --- a/dd-java-agent/agent-profiling/profiling-utils/src/main/java/com/datadog/profiling/utils/ProfilingMode.java +++ b/dd-java-agent/agent-profiling/profiling-utils/src/main/java/com/datadog/profiling/utils/ProfilingMode.java @@ -7,7 +7,8 @@ public enum ProfilingMode { CPU(1 << 0), WALL(1 << 1), ALLOCATION(1 << 2), - MEMLEAK(1 << 3); + MEMLEAK(1 << 3), + NATIVEMEM(1 << 4); public final int bitmask; diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/ProfilingConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/ProfilingConfig.java index 6d60ab686a7..bf5f7a552e9 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/ProfilingConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/ProfilingConfig.java @@ -180,6 +180,13 @@ public final class ProfilingConfig { public static final int PROFILING_DATADOG_PROFILER_LIVEHEAP_SAMPLE_PERCENT_DEFAULT = 50; // default to 10% of allocation samples + public static final String PROFILING_DATADOG_PROFILER_NATIVEMEM_ENABLED = + "profiling.experimental.ddprof.nativemem.enabled"; + public static final boolean PROFILING_DATADOG_PROFILER_NATIVEMEM_ENABLED_DEFAULT = false; + public static final String PROFILING_DATADOG_PROFILER_NATIVEMEM_INTERVAL = + "profiling.experimental.ddprof.nativemem.interval"; + public static final int PROFILING_DATADOG_PROFILER_NATIVEMEM_INTERVAL_DEFAULT = 512 * 1024; + public static final String PROFILING_ENDPOINT_COLLECTION_ENABLED = "profiling.endpoint.collection.enabled"; public static final boolean PROFILING_ENDPOINT_COLLECTION_ENABLED_DEFAULT = true; From 395f9f3f6c932285e6370fc2fb0e059d1784bec2 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Mon, 3 Aug 2026 18:10:06 +0200 Subject: [PATCH 2/6] Register nativemem config keys in supported-configurations.json --- metadata/supported-configurations.json | 32 ++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index d6b02fbfa28..1111765ed68 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -3153,6 +3153,22 @@ "aliases": [] } ], + "DD_PROFILING_EXPERIMENTAL_ASYNC_NATIVEMEM_ENABLED": [ + { + "version": "A", + "type": "boolean", + "default": "false", + "aliases": [] + } + ], + "DD_PROFILING_EXPERIMENTAL_ASYNC_NATIVEMEM_INTERVAL": [ + { + "version": "A", + "type": "int", + "default": "524288", + "aliases": [] + } + ], "DD_PROFILING_EXPERIMENTAL_ASYNC_SCHEDULING_EVENT": [ { "version": "A", @@ -3185,6 +3201,22 @@ "aliases": [] } ], + "DD_PROFILING_EXPERIMENTAL_DDPROF_NATIVEMEM_ENABLED": [ + { + "version": "A", + "type": "boolean", + "default": "false", + "aliases": [] + } + ], + "DD_PROFILING_EXPERIMENTAL_DDPROF_NATIVEMEM_INTERVAL": [ + { + "version": "A", + "type": "int", + "default": "524288", + "aliases": [] + } + ], "DD_PROFILING_EXPERIMENTAL_DDPROF_SCHEDULING_EVENT": [ { "version": "A", From b78d2af68161d4fb4d60d0f1e54c023762959a08 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Tue, 4 Aug 2026 13:18:56 +0200 Subject: [PATCH 3/6] Ignore JAVA_TOOL_OPTIONS stderr noise in verifyAgentJarIntegrations CI proxy config sets JAVA_TOOL_OPTIONS, which the JVM echoes to stderr on every launch, causing a false-positive module-load-failure error. Co-Authored-By: Claude Sonnet 5 --- dd-java-agent/build.gradle | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dd-java-agent/build.gradle b/dd-java-agent/build.gradle index 66917378481..32d3a0e5756 100644 --- a/dd-java-agent/build.gradle +++ b/dd-java-agent/build.gradle @@ -634,7 +634,11 @@ tasks.register('verifyAgentJarIntegrations', JavaExec) { errorOutput = capturedError doLast { - def stderr = capturedError.toString() + // The JVM prints this diagnostic line to stderr whenever JAVA_TOOL_OPTIONS is set in the + // environment (e.g. CI proxy configuration) — it is not a module load failure. + def stderr = capturedError.toString().readLines() + .findAll { !it.startsWith('Picked up JAVA_TOOL_OPTIONS:') } + .join('\n') if (!stderr.isBlank()) { throw new GradleException( "--list-integrations produced unexpected stderr output " + From 3f9700e7b46aff38de84f1a63d52895175e82022 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Tue, 4 Aug 2026 14:49:13 +0200 Subject: [PATCH 4/6] Spotless!!! --- metadata/supported-configurations.json | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index b00d8b94249..64f9b563655 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -3201,22 +3201,6 @@ "aliases": [] } ], - "DD_PROFILING_EXPERIMENTAL_ASYNC_NATIVEMEM_ENABLED": [ - { - "version": "A", - "type": "boolean", - "default": "false", - "aliases": [] - } - ], - "DD_PROFILING_EXPERIMENTAL_ASYNC_NATIVEMEM_INTERVAL": [ - { - "version": "A", - "type": "int", - "default": "524288", - "aliases": [] - } - ], "DD_PROFILING_EXPERIMENTAL_ASYNC_SCHEDULING_EVENT": [ { "version": "A", From 0105ac2516865f69a49076fbff31f1689a5ea4ca Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Tue, 4 Aug 2026 15:05:17 +0200 Subject: [PATCH 5/6] Spotless!!! --- .../profiling/ddprof/DatadogProfilerConfig.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilerConfig.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilerConfig.java index 88fb0c023f7..c95db897fb3 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilerConfig.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilerConfig.java @@ -295,8 +295,9 @@ public static int getMemleakCapacity() { } public static boolean isNativeMemoryProfilingEnabled(ConfigProvider configProvider) { - return getBoolean( - configProvider, + // nativemem never shipped under the legacy '.async.' naming, so it has no async-translated + // form to fall back to — call the provider directly instead of the ddprof->async helper. + return configProvider.getBoolean( PROFILING_DATADOG_PROFILER_NATIVEMEM_ENABLED, PROFILING_DATADOG_PROFILER_NATIVEMEM_ENABLED_DEFAULT); } @@ -306,8 +307,9 @@ public static boolean isNativeMemoryProfilingEnabled() { } public static int getNativeMemoryInterval(ConfigProvider configProvider) { - return getInteger( - configProvider, + // nativemem never shipped under the legacy '.async.' naming, so it has no async-translated + // form to fall back to — call the provider directly instead of the ddprof->async helper. + return configProvider.getInteger( PROFILING_DATADOG_PROFILER_NATIVEMEM_INTERVAL, PROFILING_DATADOG_PROFILER_NATIVEMEM_INTERVAL_DEFAULT); } From ae2b23513e674f0aea0eab4a27762eedb1064e19 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Thu, 6 Aug 2026 17:23:16 +0200 Subject: [PATCH 6/6] Try remove JAVA_TOOL_OPTIONS --- dd-java-agent/build.gradle | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/dd-java-agent/build.gradle b/dd-java-agent/build.gradle index 32d3a0e5756..66917378481 100644 --- a/dd-java-agent/build.gradle +++ b/dd-java-agent/build.gradle @@ -634,11 +634,7 @@ tasks.register('verifyAgentJarIntegrations', JavaExec) { errorOutput = capturedError doLast { - // The JVM prints this diagnostic line to stderr whenever JAVA_TOOL_OPTIONS is set in the - // environment (e.g. CI proxy configuration) — it is not a module load failure. - def stderr = capturedError.toString().readLines() - .findAll { !it.startsWith('Picked up JAVA_TOOL_OPTIONS:') } - .join('\n') + def stderr = capturedError.toString() if (!stderr.isBlank()) { throw new GradleException( "--list-integrations produced unexpected stderr output " +