From cec2fd98b751d5b37bd681940eefb3bba259160f Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 1 Apr 2026 12:39:27 +0200 Subject: [PATCH 1/3] feat: normalize benchmark mode to AverageTime under CodSpeed When CodSpeed instrumentation is active, all benchmark modes (Throughput, SampleTime, SingleShotTime, Mode.All, and CLI -bm overrides) are now normalized to AverageTime. This ensures consistent result shapes that match how criterion (codspeed-rust) collects walltime data: each round has a variable iteration count and a measured total wall time. Also simplifies CodSpeedResultCollector to only handle AverageTime, removing the SampleTime histogram expansion and Throughput back- calculation paths. Resolves COD-2446 --- .../result/CodSpeedResultCollector.java | 84 +++---------------- .../java/org/openjdk/jmh/runner/Runner.java | 35 ++++---- 2 files changed, 27 insertions(+), 92 deletions(-) diff --git a/jmh-fork/jmh-core/src/main/java/io/codspeed/result/CodSpeedResultCollector.java b/jmh-fork/jmh-core/src/main/java/io/codspeed/result/CodSpeedResultCollector.java index 3f9dc17..fb790f6 100644 --- a/jmh-fork/jmh-core/src/main/java/io/codspeed/result/CodSpeedResultCollector.java +++ b/jmh-fork/jmh-core/src/main/java/io/codspeed/result/CodSpeedResultCollector.java @@ -7,16 +7,12 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; -import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.concurrent.TimeUnit; -import org.openjdk.jmh.annotations.Mode; import org.openjdk.jmh.infra.BenchmarkParams; import org.openjdk.jmh.results.BenchmarkResult; import org.openjdk.jmh.results.IterationResult; import org.openjdk.jmh.results.RunResult; -import org.openjdk.jmh.util.Statistics; public class CodSpeedResultCollector { @@ -31,17 +27,12 @@ public static void collectAndWrite( for (RunResult runResult : runResults) { BenchmarkParams params = runResult.getParams(); - Mode mode = params.getMode(); TimeUnit timeUnit = params.getTimeUnit(); String benchmarkName = params.getBenchmark(); String uri = BenchmarkUri.fromBenchmarkParams(params); double nanosPerUnit = TimeUnit.NANOSECONDS.convert(1, timeUnit); - if (mode == Mode.SampleTime) { - collectSampleTime(runResult, benchmarkName, uri, nanosPerUnit, benchmarks); - } else { - collectIterationBased(runResult, benchmarkName, uri, mode, nanosPerUnit, benchmarks); - } + collectAverageTime(runResult, benchmarkName, uri, nanosPerUnit, benchmarks); } if (benchmarks.isEmpty()) { @@ -59,21 +50,19 @@ public static void collectAndWrite( } /** - * Collects results from iteration-based modes (Throughput, AverageTime, SingleShotTime). + * Collects results from AverageTime mode. * - *

In these modes, JMH runs a fixed time window per iteration and reports an aggregated score. - * Each iteration produces one data point: the total wall time for all ops in that iteration. We - * back-calculate the total wall time from the score (see inline comments for the formulas per - * mode). + *

JMH runs a fixed time window per iteration and reports the average time per operation. Each + * iteration produces one data point: the total wall time for all ops in that iteration, which we + * back-calculate from the score: {@code timeNs = score * ops * nanosPerUnit}. * - *

Compare with {@link #collectSampleTime}, where JMH records per-operation latencies in a - * histogram, giving finer-grained data. + *

This is structurally equivalent to how criterion (codspeed-rust) collects walltime data: + * each round has a variable iteration count and a measured total wall time. */ - private static void collectIterationBased( + private static void collectAverageTime( RunResult runResult, String benchmarkName, String uri, - Mode mode, double nanosPerUnit, List benchmarks) { List itersPerRound = new ArrayList<>(); @@ -84,14 +73,8 @@ private static void collectIterationBased( long ops = ir.getMetadata().getMeasuredOps(); double score = ir.getPrimaryResult().getScore(); - long timeNs; - if (mode == Mode.Throughput) { - // score = ops * nanosPerUnit / durationNs - timeNs = Math.round(ops * nanosPerUnit / score); - } else { - // avgt/ss: score = durationNs / (ops * nanosPerUnit) - timeNs = Math.round(score * ops * nanosPerUnit); - } + // avgt: score = durationNs / (ops * nanosPerUnit) + long timeNs = Math.round(score * ops * nanosPerUnit); itersPerRound.add(ops); timesPerRoundNs.add(timeNs); @@ -107,53 +90,6 @@ private static void collectIterationBased( benchmarkName, uri, toLongArray(itersPerRound), toLongArray(timesPerRoundNs), null)); } - /** - * Collects results from SampleTime mode. - * - *

In this mode, JMH measures each individual operation's latency separately and records them - * in a histogram. Each sample is a direct timing of one op, giving finer-grained data than - * iteration-based modes. We treat each sample as its own round with 1 iteration. - */ - private static void collectSampleTime( - RunResult runResult, - String benchmarkName, - String uri, - double nanosPerUnit, - List benchmarks) { - List itersPerRound = new ArrayList<>(); - List timesPerRoundNs = new ArrayList<>(); - - for (BenchmarkResult br : runResult.getBenchmarkResults()) { - for (IterationResult ir : br.getIterationResults()) { - Statistics stats = ir.getPrimaryResult().getStatistics(); - Iterator> rawData = stats.getRawData(); - - while (rawData.hasNext()) { - Map.Entry entry = rawData.next(); - double valueInUnit = entry.getKey(); - long count = entry.getValue(); - - // Each sample is one op's time in the output unit. - // Convert back to nanoseconds. - long timeNs = Math.round(valueInUnit * nanosPerUnit); - - for (long i = 0; i < count; i++) { - itersPerRound.add(1L); - timesPerRoundNs.add(timeNs); - } - } - } - } - - if (itersPerRound.isEmpty()) { - return; - } - - benchmarks.add( - WalltimeBenchmark.fromRuntimeData( - benchmarkName, uri, toLongArray(itersPerRound), toLongArray(timesPerRoundNs), null)); - } - private static long[] toLongArray(List list) { long[] arr = new long[list.size()]; for (int i = 0; i < list.size(); i++) { diff --git a/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java b/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java index 59e6ab0..3816010 100644 --- a/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java +++ b/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java @@ -266,13 +266,12 @@ private Collection internalRun() throws RunnerException { if (!options.getBenchModes().isEmpty()) { Collection benchModes = options.getBenchModes(); - // CodSpeed: warn and pick first mode if multiple are specified via CLI (-bm flag) - if (InstrumentHooks.getInstance().isInstrumented() && benchModes.size() > 1) { - Mode firstMode = benchModes.iterator().next(); - out.println("WARNING: CodSpeed does not support multiple benchmark modes. " + - "The -bm flag specifies " + benchModes.size() + " modes: " + benchModes + ". " + - "Using only the first mode: " + firstMode); - benchModes = Collections.singleton(firstMode); + // CodSpeed: normalize CLI-specified modes to AverageTime + if (InstrumentHooks.getInstance().isInstrumented()) { + if (!benchModes.equals(Collections.singleton(Mode.AverageTime))) { + out.println("# CodSpeed: normalizing benchmark mode to AverageTime (was: " + benchModes + ")."); + } + benchModes = Collections.singleton(Mode.AverageTime); } List newBenchmarks = new ArrayList<>(); @@ -292,24 +291,24 @@ private Collection internalRun() throws RunnerException { boolean isInstrumented = InstrumentHooks.getInstance().isInstrumented(); List newBenchmarks = new ArrayList<>(); for (BenchmarkListEntry br : benchmarks) { - if (br.getMode() != Mode.All) { - newBenchmarks.add(br); + if (isInstrumented) { + // CodSpeed: normalize any mode to AverageTime + if (br.getMode() != Mode.AverageTime) { + out.println("# CodSpeed: normalizing benchmark mode to AverageTime " + + "(was: " + br.getMode() + ") for " + br.getUsername() + "."); + } + newBenchmarks.add(br.cloneWith(Mode.AverageTime)); continue; } - if (isInstrumented) { - // CodSpeed: Mode.All would expand into multiple modes per benchmark, - // which we don't support. Pick a single mode and warn. - Mode firstMode = Mode.SampleTime; - out.println("WARNING: CodSpeed does not support multiple benchmark modes. " + - br.getUsername() + " uses Mode.All, using " + firstMode + " instead."); - newBenchmarks.add(br.cloneWith(firstMode)); - } else { - // Standard JMH: expand Mode.All into all concrete modes + // Standard JMH: expand Mode.All into all concrete modes + if (br.getMode() == Mode.All) { for (Mode m : Mode.values()) { if (m == Mode.All) continue; newBenchmarks.add(br.cloneWith(m)); } + } else { + newBenchmarks.add(br); } } From ba36d4cd090dbe055177e1ad9e85cd90dae2eab4 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 1 Apr 2026 12:39:36 +0200 Subject: [PATCH 2/3] fix: match codspeed-rust STDEV_OUTLIER_FACTOR (3.0) --- .../src/main/java/io/codspeed/result/WalltimeBenchmark.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jmh-fork/jmh-core/src/main/java/io/codspeed/result/WalltimeBenchmark.java b/jmh-fork/jmh-core/src/main/java/io/codspeed/result/WalltimeBenchmark.java index 875242f..d281c32 100644 --- a/jmh-fork/jmh-core/src/main/java/io/codspeed/result/WalltimeBenchmark.java +++ b/jmh-fork/jmh-core/src/main/java/io/codspeed/result/WalltimeBenchmark.java @@ -5,7 +5,7 @@ public class WalltimeBenchmark { private static final double IQR_OUTLIER_FACTOR = 1.5; - private static final double STDEV_OUTLIER_FACTOR = 2.0; + private static final double STDEV_OUTLIER_FACTOR = 3.0; private final String name; private final String uri; From 8cb09f5217f753158c1441031aa2c465e13581ee Mon Sep 17 00:00:00 2001 From: not-matthias Date: Tue, 7 Apr 2026 12:22:36 +0200 Subject: [PATCH 3/3] feat(jmh-fork): add CodSpeed mode to capture raw ops and duration per iteration Add Mode.CodSpeed, an internal benchmark mode that runs identically to AverageTime but preserves the raw (ops, durationNs) pair per iteration in a new CodSpeedResult class. This eliminates the lossy floating-point roundtrip that the previous approach required (score = time/ops, then back-calculate time = score * ops). When CodSpeed instrumentation is detected, the Runner automatically overrides all declared benchmark modes to CodSpeed and deduplicates by benchmark name so multi-mode declarations run exactly once. CodSpeedResultCollector is simplified to only handle Mode.CodSpeed, reading raw values directly from CodSpeedResult with no back-calculation. --- .../result/CodSpeedResultCollector.java | 67 ++++------ .../org/openjdk/jmh/annotations/Mode.java | 14 +++ .../generators/core/BenchmarkGenerator.java | 23 +++- .../openjdk/jmh/results/CodSpeedResult.java | 114 ++++++++++++++++++ .../java/org/openjdk/jmh/runner/Runner.java | 28 +++-- 5 files changed, 184 insertions(+), 62 deletions(-) create mode 100644 jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/results/CodSpeedResult.java diff --git a/jmh-fork/jmh-core/src/main/java/io/codspeed/result/CodSpeedResultCollector.java b/jmh-fork/jmh-core/src/main/java/io/codspeed/result/CodSpeedResultCollector.java index fb790f6..3f5ab12 100644 --- a/jmh-fork/jmh-core/src/main/java/io/codspeed/result/CodSpeedResultCollector.java +++ b/jmh-fork/jmh-core/src/main/java/io/codspeed/result/CodSpeedResultCollector.java @@ -8,9 +8,10 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Mode; import org.openjdk.jmh.infra.BenchmarkParams; import org.openjdk.jmh.results.BenchmarkResult; +import org.openjdk.jmh.results.CodSpeedResult; import org.openjdk.jmh.results.IterationResult; import org.openjdk.jmh.results.RunResult; @@ -27,12 +28,29 @@ public static void collectAndWrite( for (RunResult runResult : runResults) { BenchmarkParams params = runResult.getParams(); - TimeUnit timeUnit = params.getTimeUnit(); + Mode mode = params.getMode(); String benchmarkName = params.getBenchmark(); String uri = BenchmarkUri.fromBenchmarkParams(params); - double nanosPerUnit = TimeUnit.NANOSECONDS.convert(1, timeUnit); - collectAverageTime(runResult, benchmarkName, uri, nanosPerUnit, benchmarks); + if (mode != Mode.CodSpeed) { + throw new IllegalStateException( + "CodSpeedResultCollector only supports Mode.CodSpeed, got: " + mode); + } + + List itersPerRound = new ArrayList<>(); + List timesPerRoundNs = new ArrayList<>(); + + for (BenchmarkResult br : runResult.getBenchmarkResults()) { + for (IterationResult ir : br.getIterationResults()) { + CodSpeedResult result = (CodSpeedResult) ir.getPrimaryResult(); + itersPerRound.add(result.getRawOps()); + timesPerRoundNs.add(result.getRawDurationNs()); + } + } + + benchmarks.add( + WalltimeBenchmark.fromRuntimeData( + benchmarkName, uri, toLongArray(itersPerRound), toLongArray(timesPerRoundNs), null)); } if (benchmarks.isEmpty()) { @@ -49,47 +67,6 @@ public static void collectAndWrite( Files.writeString(outputFile, gson.toJson(results)); } - /** - * Collects results from AverageTime mode. - * - *

JMH runs a fixed time window per iteration and reports the average time per operation. Each - * iteration produces one data point: the total wall time for all ops in that iteration, which we - * back-calculate from the score: {@code timeNs = score * ops * nanosPerUnit}. - * - *

This is structurally equivalent to how criterion (codspeed-rust) collects walltime data: - * each round has a variable iteration count and a measured total wall time. - */ - private static void collectAverageTime( - RunResult runResult, - String benchmarkName, - String uri, - double nanosPerUnit, - List benchmarks) { - List itersPerRound = new ArrayList<>(); - List timesPerRoundNs = new ArrayList<>(); - - for (BenchmarkResult br : runResult.getBenchmarkResults()) { - for (IterationResult ir : br.getIterationResults()) { - long ops = ir.getMetadata().getMeasuredOps(); - double score = ir.getPrimaryResult().getScore(); - - // avgt: score = durationNs / (ops * nanosPerUnit) - long timeNs = Math.round(score * ops * nanosPerUnit); - - itersPerRound.add(ops); - timesPerRoundNs.add(timeNs); - } - } - - if (itersPerRound.isEmpty()) { - return; - } - - benchmarks.add( - WalltimeBenchmark.fromRuntimeData( - benchmarkName, uri, toLongArray(itersPerRound), toLongArray(timesPerRoundNs), null)); - } - private static long[] toLongArray(List list) { long[] arr = new long[list.size()]; for (int i = 0; i < list.size(); i++) { diff --git a/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/annotations/Mode.java b/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/annotations/Mode.java index 6c7c763..8ea7007 100644 --- a/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/annotations/Mode.java +++ b/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/annotations/Mode.java @@ -79,6 +79,20 @@ public enum Mode { */ SingleShotTime("ss", "Single shot invocation time"), + /** + *

CodSpeed walltime: captures raw per-iteration ops and duration.

+ * + *

Runs identically to {@link Mode#AverageTime} but preserves the raw + * {@code (ops, durationNs)} pair for each iteration instead of computing + * an averaged score. This allows CodSpeed to consume lossless walltime + * data without floating-point back-calculation.

+ * + *

This mode is applied automatically by the runner when CodSpeed + * instrumentation is detected. It is not intended for direct use via + * {@code @BenchmarkMode}.

+ */ + CodSpeed("cs", "CodSpeed walltime"), + /** * Meta-mode: all the benchmark modes. * This is mostly useful for internal JMH testing. diff --git a/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/generators/core/BenchmarkGenerator.java b/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/generators/core/BenchmarkGenerator.java index 60e2372..436c931 100644 --- a/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/generators/core/BenchmarkGenerator.java +++ b/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/generators/core/BenchmarkGenerator.java @@ -511,7 +511,7 @@ private void generateImport(PrintWriter writer) { TimeUnit.class, CompilerControl.class, InfraControl.class, ThreadParams.class, BenchmarkTaskResult.class, - Result.class, ThroughputResult.class, AverageTimeResult.class, + Result.class, ThroughputResult.class, AverageTimeResult.class, CodSpeedResult.class, SampleTimeResult.class, SingleShotResult.class, SampleBuffer.class, Mode.class, Fork.class, Measurement.class, Threads.class, Warmup.class, BenchmarkMode.class, RawResults.class, ResultRole.class, @@ -542,6 +542,9 @@ private void generateMethod(Mode benchmarkKind, PrintWriter writer, MethodGroup case SampleTime: generateSampleTime(writer, benchmarkKind, methodGroup, states); break; + case CodSpeed: + generateCodSpeed(writer, benchmarkKind, methodGroup, states); + break; case SingleShotTime: generateSingleShotTime(writer, benchmarkKind, methodGroup, states); break; @@ -687,6 +690,18 @@ private void addAuxCounters(PrintWriter writer, String resName, StateObjectHandl } private void generateAverageTime(PrintWriter writer, Mode benchmarkKind, MethodGroup methodGroup, StateObjectHandler states) { + generateAverageTimeImpl(writer, benchmarkKind, methodGroup, states, "AverageTimeResult"); + } + + /** + * Generates CodSpeed mode — identical measurement loop to AverageTime, but constructs + * CodSpeedResult (which retains raw ops + durationNs) instead of AverageTimeResult. + */ + private void generateCodSpeed(PrintWriter writer, Mode benchmarkKind, MethodGroup methodGroup, StateObjectHandler states) { + generateAverageTimeImpl(writer, benchmarkKind, methodGroup, states, "CodSpeedResult"); + } + + private void generateAverageTimeImpl(PrintWriter writer, Mode benchmarkKind, MethodGroup methodGroup, StateObjectHandler states, String resultClass) { writer.println(ident(1) + "public BenchmarkTaskResult " + methodGroup.getName() + "_" + benchmarkKind + "(InfraControl control, ThreadParams threadParams) throws Throwable {"); @@ -770,10 +785,10 @@ private void generateAverageTime(PrintWriter writer, Mode benchmarkKind, MethodG writer.println(ident(3) + "BenchmarkTaskResult results = new BenchmarkTaskResult((long)res.allOps, (long)res.measuredOps);"); if (isSingleMethod) { - writer.println(ident(3) + "results.add(new AverageTimeResult(ResultRole.PRIMARY, \"" + method.getName() + "\", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));"); + writer.println(ident(3) + "results.add(new " + resultClass + "(ResultRole.PRIMARY, \"" + method.getName() + "\", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));"); } else { - writer.println(ident(3) + "results.add(new AverageTimeResult(ResultRole.PRIMARY, \"" + methodGroup.getName() + "\", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));"); - writer.println(ident(3) + "results.add(new AverageTimeResult(ResultRole.SECONDARY, \"" + method.getName() + "\", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));"); + writer.println(ident(3) + "results.add(new " + resultClass + "(ResultRole.PRIMARY, \"" + methodGroup.getName() + "\", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));"); + writer.println(ident(3) + "results.add(new " + resultClass + "(ResultRole.SECONDARY, \"" + method.getName() + "\", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));"); } addAuxCounters(writer, "AverageTimeResult", states, method); diff --git a/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/results/CodSpeedResult.java b/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/results/CodSpeedResult.java new file mode 100644 index 0000000..2a0281b --- /dev/null +++ b/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/results/CodSpeedResult.java @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.jmh.results; + +import java.util.Collection; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.runner.options.TimeValue; +import org.openjdk.jmh.util.ListStatistics; +import org.openjdk.jmh.util.Statistics; + +/** + * Result class that stores raw per-iteration ops and duration for CodSpeed walltime collection. + * + *

Behaves like {@link AverageTimeResult} for display purposes (shows avg time/op), but + * retains the raw {@code (ops, durationNs)} pair so that {@code CodSpeedResultCollector} can + * read them without floating-point back-calculation. + */ +public class CodSpeedResult extends Result { + private static final long serialVersionUID = 1L; + + private final long rawOps; + private final long rawDurationNs; + + public CodSpeedResult( + ResultRole role, String label, double ops, long durationNs, TimeUnit tu) { + this( + role, + label, + of(durationNs / (ops * TimeUnit.NANOSECONDS.convert(1, tu))), + TimeValue.tuToString(tu) + "/op", + Math.round(ops), + durationNs); + } + + CodSpeedResult( + ResultRole role, + String label, + Statistics value, + String unit, + long rawOps, + long rawDurationNs) { + super(role, label, value, unit, AggregationPolicy.AVG); + this.rawOps = rawOps; + this.rawDurationNs = rawDurationNs; + } + + public long getRawOps() { + return rawOps; + } + + public long getRawDurationNs() { + return rawDurationNs; + } + + @Override + protected Aggregator getThreadAggregator() { + return new SummingAggregator(); + } + + @Override + protected Aggregator getIterationAggregator() { + return new SummingAggregator(); + } + + /** + * Sums raw ops and durations across threads/iterations, recomputes the avg score. + */ + static class SummingAggregator implements Aggregator { + @Override + public CodSpeedResult aggregate(Collection results) { + long totalOps = 0; + long totalDurationNs = 0; + for (CodSpeedResult r : results) { + totalOps += r.rawOps; + totalDurationNs += r.rawDurationNs; + } + + ListStatistics stat = new ListStatistics(); + for (CodSpeedResult r : results) { + stat.addValue(r.getScore()); + } + + return new CodSpeedResult( + AggregatorUtils.aggregateRoles(results), + AggregatorUtils.aggregateLabels(results), + stat, + AggregatorUtils.aggregateUnits(results), + totalOps, + totalDurationNs); + } + } +} diff --git a/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java b/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java index 3816010..e78c729 100644 --- a/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java +++ b/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java @@ -290,25 +290,27 @@ private Collection internalRun() throws RunnerException { { boolean isInstrumented = InstrumentHooks.getInstance().isInstrumented(); List newBenchmarks = new ArrayList<>(); - for (BenchmarkListEntry br : benchmarks) { - if (isInstrumented) { - // CodSpeed: normalize any mode to AverageTime - if (br.getMode() != Mode.AverageTime) { - out.println("# CodSpeed: normalizing benchmark mode to AverageTime " + - "(was: " + br.getMode() + ") for " + br.getUsername() + "."); + + if (isInstrumented) { + // CodSpeed: override all modes to CodSpeed, deduplicate by benchmark name + // so that multi-mode declarations (e.g. {Throughput, AverageTime}) run once. + Set seen = new LinkedHashSet<>(); + for (BenchmarkListEntry br : benchmarks) { + if (seen.add(br.getUsername())) { + newBenchmarks.add(br.cloneWith(Mode.CodSpeed)); } - newBenchmarks.add(br.cloneWith(Mode.AverageTime)); - continue; } - + } else { // Standard JMH: expand Mode.All into all concrete modes - if (br.getMode() == Mode.All) { + for (BenchmarkListEntry br : benchmarks) { + if (br.getMode() != Mode.All) { + newBenchmarks.add(br); + continue; + } for (Mode m : Mode.values()) { - if (m == Mode.All) continue; + if (m == Mode.All || m == Mode.CodSpeed) continue; newBenchmarks.add(br.cloneWith(m)); } - } else { - newBenchmarks.add(br); } }