diff --git a/jmh-fork/jmh-core/src/main/java/io/codspeed/result/CodSpeedResultCollector.java b/jmh-fork/jmh-core/src/main/java/io/codspeed/result/CodSpeedResultCollector.java index 3f9dc17..3f5ab12 100644 --- a/jmh-fork/jmh-core/src/main/java/io/codspeed/result/CodSpeedResultCollector.java +++ b/jmh-fork/jmh-core/src/main/java/io/codspeed/result/CodSpeedResultCollector.java @@ -7,16 +7,13 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; -import java.util.Iterator; import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; import org.openjdk.jmh.annotations.Mode; import org.openjdk.jmh.infra.BenchmarkParams; import org.openjdk.jmh.results.BenchmarkResult; +import org.openjdk.jmh.results.CodSpeedResult; import org.openjdk.jmh.results.IterationResult; import org.openjdk.jmh.results.RunResult; -import org.openjdk.jmh.util.Statistics; public class CodSpeedResultCollector { @@ -32,16 +29,28 @@ public static void collectAndWrite( for (RunResult runResult : runResults) { BenchmarkParams params = runResult.getParams(); Mode mode = params.getMode(); - TimeUnit timeUnit = params.getTimeUnit(); String benchmarkName = params.getBenchmark(); String uri = BenchmarkUri.fromBenchmarkParams(params); - double nanosPerUnit = TimeUnit.NANOSECONDS.convert(1, timeUnit); - if (mode == Mode.SampleTime) { - collectSampleTime(runResult, benchmarkName, uri, nanosPerUnit, benchmarks); - } else { - collectIterationBased(runResult, benchmarkName, uri, mode, nanosPerUnit, benchmarks); + if (mode != Mode.CodSpeed) { + throw new IllegalStateException( + "CodSpeedResultCollector only supports Mode.CodSpeed, got: " + mode); } + + List itersPerRound = new ArrayList<>(); + List timesPerRoundNs = new ArrayList<>(); + + for (BenchmarkResult br : runResult.getBenchmarkResults()) { + for (IterationResult ir : br.getIterationResults()) { + CodSpeedResult result = (CodSpeedResult) ir.getPrimaryResult(); + itersPerRound.add(result.getRawOps()); + timesPerRoundNs.add(result.getRawDurationNs()); + } + } + + benchmarks.add( + WalltimeBenchmark.fromRuntimeData( + benchmarkName, uri, toLongArray(itersPerRound), toLongArray(timesPerRoundNs), null)); } if (benchmarks.isEmpty()) { @@ -58,102 +67,6 @@ public static void collectAndWrite( Files.writeString(outputFile, gson.toJson(results)); } - /** - * Collects results from iteration-based modes (Throughput, AverageTime, SingleShotTime). - * - *

In these modes, JMH runs a fixed time window per iteration and reports an aggregated score. - * Each iteration produces one data point: the total wall time for all ops in that iteration. We - * back-calculate the total wall time from the score (see inline comments for the formulas per - * mode). - * - *

Compare with {@link #collectSampleTime}, where JMH records per-operation latencies in a - * histogram, giving finer-grained data. - */ - private static void collectIterationBased( - RunResult runResult, - String benchmarkName, - String uri, - Mode mode, - double nanosPerUnit, - List benchmarks) { - List itersPerRound = new ArrayList<>(); - List timesPerRoundNs = new ArrayList<>(); - - for (BenchmarkResult br : runResult.getBenchmarkResults()) { - for (IterationResult ir : br.getIterationResults()) { - long ops = ir.getMetadata().getMeasuredOps(); - double score = ir.getPrimaryResult().getScore(); - - long timeNs; - if (mode == Mode.Throughput) { - // score = ops * nanosPerUnit / durationNs - timeNs = Math.round(ops * nanosPerUnit / score); - } else { - // avgt/ss: score = durationNs / (ops * nanosPerUnit) - timeNs = Math.round(score * ops * nanosPerUnit); - } - - itersPerRound.add(ops); - timesPerRoundNs.add(timeNs); - } - } - - if (itersPerRound.isEmpty()) { - return; - } - - benchmarks.add( - WalltimeBenchmark.fromRuntimeData( - benchmarkName, uri, toLongArray(itersPerRound), toLongArray(timesPerRoundNs), null)); - } - - /** - * Collects results from SampleTime mode. - * - *

In this mode, JMH measures each individual operation's latency separately and records them - * in a histogram. Each sample is a direct timing of one op, giving finer-grained data than - * iteration-based modes. We treat each sample as its own round with 1 iteration. - */ - private static void collectSampleTime( - RunResult runResult, - String benchmarkName, - String uri, - double nanosPerUnit, - List benchmarks) { - List itersPerRound = new ArrayList<>(); - List timesPerRoundNs = new ArrayList<>(); - - for (BenchmarkResult br : runResult.getBenchmarkResults()) { - for (IterationResult ir : br.getIterationResults()) { - Statistics stats = ir.getPrimaryResult().getStatistics(); - Iterator> rawData = stats.getRawData(); - - while (rawData.hasNext()) { - Map.Entry entry = rawData.next(); - double valueInUnit = entry.getKey(); - long count = entry.getValue(); - - // Each sample is one op's time in the output unit. - // Convert back to nanoseconds. - long timeNs = Math.round(valueInUnit * nanosPerUnit); - - for (long i = 0; i < count; i++) { - itersPerRound.add(1L); - timesPerRoundNs.add(timeNs); - } - } - } - } - - if (itersPerRound.isEmpty()) { - return; - } - - benchmarks.add( - WalltimeBenchmark.fromRuntimeData( - benchmarkName, uri, toLongArray(itersPerRound), toLongArray(timesPerRoundNs), null)); - } - private static long[] toLongArray(List list) { long[] arr = new long[list.size()]; for (int i = 0; i < list.size(); i++) { diff --git a/jmh-fork/jmh-core/src/main/java/io/codspeed/result/WalltimeBenchmark.java b/jmh-fork/jmh-core/src/main/java/io/codspeed/result/WalltimeBenchmark.java index 875242f..d281c32 100644 --- a/jmh-fork/jmh-core/src/main/java/io/codspeed/result/WalltimeBenchmark.java +++ b/jmh-fork/jmh-core/src/main/java/io/codspeed/result/WalltimeBenchmark.java @@ -5,7 +5,7 @@ public class WalltimeBenchmark { private static final double IQR_OUTLIER_FACTOR = 1.5; - private static final double STDEV_OUTLIER_FACTOR = 2.0; + private static final double STDEV_OUTLIER_FACTOR = 3.0; private final String name; private final String uri; diff --git a/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/annotations/Mode.java b/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/annotations/Mode.java index 6c7c763..8ea7007 100644 --- a/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/annotations/Mode.java +++ b/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/annotations/Mode.java @@ -79,6 +79,20 @@ public enum Mode { */ SingleShotTime("ss", "Single shot invocation time"), + /** + *

CodSpeed walltime: captures raw per-iteration ops and duration.

+ * + *

Runs identically to {@link Mode#AverageTime} but preserves the raw + * {@code (ops, durationNs)} pair for each iteration instead of computing + * an averaged score. This allows CodSpeed to consume lossless walltime + * data without floating-point back-calculation.

+ * + *

This mode is applied automatically by the runner when CodSpeed + * instrumentation is detected. It is not intended for direct use via + * {@code @BenchmarkMode}.

+ */ + CodSpeed("cs", "CodSpeed walltime"), + /** * Meta-mode: all the benchmark modes. * This is mostly useful for internal JMH testing. diff --git a/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/generators/core/BenchmarkGenerator.java b/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/generators/core/BenchmarkGenerator.java index 60e2372..436c931 100644 --- a/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/generators/core/BenchmarkGenerator.java +++ b/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/generators/core/BenchmarkGenerator.java @@ -511,7 +511,7 @@ private void generateImport(PrintWriter writer) { TimeUnit.class, CompilerControl.class, InfraControl.class, ThreadParams.class, BenchmarkTaskResult.class, - Result.class, ThroughputResult.class, AverageTimeResult.class, + Result.class, ThroughputResult.class, AverageTimeResult.class, CodSpeedResult.class, SampleTimeResult.class, SingleShotResult.class, SampleBuffer.class, Mode.class, Fork.class, Measurement.class, Threads.class, Warmup.class, BenchmarkMode.class, RawResults.class, ResultRole.class, @@ -542,6 +542,9 @@ private void generateMethod(Mode benchmarkKind, PrintWriter writer, MethodGroup case SampleTime: generateSampleTime(writer, benchmarkKind, methodGroup, states); break; + case CodSpeed: + generateCodSpeed(writer, benchmarkKind, methodGroup, states); + break; case SingleShotTime: generateSingleShotTime(writer, benchmarkKind, methodGroup, states); break; @@ -687,6 +690,18 @@ private void addAuxCounters(PrintWriter writer, String resName, StateObjectHandl } private void generateAverageTime(PrintWriter writer, Mode benchmarkKind, MethodGroup methodGroup, StateObjectHandler states) { + generateAverageTimeImpl(writer, benchmarkKind, methodGroup, states, "AverageTimeResult"); + } + + /** + * Generates CodSpeed mode — identical measurement loop to AverageTime, but constructs + * CodSpeedResult (which retains raw ops + durationNs) instead of AverageTimeResult. + */ + private void generateCodSpeed(PrintWriter writer, Mode benchmarkKind, MethodGroup methodGroup, StateObjectHandler states) { + generateAverageTimeImpl(writer, benchmarkKind, methodGroup, states, "CodSpeedResult"); + } + + private void generateAverageTimeImpl(PrintWriter writer, Mode benchmarkKind, MethodGroup methodGroup, StateObjectHandler states, String resultClass) { writer.println(ident(1) + "public BenchmarkTaskResult " + methodGroup.getName() + "_" + benchmarkKind + "(InfraControl control, ThreadParams threadParams) throws Throwable {"); @@ -770,10 +785,10 @@ private void generateAverageTime(PrintWriter writer, Mode benchmarkKind, MethodG writer.println(ident(3) + "BenchmarkTaskResult results = new BenchmarkTaskResult((long)res.allOps, (long)res.measuredOps);"); if (isSingleMethod) { - writer.println(ident(3) + "results.add(new AverageTimeResult(ResultRole.PRIMARY, \"" + method.getName() + "\", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));"); + writer.println(ident(3) + "results.add(new " + resultClass + "(ResultRole.PRIMARY, \"" + method.getName() + "\", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));"); } else { - writer.println(ident(3) + "results.add(new AverageTimeResult(ResultRole.PRIMARY, \"" + methodGroup.getName() + "\", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));"); - writer.println(ident(3) + "results.add(new AverageTimeResult(ResultRole.SECONDARY, \"" + method.getName() + "\", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));"); + writer.println(ident(3) + "results.add(new " + resultClass + "(ResultRole.PRIMARY, \"" + methodGroup.getName() + "\", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));"); + writer.println(ident(3) + "results.add(new " + resultClass + "(ResultRole.SECONDARY, \"" + method.getName() + "\", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));"); } addAuxCounters(writer, "AverageTimeResult", states, method); diff --git a/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/results/CodSpeedResult.java b/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/results/CodSpeedResult.java new file mode 100644 index 0000000..2a0281b --- /dev/null +++ b/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/results/CodSpeedResult.java @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.jmh.results; + +import java.util.Collection; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.runner.options.TimeValue; +import org.openjdk.jmh.util.ListStatistics; +import org.openjdk.jmh.util.Statistics; + +/** + * Result class that stores raw per-iteration ops and duration for CodSpeed walltime collection. + * + *

Behaves like {@link AverageTimeResult} for display purposes (shows avg time/op), but + * retains the raw {@code (ops, durationNs)} pair so that {@code CodSpeedResultCollector} can + * read them without floating-point back-calculation. + */ +public class CodSpeedResult extends Result { + private static final long serialVersionUID = 1L; + + private final long rawOps; + private final long rawDurationNs; + + public CodSpeedResult( + ResultRole role, String label, double ops, long durationNs, TimeUnit tu) { + this( + role, + label, + of(durationNs / (ops * TimeUnit.NANOSECONDS.convert(1, tu))), + TimeValue.tuToString(tu) + "/op", + Math.round(ops), + durationNs); + } + + CodSpeedResult( + ResultRole role, + String label, + Statistics value, + String unit, + long rawOps, + long rawDurationNs) { + super(role, label, value, unit, AggregationPolicy.AVG); + this.rawOps = rawOps; + this.rawDurationNs = rawDurationNs; + } + + public long getRawOps() { + return rawOps; + } + + public long getRawDurationNs() { + return rawDurationNs; + } + + @Override + protected Aggregator getThreadAggregator() { + return new SummingAggregator(); + } + + @Override + protected Aggregator getIterationAggregator() { + return new SummingAggregator(); + } + + /** + * Sums raw ops and durations across threads/iterations, recomputes the avg score. + */ + static class SummingAggregator implements Aggregator { + @Override + public CodSpeedResult aggregate(Collection results) { + long totalOps = 0; + long totalDurationNs = 0; + for (CodSpeedResult r : results) { + totalOps += r.rawOps; + totalDurationNs += r.rawDurationNs; + } + + ListStatistics stat = new ListStatistics(); + for (CodSpeedResult r : results) { + stat.addValue(r.getScore()); + } + + return new CodSpeedResult( + AggregatorUtils.aggregateRoles(results), + AggregatorUtils.aggregateLabels(results), + stat, + AggregatorUtils.aggregateUnits(results), + totalOps, + totalDurationNs); + } + } +} diff --git a/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java b/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java index 59e6ab0..e78c729 100644 --- a/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java +++ b/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java @@ -266,13 +266,12 @@ private Collection internalRun() throws RunnerException { if (!options.getBenchModes().isEmpty()) { Collection benchModes = options.getBenchModes(); - // CodSpeed: warn and pick first mode if multiple are specified via CLI (-bm flag) - if (InstrumentHooks.getInstance().isInstrumented() && benchModes.size() > 1) { - Mode firstMode = benchModes.iterator().next(); - out.println("WARNING: CodSpeed does not support multiple benchmark modes. " + - "The -bm flag specifies " + benchModes.size() + " modes: " + benchModes + ". " + - "Using only the first mode: " + firstMode); - benchModes = Collections.singleton(firstMode); + // CodSpeed: normalize CLI-specified modes to AverageTime + if (InstrumentHooks.getInstance().isInstrumented()) { + if (!benchModes.equals(Collections.singleton(Mode.AverageTime))) { + out.println("# CodSpeed: normalizing benchmark mode to AverageTime (was: " + benchModes + ")."); + } + benchModes = Collections.singleton(Mode.AverageTime); } List newBenchmarks = new ArrayList<>(); @@ -291,23 +290,25 @@ private Collection internalRun() throws RunnerException { { boolean isInstrumented = InstrumentHooks.getInstance().isInstrumented(); List newBenchmarks = new ArrayList<>(); - for (BenchmarkListEntry br : benchmarks) { - if (br.getMode() != Mode.All) { - newBenchmarks.add(br); - continue; - } - if (isInstrumented) { - // CodSpeed: Mode.All would expand into multiple modes per benchmark, - // which we don't support. Pick a single mode and warn. - Mode firstMode = Mode.SampleTime; - out.println("WARNING: CodSpeed does not support multiple benchmark modes. " + - br.getUsername() + " uses Mode.All, using " + firstMode + " instead."); - newBenchmarks.add(br.cloneWith(firstMode)); - } else { - // Standard JMH: expand Mode.All into all concrete modes + if (isInstrumented) { + // CodSpeed: override all modes to CodSpeed, deduplicate by benchmark name + // so that multi-mode declarations (e.g. {Throughput, AverageTime}) run once. + Set seen = new LinkedHashSet<>(); + for (BenchmarkListEntry br : benchmarks) { + if (seen.add(br.getUsername())) { + newBenchmarks.add(br.cloneWith(Mode.CodSpeed)); + } + } + } else { + // Standard JMH: expand Mode.All into all concrete modes + for (BenchmarkListEntry br : benchmarks) { + if (br.getMode() != Mode.All) { + newBenchmarks.add(br); + continue; + } for (Mode m : Mode.values()) { - if (m == Mode.All) continue; + if (m == Mode.All || m == Mode.CodSpeed) continue; newBenchmarks.add(br.cloneWith(m)); } }