feat: normalize benchmark mode to AverageTime under CodSpeed

jmh-fork/jmh-core/src/main/java/io/codspeed/result/CodSpeedResultCollector.java

@@ -7,16 +7,13 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
-import java.util.Iterator;
 import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
 import org.openjdk.jmh.annotations.Mode;
 import org.openjdk.jmh.infra.BenchmarkParams;
 import org.openjdk.jmh.results.BenchmarkResult;
+import org.openjdk.jmh.results.CodSpeedResult;
 import org.openjdk.jmh.results.IterationResult;
 import org.openjdk.jmh.results.RunResult;
-import org.openjdk.jmh.util.Statistics;
 
 public class CodSpeedResultCollector {
 
@@ -32,16 +29,28 @@ public static void collectAndWrite(
     for (RunResult runResult : runResults) {
       BenchmarkParams params = runResult.getParams();
       Mode mode = params.getMode();
-      TimeUnit timeUnit = params.getTimeUnit();
       String benchmarkName = params.getBenchmark();
       String uri = BenchmarkUri.fromBenchmarkParams(params);
-      double nanosPerUnit = TimeUnit.NANOSECONDS.convert(1, timeUnit);
 
-      if (mode == Mode.SampleTime) {
-        collectSampleTime(runResult, benchmarkName, uri, nanosPerUnit, benchmarks);
-      } else {
-        collectIterationBased(runResult, benchmarkName, uri, mode, nanosPerUnit, benchmarks);
+      if (mode != Mode.CodSpeed) {
+        throw new IllegalStateException(
+            "CodSpeedResultCollector only supports Mode.CodSpeed, got: " + mode);
       }
+
+      List<Long> itersPerRound = new ArrayList<>();
+      List<Long> timesPerRoundNs = new ArrayList<>();
+
+      for (BenchmarkResult br : runResult.getBenchmarkResults()) {
+        for (IterationResult ir : br.getIterationResults()) {
+          CodSpeedResult result = (CodSpeedResult) ir.getPrimaryResult();
+          itersPerRound.add(result.getRawOps());
+          timesPerRoundNs.add(result.getRawDurationNs());
+        }
+      }
+
+      benchmarks.add(
+          WalltimeBenchmark.fromRuntimeData(
+              benchmarkName, uri, toLongArray(itersPerRound), toLongArray(timesPerRoundNs), null));
     }
 
     if (benchmarks.isEmpty()) {
@@ -58,102 +67,6 @@ public static void collectAndWrite(
     Files.writeString(outputFile, gson.toJson(results));
   }
 
-  /**
-   * Collects results from iteration-based modes (Throughput, AverageTime, SingleShotTime).
-   *
-   * <p>In these modes, JMH runs a fixed time window per iteration and reports an aggregated score.
-   * Each iteration produces one data point: the total wall time for all ops in that iteration. We
-   * back-calculate the total wall time from the score (see inline comments for the formulas per
-   * mode).
-   *
-   * <p>Compare with {@link #collectSampleTime}, where JMH records per-operation latencies in a
-   * histogram, giving finer-grained data.
-   */
-  private static void collectIterationBased(
-      RunResult runResult,
-      String benchmarkName,
-      String uri,
-      Mode mode,
-      double nanosPerUnit,
-      List<WalltimeBenchmark> benchmarks) {
-    List<Long> itersPerRound = new ArrayList<>();
-    List<Long> timesPerRoundNs = new ArrayList<>();
-
-    for (BenchmarkResult br : runResult.getBenchmarkResults()) {
-      for (IterationResult ir : br.getIterationResults()) {
-        long ops = ir.getMetadata().getMeasuredOps();
-        double score = ir.getPrimaryResult().getScore();
-
-        long timeNs;
-        if (mode == Mode.Throughput) {
-          // score = ops * nanosPerUnit / durationNs
-          timeNs = Math.round(ops * nanosPerUnit / score);
-        } else {
-          // avgt/ss: score = durationNs / (ops * nanosPerUnit)
-          timeNs = Math.round(score * ops * nanosPerUnit);
-        }
-
-        itersPerRound.add(ops);
-        timesPerRoundNs.add(timeNs);
-      }
-    }
-
-    if (itersPerRound.isEmpty()) {
-      return;
-    }
-
-    benchmarks.add(
-        WalltimeBenchmark.fromRuntimeData(
-            benchmarkName, uri, toLongArray(itersPerRound), toLongArray(timesPerRoundNs), null));
-  }
-
-  /**
-   * Collects results from SampleTime mode.
-   *
-   * <p>In this mode, JMH measures each individual operation's latency separately and records them
-   * in a histogram. Each sample is a direct timing of one op, giving finer-grained data than
-   * iteration-based modes. We treat each sample as its own round with 1 iteration.
-   */
-  private static void collectSampleTime(
-      RunResult runResult,
-      String benchmarkName,
-      String uri,
-      double nanosPerUnit,
-      List<WalltimeBenchmark> benchmarks) {
-    List<Long> itersPerRound = new ArrayList<>();
-    List<Long> timesPerRoundNs = new ArrayList<>();
-
-    for (BenchmarkResult br : runResult.getBenchmarkResults()) {
-      for (IterationResult ir : br.getIterationResults()) {
-        Statistics stats = ir.getPrimaryResult().getStatistics();
-        Iterator<Map.Entry<Double, Long>> rawData = stats.getRawData();
-
-        while (rawData.hasNext()) {
-          Map.Entry<Double, Long> entry = rawData.next();
-          double valueInUnit = entry.getKey();
-          long count = entry.getValue();
-
-          // Each sample is one op's time in the output unit.
-          // Convert back to nanoseconds.
-          long timeNs = Math.round(valueInUnit * nanosPerUnit);
-
-          for (long i = 0; i < count; i++) {
-            itersPerRound.add(1L);
-            timesPerRoundNs.add(timeNs);
-          }
-        }
-      }
-    }
-
-    if (itersPerRound.isEmpty()) {
-      return;
-    }
-
-    benchmarks.add(
-        WalltimeBenchmark.fromRuntimeData(
-            benchmarkName, uri, toLongArray(itersPerRound), toLongArray(timesPerRoundNs), null));
-  }
-
   private static long[] toLongArray(List<Long> list) {
     long[] arr = new long[list.size()];
     for (int i = 0; i < list.size(); i++) {

jmh-fork/jmh-core/src/main/java/io/codspeed/result/WalltimeBenchmark.java

@@ -5,7 +5,7 @@
 
 public class WalltimeBenchmark {
   private static final double IQR_OUTLIER_FACTOR = 1.5;
-  private static final double STDEV_OUTLIER_FACTOR = 2.0;
+  private static final double STDEV_OUTLIER_FACTOR = 3.0;
 
   private final String name;
   private final String uri;

jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/annotations/Mode.java

@@ -79,6 +79,20 @@ public enum Mode {
      */
     SingleShotTime("ss", "Single shot invocation time"),
 
+    /**
+     * <p>CodSpeed walltime: captures raw per-iteration ops and duration.</p>
+     *
+     * <p>Runs identically to {@link Mode#AverageTime} but preserves the raw
+     * {@code (ops, durationNs)} pair for each iteration instead of computing
+     * an averaged score. This allows CodSpeed to consume lossless walltime
+     * data without floating-point back-calculation.</p>
+     *
+     * <p>This mode is applied automatically by the runner when CodSpeed
+     * instrumentation is detected. It is not intended for direct use via
+     * {@code @BenchmarkMode}.</p>
+     */
+    CodSpeed("cs", "CodSpeed walltime"),
+
     /**
      * Meta-mode: all the benchmark modes.
      * This is mostly useful for internal JMH testing.

jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/generators/core/BenchmarkGenerator.java

@@ -511,7 +511,7 @@ private void generateImport(PrintWriter writer) {
                 TimeUnit.class, CompilerControl.class,
                 InfraControl.class, ThreadParams.class,
                 BenchmarkTaskResult.class,
-                Result.class, ThroughputResult.class, AverageTimeResult.class,
+                Result.class, ThroughputResult.class, AverageTimeResult.class, CodSpeedResult.class,
                 SampleTimeResult.class, SingleShotResult.class, SampleBuffer.class,
                 Mode.class, Fork.class, Measurement.class, Threads.class, Warmup.class,
                 BenchmarkMode.class, RawResults.class, ResultRole.class,
@@ -542,6 +542,9 @@ private void generateMethod(Mode benchmarkKind, PrintWriter writer, MethodGroup
             case SampleTime:
                 generateSampleTime(writer, benchmarkKind, methodGroup, states);
                 break;
+            case CodSpeed:
+                generateCodSpeed(writer, benchmarkKind, methodGroup, states);
+                break;
             case SingleShotTime:
                 generateSingleShotTime(writer, benchmarkKind, methodGroup, states);
                 break;
@@ -687,6 +690,18 @@ private void addAuxCounters(PrintWriter writer, String resName, StateObjectHandl
     }
 
     private void generateAverageTime(PrintWriter writer, Mode benchmarkKind, MethodGroup methodGroup, StateObjectHandler states) {
+        generateAverageTimeImpl(writer, benchmarkKind, methodGroup, states, "AverageTimeResult");
+    }
+
+    /**
+     * Generates CodSpeed mode — identical measurement loop to AverageTime, but constructs
+     * CodSpeedResult (which retains raw ops + durationNs) instead of AverageTimeResult.
+     */
+    private void generateCodSpeed(PrintWriter writer, Mode benchmarkKind, MethodGroup methodGroup, StateObjectHandler states) {
+        generateAverageTimeImpl(writer, benchmarkKind, methodGroup, states, "CodSpeedResult");
+    }
+
+    private void generateAverageTimeImpl(PrintWriter writer, Mode benchmarkKind, MethodGroup methodGroup, StateObjectHandler states, String resultClass) {
         writer.println(ident(1) + "public BenchmarkTaskResult " + methodGroup.getName() + "_" + benchmarkKind +
                 "(InfraControl control, ThreadParams threadParams) throws Throwable {");
 
@@ -770,10 +785,10 @@ private void generateAverageTime(PrintWriter writer, Mode benchmarkKind, MethodG
 
             writer.println(ident(3) + "BenchmarkTaskResult results = new BenchmarkTaskResult((long)res.allOps, (long)res.measuredOps);");
             if (isSingleMethod) {
-                writer.println(ident(3) + "results.add(new AverageTimeResult(ResultRole.PRIMARY, \"" + method.getName() + "\", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));");
+                writer.println(ident(3) + "results.add(new " + resultClass + "(ResultRole.PRIMARY, \"" + method.getName() + "\", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));");
             } else {
-                writer.println(ident(3) + "results.add(new AverageTimeResult(ResultRole.PRIMARY, \"" + methodGroup.getName() + "\", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));");
-                writer.println(ident(3) + "results.add(new AverageTimeResult(ResultRole.SECONDARY, \"" + method.getName() + "\", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));");
+                writer.println(ident(3) + "results.add(new " + resultClass + "(ResultRole.PRIMARY, \"" + methodGroup.getName() + "\", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));");
+                writer.println(ident(3) + "results.add(new " + resultClass + "(ResultRole.SECONDARY, \"" + method.getName() + "\", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));");
             }
             addAuxCounters(writer, "AverageTimeResult", states, method);
 

jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/results/CodSpeedResult.java

@@ -0,0 +1,114 @@
+/*
+ * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.jmh.results;
+
+import java.util.Collection;
+import java.util.concurrent.TimeUnit;
+import org.openjdk.jmh.runner.options.TimeValue;
+import org.openjdk.jmh.util.ListStatistics;
+import org.openjdk.jmh.util.Statistics;
+
+/**
+ * Result class that stores raw per-iteration ops and duration for CodSpeed walltime collection.
+ *
+ * <p>Behaves like {@link AverageTimeResult} for display purposes (shows avg time/op), but
+ * retains the raw {@code (ops, durationNs)} pair so that {@code CodSpeedResultCollector} can
+ * read them without floating-point back-calculation.
+ */
+public class CodSpeedResult extends Result<CodSpeedResult> {
+  private static final long serialVersionUID = 1L;
+
+  private final long rawOps;
+  private final long rawDurationNs;
+
+  public CodSpeedResult(
+      ResultRole role, String label, double ops, long durationNs, TimeUnit tu) {
+    this(
+        role,
+        label,
+        of(durationNs / (ops * TimeUnit.NANOSECONDS.convert(1, tu))),
+        TimeValue.tuToString(tu) + "/op",
+        Math.round(ops),
+        durationNs);
+  }
+
+  CodSpeedResult(
+      ResultRole role,
+      String label,
+      Statistics value,
+      String unit,
+      long rawOps,
+      long rawDurationNs) {
+    super(role, label, value, unit, AggregationPolicy.AVG);
+    this.rawOps = rawOps;
+    this.rawDurationNs = rawDurationNs;
+  }
+
+  public long getRawOps() {
+    return rawOps;
+  }
+
+  public long getRawDurationNs() {
+    return rawDurationNs;
+  }
+
+  @Override
+  protected Aggregator<CodSpeedResult> getThreadAggregator() {
+    return new SummingAggregator();
+  }
+
+  @Override
+  protected Aggregator<CodSpeedResult> getIterationAggregator() {
+    return new SummingAggregator();
+  }
+
+  /**
+   * Sums raw ops and durations across threads/iterations, recomputes the avg score.
+   */
+  static class SummingAggregator implements Aggregator<CodSpeedResult> {
+    @Override
+    public CodSpeedResult aggregate(Collection<CodSpeedResult> results) {
+      long totalOps = 0;
+      long totalDurationNs = 0;
+      for (CodSpeedResult r : results) {
+        totalOps += r.rawOps;
+        totalDurationNs += r.rawDurationNs;
+      }
+
+      ListStatistics stat = new ListStatistics();
+      for (CodSpeedResult r : results) {
+        stat.addValue(r.getScore());
+      }
+
+      return new CodSpeedResult(
+          AggregatorUtils.aggregateRoles(results),
+          AggregatorUtils.aggregateLabels(results),
+          stat,
+          AggregatorUtils.aggregateUnits(results),
+          totalOps,
+          totalDurationNs);
+    }
+  }
+}