Skip to content

Commit 8cb09f5

Browse files
not-matthiasnot-matthias
committed
feat(jmh-fork): add CodSpeed mode to capture raw ops and duration per iteration
Add Mode.CodSpeed, an internal benchmark mode that runs identically to AverageTime but preserves the raw (ops, durationNs) pair per iteration in a new CodSpeedResult class. This eliminates the lossy floating-point roundtrip that the previous approach required (score = time/ops, then back-calculate time = score * ops). When CodSpeed instrumentation is detected, the Runner automatically overrides all declared benchmark modes to CodSpeed and deduplicates by benchmark name so multi-mode declarations run exactly once. CodSpeedResultCollector is simplified to only handle Mode.CodSpeed, reading raw values directly from CodSpeedResult with no back-calculation.
1 parent ba36d4c commit 8cb09f5

5 files changed

Lines changed: 184 additions & 62 deletions

File tree

‎jmh-fork/jmh-core/src/main/java/io/codspeed/result/CodSpeedResultCollector.java‎

Lines changed: 22 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,10 @@
88
import java.nio.file.Path;
99
import java.util.ArrayList;
1010
import java.util.List;
11-
import java.util.concurrent.TimeUnit;
11+
import org.openjdk.jmh.annotations.Mode;
1212
import org.openjdk.jmh.infra.BenchmarkParams;
1313
import org.openjdk.jmh.results.BenchmarkResult;
14+
import org.openjdk.jmh.results.CodSpeedResult;
1415
import org.openjdk.jmh.results.IterationResult;
1516
import org.openjdk.jmh.results.RunResult;
1617

@@ -27,12 +28,29 @@ public static void collectAndWrite(
2728

2829
for (RunResult runResult : runResults) {
2930
BenchmarkParams params = runResult.getParams();
30-
TimeUnit timeUnit = params.getTimeUnit();
31+
Mode mode = params.getMode();
3132
String benchmarkName = params.getBenchmark();
3233
String uri = BenchmarkUri.fromBenchmarkParams(params);
33-
double nanosPerUnit = TimeUnit.NANOSECONDS.convert(1, timeUnit);
3434

35-
collectAverageTime(runResult, benchmarkName, uri, nanosPerUnit, benchmarks);
35+
if (mode != Mode.CodSpeed) {
36+
throw new IllegalStateException(
37+
"CodSpeedResultCollector only supports Mode.CodSpeed, got: " + mode);
38+
}
39+
40+
List<Long> itersPerRound = new ArrayList<>();
41+
List<Long> timesPerRoundNs = new ArrayList<>();
42+
43+
for (BenchmarkResult br : runResult.getBenchmarkResults()) {
44+
for (IterationResult ir : br.getIterationResults()) {
45+
CodSpeedResult result = (CodSpeedResult) ir.getPrimaryResult();
46+
itersPerRound.add(result.getRawOps());
47+
timesPerRoundNs.add(result.getRawDurationNs());
48+
}
49+
}
50+
51+
benchmarks.add(
52+
WalltimeBenchmark.fromRuntimeData(
53+
benchmarkName, uri, toLongArray(itersPerRound), toLongArray(timesPerRoundNs), null));
3654
}
3755

3856
if (benchmarks.isEmpty()) {
@@ -49,47 +67,6 @@ public static void collectAndWrite(
4967
Files.writeString(outputFile, gson.toJson(results));
5068
}
5169

52-
/**
53-
* Collects results from AverageTime mode.
54-
*
55-
* <p>JMH runs a fixed time window per iteration and reports the average time per operation. Each
56-
* iteration produces one data point: the total wall time for all ops in that iteration, which we
57-
* back-calculate from the score: {@code timeNs = score * ops * nanosPerUnit}.
58-
*
59-
* <p>This is structurally equivalent to how criterion (codspeed-rust) collects walltime data:
60-
* each round has a variable iteration count and a measured total wall time.
61-
*/
62-
private static void collectAverageTime(
63-
RunResult runResult,
64-
String benchmarkName,
65-
String uri,
66-
double nanosPerUnit,
67-
List<WalltimeBenchmark> benchmarks) {
68-
List<Long> itersPerRound = new ArrayList<>();
69-
List<Long> timesPerRoundNs = new ArrayList<>();
70-
71-
for (BenchmarkResult br : runResult.getBenchmarkResults()) {
72-
for (IterationResult ir : br.getIterationResults()) {
73-
long ops = ir.getMetadata().getMeasuredOps();
74-
double score = ir.getPrimaryResult().getScore();
75-
76-
// avgt: score = durationNs / (ops * nanosPerUnit)
77-
long timeNs = Math.round(score * ops * nanosPerUnit);
78-
79-
itersPerRound.add(ops);
80-
timesPerRoundNs.add(timeNs);
81-
}
82-
}
83-
84-
if (itersPerRound.isEmpty()) {
85-
return;
86-
}
87-
88-
benchmarks.add(
89-
WalltimeBenchmark.fromRuntimeData(
90-
benchmarkName, uri, toLongArray(itersPerRound), toLongArray(timesPerRoundNs), null));
91-
}
92-
9370
private static long[] toLongArray(List<Long> list) {
9471
long[] arr = new long[list.size()];
9572
for (int i = 0; i < list.size(); i++) {

‎jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/annotations/Mode.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,20 @@ public enum Mode {
7979
*/
8080
SingleShotTime("ss", "Single shot invocation time"),
8181

82+
/**
83+
* <p>CodSpeed walltime: captures raw per-iteration ops and duration.</p>
84+
*
85+
* <p>Runs identically to {@link Mode#AverageTime} but preserves the raw
86+
* {@code (ops, durationNs)} pair for each iteration instead of computing
87+
* an averaged score. This allows CodSpeed to consume lossless walltime
88+
* data without floating-point back-calculation.</p>
89+
*
90+
* <p>This mode is applied automatically by the runner when CodSpeed
91+
* instrumentation is detected. It is not intended for direct use via
92+
* {@code @BenchmarkMode}.</p>
93+
*/
94+
CodSpeed("cs", "CodSpeed walltime"),
95+
8296
/**
8397
* Meta-mode: all the benchmark modes.
8498
* This is mostly useful for internal JMH testing.

‎jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/generators/core/BenchmarkGenerator.java‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -511,7 +511,7 @@ private void generateImport(PrintWriter writer) {
511511
TimeUnit.class, CompilerControl.class,
512512
InfraControl.class, ThreadParams.class,
513513
BenchmarkTaskResult.class,
514-
Result.class, ThroughputResult.class, AverageTimeResult.class,
514+
Result.class, ThroughputResult.class, AverageTimeResult.class, CodSpeedResult.class,
515515
SampleTimeResult.class, SingleShotResult.class, SampleBuffer.class,
516516
Mode.class, Fork.class, Measurement.class, Threads.class, Warmup.class,
517517
BenchmarkMode.class, RawResults.class, ResultRole.class,
@@ -542,6 +542,9 @@ private void generateMethod(Mode benchmarkKind, PrintWriter writer, MethodGroup
542542
case SampleTime:
543543
generateSampleTime(writer, benchmarkKind, methodGroup, states);
544544
break;
545+
case CodSpeed:
546+
generateCodSpeed(writer, benchmarkKind, methodGroup, states);
547+
break;
545548
case SingleShotTime:
546549
generateSingleShotTime(writer, benchmarkKind, methodGroup, states);
547550
break;
@@ -687,6 +690,18 @@ private void addAuxCounters(PrintWriter writer, String resName, StateObjectHandl
687690
}
688691

689692
private void generateAverageTime(PrintWriter writer, Mode benchmarkKind, MethodGroup methodGroup, StateObjectHandler states) {
693+
generateAverageTimeImpl(writer, benchmarkKind, methodGroup, states, "AverageTimeResult");
694+
}
695+
696+
/**
697+
* Generates CodSpeed mode — identical measurement loop to AverageTime, but constructs
698+
* CodSpeedResult (which retains raw ops + durationNs) instead of AverageTimeResult.
699+
*/
700+
private void generateCodSpeed(PrintWriter writer, Mode benchmarkKind, MethodGroup methodGroup, StateObjectHandler states) {
701+
generateAverageTimeImpl(writer, benchmarkKind, methodGroup, states, "CodSpeedResult");
702+
}
703+
704+
private void generateAverageTimeImpl(PrintWriter writer, Mode benchmarkKind, MethodGroup methodGroup, StateObjectHandler states, String resultClass) {
690705
writer.println(ident(1) + "public BenchmarkTaskResult " + methodGroup.getName() + "_" + benchmarkKind +
691706
"(InfraControl control, ThreadParams threadParams) throws Throwable {");
692707

@@ -770,10 +785,10 @@ private void generateAverageTime(PrintWriter writer, Mode benchmarkKind, MethodG
770785

771786
writer.println(ident(3) + "BenchmarkTaskResult results = new BenchmarkTaskResult((long)res.allOps, (long)res.measuredOps);");
772787
if (isSingleMethod) {
773-
writer.println(ident(3) + "results.add(new AverageTimeResult(ResultRole.PRIMARY, \"" + method.getName() + "\", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));");
788+
writer.println(ident(3) + "results.add(new " + resultClass + "(ResultRole.PRIMARY, \"" + method.getName() + "\", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));");
774789
} else {
775-
writer.println(ident(3) + "results.add(new AverageTimeResult(ResultRole.PRIMARY, \"" + methodGroup.getName() + "\", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));");
776-
writer.println(ident(3) + "results.add(new AverageTimeResult(ResultRole.SECONDARY, \"" + method.getName() + "\", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));");
790+
writer.println(ident(3) + "results.add(new " + resultClass + "(ResultRole.PRIMARY, \"" + methodGroup.getName() + "\", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));");
791+
writer.println(ident(3) + "results.add(new " + resultClass + "(ResultRole.SECONDARY, \"" + method.getName() + "\", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));");
777792
}
778793
addAuxCounters(writer, "AverageTimeResult", states, method);
779794

‎jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/results/CodSpeedResult.java‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
/*
2+
* Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved.
3+
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4+
*
5+
* This code is free software; you can redistribute it and/or modify it
6+
* under the terms of the GNU General Public License version 2 only, as
7+
* published by the Free Software Foundation. Oracle designates this
8+
* particular file as subject to the "Classpath" exception as provided
9+
* by Oracle in the LICENSE file that accompanied this code.
10+
*
11+
* This code is distributed in the hope that it will be useful, but WITHOUT
12+
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13+
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14+
* version 2 for more details (a copy is included in the LICENSE file that
15+
* accompanied this code).
16+
*
17+
* You should have received a copy of the GNU General Public License version
18+
* 2 along with this work; if not, write to the Free Software Foundation,
19+
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20+
*
21+
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22+
* or visit www.oracle.com if you need additional information or have any
23+
* questions.
24+
*/
25+
package org.openjdk.jmh.results;
26+
27+
import java.util.Collection;
28+
import java.util.concurrent.TimeUnit;
29+
import org.openjdk.jmh.runner.options.TimeValue;
30+
import org.openjdk.jmh.util.ListStatistics;
31+
import org.openjdk.jmh.util.Statistics;
32+
33+
/**
34+
* Result class that stores raw per-iteration ops and duration for CodSpeed walltime collection.
35+
*
36+
* <p>Behaves like {@link AverageTimeResult} for display purposes (shows avg time/op), but
37+
* retains the raw {@code (ops, durationNs)} pair so that {@code CodSpeedResultCollector} can
38+
* read them without floating-point back-calculation.
39+
*/
40+
public class CodSpeedResult extends Result<CodSpeedResult> {
41+
private static final long serialVersionUID = 1L;
42+
43+
private final long rawOps;
44+
private final long rawDurationNs;
45+
46+
public CodSpeedResult(
47+
ResultRole role, String label, double ops, long durationNs, TimeUnit tu) {
48+
this(
49+
role,
50+
label,
51+
of(durationNs / (ops * TimeUnit.NANOSECONDS.convert(1, tu))),
52+
TimeValue.tuToString(tu) + "/op",
53+
Math.round(ops),
54+
durationNs);
55+
}
56+
57+
CodSpeedResult(
58+
ResultRole role,
59+
String label,
60+
Statistics value,
61+
String unit,
62+
long rawOps,
63+
long rawDurationNs) {
64+
super(role, label, value, unit, AggregationPolicy.AVG);
65+
this.rawOps = rawOps;
66+
this.rawDurationNs = rawDurationNs;
67+
}
68+
69+
public long getRawOps() {
70+
return rawOps;
71+
}
72+
73+
public long getRawDurationNs() {
74+
return rawDurationNs;
75+
}
76+
77+
@Override
78+
protected Aggregator<CodSpeedResult> getThreadAggregator() {
79+
return new SummingAggregator();
80+
}
81+
82+
@Override
83+
protected Aggregator<CodSpeedResult> getIterationAggregator() {
84+
return new SummingAggregator();
85+
}
86+
87+
/**
88+
* Sums raw ops and durations across threads/iterations, recomputes the avg score.
89+
*/
90+
static class SummingAggregator implements Aggregator<CodSpeedResult> {
91+
@Override
92+
public CodSpeedResult aggregate(Collection<CodSpeedResult> results) {
93+
long totalOps = 0;
94+
long totalDurationNs = 0;
95+
for (CodSpeedResult r : results) {
96+
totalOps += r.rawOps;
97+
totalDurationNs += r.rawDurationNs;
98+
}
99+
100+
ListStatistics stat = new ListStatistics();
101+
for (CodSpeedResult r : results) {
102+
stat.addValue(r.getScore());
103+
}
104+
105+
return new CodSpeedResult(
106+
AggregatorUtils.aggregateRoles(results),
107+
AggregatorUtils.aggregateLabels(results),
108+
stat,
109+
AggregatorUtils.aggregateUnits(results),
110+
totalOps,
111+
totalDurationNs);
112+
}
113+
}
114+
}

‎jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java‎

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -290,25 +290,27 @@ private Collection<RunResult> internalRun() throws RunnerException {
290290
{
291291
boolean isInstrumented = InstrumentHooks.getInstance().isInstrumented();
292292
List<BenchmarkListEntry> newBenchmarks = new ArrayList<>();
293-
for (BenchmarkListEntry br : benchmarks) {
294-
if (isInstrumented) {
295-
// CodSpeed: normalize any mode to AverageTime
296-
if (br.getMode() != Mode.AverageTime) {
297-
out.println("# CodSpeed: normalizing benchmark mode to AverageTime " +
298-
"(was: " + br.getMode() + ") for " + br.getUsername() + ".");
293+
294+
if (isInstrumented) {
295+
// CodSpeed: override all modes to CodSpeed, deduplicate by benchmark name
296+
// so that multi-mode declarations (e.g. {Throughput, AverageTime}) run once.
297+
Set<String> seen = new LinkedHashSet<>();
298+
for (BenchmarkListEntry br : benchmarks) {
299+
if (seen.add(br.getUsername())) {
300+
newBenchmarks.add(br.cloneWith(Mode.CodSpeed));
299301
}
300-
newBenchmarks.add(br.cloneWith(Mode.AverageTime));
301-
continue;
302302
}
303-
303+
} else {
304304
// Standard JMH: expand Mode.All into all concrete modes
305-
if (br.getMode() == Mode.All) {
305+
for (BenchmarkListEntry br : benchmarks) {
306+
if (br.getMode() != Mode.All) {
307+
newBenchmarks.add(br);
308+
continue;
309+
}
306310
for (Mode m : Mode.values()) {
307-
if (m == Mode.All) continue;
311+
if (m == Mode.All || m == Mode.CodSpeed) continue;
308312
newBenchmarks.add(br.cloneWith(m));
309313
}
310-
} else {
311-
newBenchmarks.add(br);
312314
}
313315
}
314316

0 commit comments

Comments
 (0)