From 38dd0196a2c1c613542eebb8286b2b0538c022db Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 9 Mar 2026 18:28:44 +0100 Subject: [PATCH 01/12] chore: add root frame in JMH --- .../generators/core/BenchmarkGenerator.java | 36 ++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/generators/core/BenchmarkGenerator.java b/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/generators/core/BenchmarkGenerator.java index 8d4f694..60e2372 100644 --- a/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/generators/core/BenchmarkGenerator.java +++ b/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/generators/core/BenchmarkGenerator.java @@ -51,6 +51,7 @@ public class BenchmarkGenerator { private static final String JMH_STUB_SUFFIX = "_jmhStub"; private static final String JMH_TESTCLASS_SUFFIX = "_jmhTest"; protected static final String JMH_GENERATED_SUBPACKAGE = "jmh_generated"; + private static final String CODSPEED_ROOT_FRAME_PREFIX = "__codspeed_root_frame__"; private final Set benchmarkInfos; private final CompilerControlPlugin compilerControl; @@ -104,6 +105,8 @@ public void generate(GeneratorSource source, GeneratorDestination destination) { compilerControl.alwaysDontInline("*", "*_" + mode.shortLabel() + JMH_STUB_SUFFIX); } + compilerControl.alwaysDontInline("*", CODSPEED_ROOT_FRAME_PREFIX + "*"); + compilerControl.process(source, destination); } catch (Throwable t) { destination.printError("Annotation generator had thrown the exception.", t); @@ -474,6 +477,11 @@ private void generateClass(GeneratorDestination destination, ClassInfo classInfo generateMethod(benchmarkKind, writer, info.methodGroup, states); } + // Generate CodSpeed root frame methods (one per benchmark method, shared across all modes) + for (MethodInfo method : info.methodGroup.methods()) { + generateRootFrame(writer, method, states, info.methodGroup.isStrictFP()); + } + // Write out state initializers for (String s : states.getStateInitializers()) { writer.println(ident(1) + s); @@ -659,7 +667,7 @@ private void generateThroughput(PrintWriter writer, Mode benchmarkKind, MethodGr writer.println(ident(2) + "do {"); invocationProlog(writer, 3, method, states, true); - writer.println(ident(3) + emitCall(method, states) + ';'); + writer.println(ident(3) + emitRootFrameCall(method, states) + ';'); invocationEpilog(writer, 3, method, states, true); writer.println(ident(3) + "operations++;"); @@ -792,7 +800,7 @@ private void generateAverageTime(PrintWriter writer, Mode benchmarkKind, MethodG writer.println(ident(2) + "do {"); invocationProlog(writer, 3, method, states, true); - writer.println(ident(3) + emitCall(method, states) + ';'); + writer.println(ident(3) + emitRootFrameCall(method, states) + ';'); invocationEpilog(writer, 3, method, states, true); writer.println(ident(3) + "operations++;"); @@ -815,6 +823,11 @@ private String getStubTypeArgs() { "Blackhole blackhole, Control notifyControl, int startRndMask"; } + /** Args for calling from within a stub (where the parameter is named "result", not "res") */ + private String getStubInternalArgs() { + return "control, result, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask"; + } + private void methodProlog(PrintWriter writer) { // do nothing writer.println(ident(2) + "this.benchmarkParams = control.benchmarkParams;"); @@ -964,7 +977,7 @@ private void generateSampleTime(PrintWriter writer, Mode benchmarkKind, MethodGr writer.println(ident(3) + "for (int b = 0; b < batchSize; b++) {"); writer.println(ident(4) + "if (control.volatileSpoiler) return;"); - writer.println(ident(4) + "" + emitCall(method, states) + ';'); + writer.println(ident(4) + "" + emitRootFrameCall(method, states) + ';'); writer.println(ident(3) + "}"); writer.println(ident(3) + "if (sample) {"); @@ -1062,7 +1075,7 @@ private void generateSingleShotTime(PrintWriter writer, Mode benchmarkKind, Meth invocationProlog(writer, 3, method, states, true); - writer.println(ident(3) + emitCall(method, states) + ';'); + writer.println(ident(3) + emitRootFrameCall(method, states) + ';'); invocationEpilog(writer, 3, method, states, true); @@ -1125,6 +1138,21 @@ private String emitCall(MethodInfo method, StateObjectHandler states) { } } + private String emitRootFrameCall(MethodInfo method, StateObjectHandler states) { + String rootFrameName = CODSPEED_ROOT_FRAME_PREFIX + method.getName(); + return rootFrameName + "(" + getStubInternalArgs() + prefix(states.getArgList(method)) + ")"; + } + + private void generateRootFrame(PrintWriter writer, MethodInfo method, StateObjectHandler states, boolean isStrictFP) { + String rootFrameName = CODSPEED_ROOT_FRAME_PREFIX + method.getName(); + writer.println(ident(1) + "@CompilerControl(CompilerControl.Mode.DONT_INLINE)"); + writer.println(ident(1) + "public static" + (isStrictFP ? " strictfp" : "") + " void " + rootFrameName + "(" + + getStubTypeArgs() + prefix(states.getTypeArgList(method)) + ") throws Throwable {"); + writer.println(ident(2) + emitCall(method, states) + ";"); + writer.println(ident(1) + "}"); + writer.println(); + } + static volatile String[] INDENTS; static final Object INDENTS_LOCK = new Object(); From 95bc197a10fc78e127dca5b5c763bcc28ae95df7 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 12 Mar 2026 10:51:04 +0100 Subject: [PATCH 02/12] fix: use fork pid with SetExecutedBenchmark --- .../jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java b/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java index 19b9577..1b57274 100644 --- a/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java +++ b/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java @@ -727,7 +727,7 @@ private Multimap runSeparate(ActionPlan action InstrumentHooks hooks = InstrumentHooks.getInstance(); hooks.stopBenchmark(); - hooks.setExecutedBenchmark((int) ProcessHandle.current().pid(), params.getBenchmark()); + hooks.setExecutedBenchmark((int) server.getClientPid(), params.getBenchmark()); } catch (IOException e) { results.clear(); From e141cdcfc61f99cf28aee52785aa1be1aa213a55 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 12 Mar 2026 10:55:31 +0100 Subject: [PATCH 03/12] chore: support non-fork benchmarks --- .../main/java/org/openjdk/jmh/runner/BaseRunner.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/BaseRunner.java b/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/BaseRunner.java index 0864155..cc31d8d 100644 --- a/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/BaseRunner.java +++ b/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/BaseRunner.java @@ -32,6 +32,7 @@ import org.openjdk.jmh.results.IterationResult; import org.openjdk.jmh.runner.format.OutputFormat; import org.openjdk.jmh.runner.options.Options; +import io.codspeed.instrument_hooks.InstrumentHooks; import org.openjdk.jmh.util.Multimap; import org.openjdk.jmh.util.TreeMultimap; import org.openjdk.jmh.util.Utils; @@ -79,12 +80,16 @@ protected void runBenchmarksForked(ActionPlan actionPlan, IterationResultAccepto protected Multimap runBenchmarksEmbedded(ActionPlan actionPlan) { Multimap results = new TreeMultimap<>(); + InstrumentHooks hooks = InstrumentHooks.getInstance(); for (Action action : actionPlan.getActions()) { BenchmarkParams params = action.getParams(); ActionMode mode = action.getMode(); long startTime = System.currentTimeMillis(); + if (mode.doMeasurement()) { + hooks.startBenchmark(); + } out.startBenchmark(params); out.println(""); etaBeforeBenchmark(); @@ -118,6 +123,11 @@ public void acceptMeta(BenchmarkResultMetaData md) { BenchmarkResult br = new BenchmarkResult(params, res, md); results.put(params, br); out.endBenchmark(br); + + if (mode.doMeasurement()) { + hooks.stopBenchmark(); + hooks.setExecutedBenchmark((int) ProcessHandle.current().pid(), params.getBenchmark()); + } } etaAfterBenchmark(params); From 2ebbe811d10a183a109f30fb5e282d09227c88ae Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 12 Mar 2026 11:03:02 +0100 Subject: [PATCH 04/12] chore(examples): add regex and TheAlgorithms benchmarks --- .github/workflows/ci.yml | 8 +- .gitmodules | 3 + example-gradle/build.gradle.kts | 11 -- .../src/main/java/bench/FibBenchmark.java | 1 - example-maven/src/main/java/bench/Rle.java | 1 - .../src/main/java/bench/RleBenchmark.java | 1 - .../src/main/java/bench/SleepBenchmark.java | 1 - examples/TheAlgorithms-Java | 1 + examples/example-gradle/build.gradle.kts | 24 +++ .../jmh/java/bench/BacktrackingBenchmark.java | 120 ++++++++++++++ .../java/bench/BitManipulationBenchmark.java | 48 ++++++ .../bench/DynamicProgrammingBenchmark.java | 151 ++++++++++++++++++ .../src/jmh/java/bench/FibBenchmark.java | 4 +- .../src/jmh/java/bench/RegexBenchmark.java | 115 +++++++++++++ .../src/jmh/java/bench/Rle.java | 0 .../src/jmh/java/bench/RleBenchmark.java | 4 +- .../src/jmh/java/bench/SleepBenchmark.java | 4 +- .../thealgorithms/sorts/SortBenchmark.java | 89 +++++++++++ .../example-maven}/pom.xml | 29 ++++ examples/example-maven/src/main/java/bench | 1 + .../java/org/openjdk/jmh/runner/Runner.java | 12 ++ settings.gradle.kts | 2 +- 22 files changed, 604 insertions(+), 26 deletions(-) delete mode 100644 example-gradle/build.gradle.kts delete mode 120000 example-maven/src/main/java/bench/FibBenchmark.java delete mode 120000 example-maven/src/main/java/bench/Rle.java delete mode 120000 example-maven/src/main/java/bench/RleBenchmark.java delete mode 120000 example-maven/src/main/java/bench/SleepBenchmark.java create mode 160000 examples/TheAlgorithms-Java create mode 100644 examples/example-gradle/build.gradle.kts create mode 100644 examples/example-gradle/src/jmh/java/bench/BacktrackingBenchmark.java create mode 100644 examples/example-gradle/src/jmh/java/bench/BitManipulationBenchmark.java create mode 100644 examples/example-gradle/src/jmh/java/bench/DynamicProgrammingBenchmark.java rename {example-gradle => examples/example-gradle}/src/jmh/java/bench/FibBenchmark.java (85%) create mode 100644 examples/example-gradle/src/jmh/java/bench/RegexBenchmark.java rename {example-gradle => examples/example-gradle}/src/jmh/java/bench/Rle.java (100%) rename {example-gradle => examples/example-gradle}/src/jmh/java/bench/RleBenchmark.java (94%) rename {example-gradle => examples/example-gradle}/src/jmh/java/bench/SleepBenchmark.java (91%) create mode 100644 examples/example-gradle/src/jmh/java/com/thealgorithms/sorts/SortBenchmark.java rename {example-maven => examples/example-maven}/pom.xml (71%) create mode 120000 examples/example-maven/src/main/java/bench diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 29f2eb1..83b7250 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,7 +55,7 @@ jobs: distribution: temurin java-version: 21 - name: Run Gradle example benchmarks - run: ./gradlew :example-gradle:jmh + run: ./gradlew :examples:example-gradle:jmh build-and-run-maven: strategy: @@ -77,9 +77,9 @@ jobs: - name: Publish to mavenLocal run: ./gradlew -p jmh-fork publishToMavenLocal - name: Build Maven example - run: cd example-maven && mvn package -q + run: cd examples/example-maven && mvn package -q - name: Run Maven example benchmarks - run: java -jar example-maven/target/example-maven-1.0-SNAPSHOT.jar + run: java -jar examples/example-maven/target/example-maven-1.0-SNAPSHOT.jar walltime-benchmarks: runs-on: codspeed-macro @@ -95,7 +95,7 @@ jobs: uses: CodSpeedHQ/action@main with: mode: walltime - run: ./gradlew :example-gradle:jmh + run: ./gradlew :examples:example-gradle:jmh check: runs-on: ubuntu-latest diff --git a/.gitmodules b/.gitmodules index ad4485f..919894f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "jmh-fork/jmh-core/src/main/java/io/codspeed/instrument_hooks/instrument-hooks"] path = jmh-fork/jmh-core/src/main/java/io/codspeed/instrument_hooks/instrument-hooks url = https://github.com/CodSpeedHQ/instrument-hooks.git +[submodule "examples/TheAlgorithms-Java"] + path = examples/TheAlgorithms-Java + url = https://github.com/TheAlgorithms/Java.git diff --git a/example-gradle/build.gradle.kts b/example-gradle/build.gradle.kts deleted file mode 100644 index a0b3723..0000000 --- a/example-gradle/build.gradle.kts +++ /dev/null @@ -1,11 +0,0 @@ -plugins { - java - id("me.champeau.jmh") version "0.7.2" -} - -group = "io.codspeed" -version = "1.0-SNAPSHOT" - -jmh { - jmhVersion.set("1.37.0-codspeed.1") -} diff --git a/example-maven/src/main/java/bench/FibBenchmark.java b/example-maven/src/main/java/bench/FibBenchmark.java deleted file mode 120000 index ecd7b1b..0000000 --- a/example-maven/src/main/java/bench/FibBenchmark.java +++ /dev/null @@ -1 +0,0 @@ -../../../../../example-gradle/src/jmh/java/bench/FibBenchmark.java \ No newline at end of file diff --git a/example-maven/src/main/java/bench/Rle.java b/example-maven/src/main/java/bench/Rle.java deleted file mode 120000 index d1157ef..0000000 --- a/example-maven/src/main/java/bench/Rle.java +++ /dev/null @@ -1 +0,0 @@ -../../../../../example-gradle/src/jmh/java/bench/Rle.java \ No newline at end of file diff --git a/example-maven/src/main/java/bench/RleBenchmark.java b/example-maven/src/main/java/bench/RleBenchmark.java deleted file mode 120000 index 98e3ca2..0000000 --- a/example-maven/src/main/java/bench/RleBenchmark.java +++ /dev/null @@ -1 +0,0 @@ -../../../../../example-gradle/src/jmh/java/bench/RleBenchmark.java \ No newline at end of file diff --git a/example-maven/src/main/java/bench/SleepBenchmark.java b/example-maven/src/main/java/bench/SleepBenchmark.java deleted file mode 120000 index 42baf17..0000000 --- a/example-maven/src/main/java/bench/SleepBenchmark.java +++ /dev/null @@ -1 +0,0 @@ -../../../../../example-gradle/src/jmh/java/bench/SleepBenchmark.java \ No newline at end of file diff --git a/examples/TheAlgorithms-Java b/examples/TheAlgorithms-Java new file mode 160000 index 0000000..5e06b15 --- /dev/null +++ b/examples/TheAlgorithms-Java @@ -0,0 +1 @@ +Subproject commit 5e06b1592638ac0826258341398f92537717eac3 diff --git a/examples/example-gradle/build.gradle.kts b/examples/example-gradle/build.gradle.kts new file mode 100644 index 0000000..2aff705 --- /dev/null +++ b/examples/example-gradle/build.gradle.kts @@ -0,0 +1,24 @@ +plugins { + java + id("me.champeau.jmh") version "0.7.2" +} + +group = "io.codspeed" +version = "1.0-SNAPSHOT" + +jmh { + jmhVersion.set("1.37.0-codspeed.1") +} + +sourceSets { + named("jmh") { + java { + srcDir("../TheAlgorithms-Java/src/main/java") + } + } +} + +dependencies { + jmh("org.apache.commons:commons-lang3:3.20.0") + jmh("org.apache.commons:commons-collections4:4.5.0") +} diff --git a/examples/example-gradle/src/jmh/java/bench/BacktrackingBenchmark.java b/examples/example-gradle/src/jmh/java/bench/BacktrackingBenchmark.java new file mode 100644 index 0000000..5d75754 --- /dev/null +++ b/examples/example-gradle/src/jmh/java/bench/BacktrackingBenchmark.java @@ -0,0 +1,120 @@ +package bench; + +import com.thealgorithms.backtracking.*; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; + +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@Warmup(iterations = 1, time = 1) +@Measurement(iterations = 3, time = 1) +@Fork(1) +public class BacktrackingBenchmark { + + // -- N-Queens -- + + @State(Scope.Benchmark) + public static class NQueensState { + @Param({"4", "5", "6", "7", "8"}) + public int nQueens; + } + + @Benchmark + public List> nQueensSolver(NQueensState state) { + return NQueens.getNQueensArrangements(state.nQueens); + } + + // -- Parentheses Generation -- + + @State(Scope.Benchmark) + public static class ParenthesesState { + @Param({"3", "4", "5", "6"}) + public int nParens; + } + + @Benchmark + public List generateParentheses(ParenthesesState state) { + return ParenthesesGenerator.generateParentheses(state.nParens); + } + + // -- Combinations -- + + @State(Scope.Benchmark) + public static class CombinationsState { + @Param({"5", "6", "7", "8", "9"}) + public int nCombinations; + } + + @Benchmark + public void generateCombinations(CombinationsState state, Blackhole bh) { + Integer[] arr = new Integer[state.nCombinations]; + for (int i = 0; i < state.nCombinations; i++) { + arr[i] = i; + } + bh.consume(Combination.combination(arr, 3)); + } + + // -- Permutations -- + + @State(Scope.Benchmark) + public static class PermutationsState { + @Param({"3", "4", "5", "6", "7"}) + public int nPermutations; + } + + @Benchmark + public void permutations(PermutationsState state, Blackhole bh) { + Integer[] arr = new Integer[state.nPermutations]; + for (int i = 0; i < state.nPermutations; i++) { + arr[i] = i; + } + bh.consume(Permutation.permutation(arr)); + } + + // -- Sudoku Solver -- + + @State(Scope.Benchmark) + public static class SudokuState { + public int[][] sudokuBoard; + + @Setup(Level.Invocation) + public void setupSudoku() { + sudokuBoard = + new int[][] { + {3, 0, 6, 5, 0, 8, 4, 0, 0}, + {5, 2, 0, 0, 0, 0, 0, 0, 0}, + {0, 8, 7, 0, 0, 0, 0, 3, 1}, + {0, 0, 3, 0, 1, 0, 0, 8, 0}, + {9, 0, 0, 8, 6, 3, 0, 0, 5}, + {0, 5, 0, 0, 9, 0, 6, 0, 0}, + {1, 3, 0, 0, 0, 0, 2, 5, 0}, + {0, 0, 0, 0, 0, 0, 0, 7, 4}, + {0, 0, 5, 2, 0, 6, 3, 0, 0}, + }; + } + } + + @Benchmark + public boolean sudokuSolver(SudokuState state) { + return SudokuSolver.solveSudoku(state.sudokuBoard); + } + + // -- Subsequences -- + + @State(Scope.Benchmark) + public static class SubsequencesState { + @Param({"8", "10", "12"}) + public int nSubsequences; + } + + @Benchmark + public void generateSubsequences(SubsequencesState state, Blackhole bh) { + List seq = new java.util.ArrayList<>(); + for (int i = 0; i < state.nSubsequences; i++) { + seq.add(i); + } + bh.consume(SubsequenceFinder.generateAll(seq)); + } +} diff --git a/examples/example-gradle/src/jmh/java/bench/BitManipulationBenchmark.java b/examples/example-gradle/src/jmh/java/bench/BitManipulationBenchmark.java new file mode 100644 index 0000000..fc45b71 --- /dev/null +++ b/examples/example-gradle/src/jmh/java/bench/BitManipulationBenchmark.java @@ -0,0 +1,48 @@ +package bench; + +import com.thealgorithms.bitmanipulation.*; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.*; + +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Benchmark) +@Warmup(iterations = 1, time = 1) +@Measurement(iterations = 3, time = 1) +@Fork(1) +public class BitManipulationBenchmark { + + @Param({"0", "42", "255", "1024", "65535"}) + private int bitValue; + + @Benchmark + public int countSetBits() { + return CountSetBits.countSetBits(bitValue); + } + + @Benchmark + public Optional findHighestSetBit() { + return HighestSetBit.findHighestSetBit(bitValue); + } + + @Benchmark + public int binaryToGray() { + return GrayCodeConversion.binaryToGray(bitValue); + } + + @Benchmark + public int grayToBinary() { + return GrayCodeConversion.grayToBinary(bitValue); + } + + @Benchmark + public int reverseBits() { + return ReverseBits.reverseBits(bitValue); + } + + @Benchmark + public boolean isPowerOfTwo() { + return IsPowerTwo.isPowerTwo(bitValue); + } +} diff --git a/examples/example-gradle/src/jmh/java/bench/DynamicProgrammingBenchmark.java b/examples/example-gradle/src/jmh/java/bench/DynamicProgrammingBenchmark.java new file mode 100644 index 0000000..8f9b704 --- /dev/null +++ b/examples/example-gradle/src/jmh/java/bench/DynamicProgrammingBenchmark.java @@ -0,0 +1,151 @@ +package bench; + +import com.thealgorithms.dynamicprogramming.*; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.*; + +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 1, time = 1) +@Measurement(iterations = 3, time = 1) +@Fork(1) +public class DynamicProgrammingBenchmark { + + // -- Fibonacci -- + + @State(Scope.Benchmark) + public static class FibonacciState { + @Param({"10", "20", "30", "40"}) + public int fibN; + } + + @Benchmark + public int fibonacciMemo(FibonacciState state) { + return Fibonacci.fibMemo(state.fibN); + } + + @Benchmark + public int fibonacciBottomUp(FibonacciState state) { + return Fibonacci.fibBotUp(state.fibN); + } + + @Benchmark + public int fibonacciOptimized(FibonacciState state) { + return Fibonacci.fibOptimized(state.fibN); + } + + // -- Knapsack -- + + @State(Scope.Benchmark) + public static class KnapsackState { + @Param({"10", "15", "20"}) + public int knapsackSize; + + public int[] knapsackWeights; + public int[] knapsackValues; + + @Setup(Level.Trial) + public void setupKnapsack() { + knapsackWeights = new int[knapsackSize]; + knapsackValues = new int[knapsackSize]; + for (int i = 0; i < knapsackSize; i++) { + knapsackWeights[i] = (i % 5) + 1; + knapsackValues[i] = (i % 10) + 1; + } + } + } + + @Benchmark + public int knapsack(KnapsackState state) { + return Knapsack.knapSack(state.knapsackSize * 2, state.knapsackWeights, state.knapsackValues); + } + + // -- Edit Distance -- + + @State(Scope.Benchmark) + public static class EditDistanceState { + @Param({"kitten", "saturday"}) + public String editWord1; + } + + @Benchmark + public int editDistance(EditDistanceState state) { + return EditDistance.minDistance(state.editWord1, "sitting"); + } + + // -- Levenshtein Distance -- + + @State(Scope.Benchmark) + public static class LevenshteinState { + @Param({"kitten sitting", "saturday sunday"}) + public String levenshteinPair; + } + + @Benchmark + public int levenshteinDistance(LevenshteinState state) { + String[] parts = state.levenshteinPair.split(" "); + return LevenshteinDistance.optimizedLevenshteinDistance(parts[0], parts[1]); + } + + // -- Longest Increasing Subsequence -- + + @State(Scope.Benchmark) + public static class LisState { + @Param({"10", "50", "100"}) + public int lisSize; + + public int[] lisArray; + + @Setup(Level.Trial) + public void setupLIS() { + lisArray = new int[lisSize]; + // Semi-random sequence with some increasing runs + for (int i = 0; i < lisSize; i++) { + lisArray[i] = (i * 7 + 3) % (lisSize * 2); + } + } + } + + @Benchmark + public int longestIncreasingSubsequence(LisState state) { + return LongestIncreasingSubsequence.lis(state.lisArray); + } + + // -- Coin Change -- + + @State(Scope.Benchmark) + public static class CoinChangeState { + @Param({"50", "100", "200"}) + public int coinAmount; + } + + private static final int[] COINS = {1, 5, 10, 25}; + + @Benchmark + public int coinChange(CoinChangeState state) { + return CoinChange.minimumCoins(COINS, state.coinAmount); + } + + // -- Subset Sum -- + + @State(Scope.Benchmark) + public static class SubsetSumState { + @Param({"10", "15", "20"}) + public int subsetSize; + + public int[] subsetArr; + + @Setup(Level.Trial) + public void setupSubsetSum() { + subsetArr = new int[subsetSize]; + for (int i = 0; i < subsetSize; i++) { + subsetArr[i] = i + 1; + } + } + } + + @Benchmark + public boolean subsetSum(SubsetSumState state) { + return SubsetSum.subsetSum(state.subsetArr, state.subsetSize * 2); + } +} diff --git a/example-gradle/src/jmh/java/bench/FibBenchmark.java b/examples/example-gradle/src/jmh/java/bench/FibBenchmark.java similarity index 85% rename from example-gradle/src/jmh/java/bench/FibBenchmark.java rename to examples/example-gradle/src/jmh/java/bench/FibBenchmark.java index f5ab4cf..272fed6 100644 --- a/example-gradle/src/jmh/java/bench/FibBenchmark.java +++ b/examples/example-gradle/src/jmh/java/bench/FibBenchmark.java @@ -6,8 +6,8 @@ @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS) @State(Scope.Benchmark) -@Warmup(iterations = 3, time = 1) -@Measurement(iterations = 5, time = 1) +@Warmup(iterations = 1, time = 1) +@Measurement(iterations = 3, time = 1) @Fork(1) public class FibBenchmark { diff --git a/examples/example-gradle/src/jmh/java/bench/RegexBenchmark.java b/examples/example-gradle/src/jmh/java/bench/RegexBenchmark.java new file mode 100644 index 0000000..3adc366 --- /dev/null +++ b/examples/example-gradle/src/jmh/java/bench/RegexBenchmark.java @@ -0,0 +1,115 @@ +package bench; + +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; + +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@State(Scope.Benchmark) +@Warmup(iterations = 1, time = 1) +@Measurement(iterations = 3, time = 1) +@Fork(1) +public class RegexBenchmark { + + // Complex pattern with alternations, named groups, lookahead, and quantifiers. + // Matches things like email addresses, URLs, ISO dates, and IPv4 addresses. + private static final String COMPLEX_PATTERN = + "(?[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,})" + + "|(?https?://[a-zA-Z0-9.-]+(?:/[a-zA-Z0-9_.~!$&'()*+,;=:@/-]*)*(?:\\?[a-zA-Z0-9_.~!$&'()*+,;=:@/?-]*)?)" + + "|(?\\d{4}-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12]\\d|3[01])T(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?Z?)" + + "|(?(?:25[0-5]|2[0-4]\\d|[01]?\\d\\d?)\\.(?:25[0-5]|2[0-4]\\d|[01]?\\d\\d?)\\.(?:25[0-5]|2[0-4]\\d|[01]?\\d\\d?)\\.(?:25[0-5]|2[0-4]\\d|[01]?\\d\\d?))"; + + // Pattern that causes significant backtracking on near-miss inputs. + // Nested quantifiers: (a+)+ applied to a string of 'a's followed by a non-matching char. + private static final String BACKTRACK_PATTERN = "^(a+)+b$"; + + @Param({"20", "24"}) + private int backtrackLength; + + private String scanInput; + private String backtrackInput; + private Pattern[] scanPatterns; + + @Setup + public void setup() { + // Build a realistic mixed-content text for scanning + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 50; i++) { + sb.append("Contact user").append(i).append("@example.com for details. "); + sb.append("Visit https://docs.example.com/api/v2/resource?page=") + .append(i) + .append("&limit=100 "); + sb.append("Created at 2025-06-") + .append(String.format("%02d", (i % 28) + 1)) + .append("T14:30:00.000Z "); + sb.append("Server at 192.168.") + .append(i % 256) + .append(".") + .append((i * 7) % 256) + .append(" responded. "); + sb.append("Some filler text with no matches here to keep the scanner busy. "); + } + scanInput = sb.toString(); + + // Near-miss input for backtracking: all 'a's with no trailing 'b' + backtrackInput = "a".repeat(backtrackLength) + "c"; + + // Multiple patterns for the tokenizer-style scan + scanPatterns = + new Pattern[] { + Pattern.compile("(?[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,})"), + Pattern.compile( + "(?https?://[a-zA-Z0-9.-]+(?:/[a-zA-Z0-9_.~!$&'()*+,;=:@/-]*)*(?:\\?[a-zA-Z0-9_.~!$&'()*+,;=:@/?-]*)?)"), + Pattern.compile( + "\\d{4}-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12]\\d|3[01])T(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?Z?"), + Pattern.compile( + "(?:25[0-5]|2[0-4]\\d|[01]?\\d\\d?)\\.(?:25[0-5]|2[0-4]\\d|[01]?\\d\\d?)\\.(?:25[0-5]|2[0-4]\\d|[01]?\\d\\d?)\\.(?:25[0-5]|2[0-4]\\d|[01]?\\d\\d?)"), + }; + } + + /** + * Compiles a complex pattern from scratch and matches against the full input. Forces the regex + * compiler path every iteration for deep compilation frames. + */ + @Benchmark + public int compileAndMatch() { + Pattern p = Pattern.compile(COMPLEX_PATTERN); + Matcher m = p.matcher(scanInput); + int count = 0; + while (m.find()) { + count++; + } + return count; + } + + /** + * Exercises catastrophic backtracking with nested quantifiers on a near-miss input. Produces deep + * recursive matching stacks. + */ + @Benchmark + public boolean backtrackHeavy() { + Pattern p = Pattern.compile(BACKTRACK_PATTERN); + return p.matcher(backtrackInput).matches(); + } + + /** + * Applies multiple pre-compiled patterns in sequence over a large text, simulating a + * tokenizer/lexer. Exercises repeated Matcher.find() loops and produces varied frames across + * different pattern types. + */ + @Benchmark + public void multiPatternScan(Blackhole bh) { + for (Pattern pattern : scanPatterns) { + Matcher m = pattern.matcher(scanInput); + int count = 0; + while (m.find()) { + bh.consume(m.group()); + count++; + } + bh.consume(count); + } + } +} diff --git a/example-gradle/src/jmh/java/bench/Rle.java b/examples/example-gradle/src/jmh/java/bench/Rle.java similarity index 100% rename from example-gradle/src/jmh/java/bench/Rle.java rename to examples/example-gradle/src/jmh/java/bench/Rle.java diff --git a/example-gradle/src/jmh/java/bench/RleBenchmark.java b/examples/example-gradle/src/jmh/java/bench/RleBenchmark.java similarity index 94% rename from example-gradle/src/jmh/java/bench/RleBenchmark.java rename to examples/example-gradle/src/jmh/java/bench/RleBenchmark.java index 9d982bf..eb44f38 100644 --- a/example-gradle/src/jmh/java/bench/RleBenchmark.java +++ b/examples/example-gradle/src/jmh/java/bench/RleBenchmark.java @@ -9,8 +9,8 @@ @BenchmarkMode(Mode.Throughput) @OutputTimeUnit(TimeUnit.MILLISECONDS) @State(Scope.Benchmark) -@Warmup(iterations = 3, time = 1) -@Measurement(iterations = 5, time = 1) +@Warmup(iterations = 1, time = 1) +@Measurement(iterations = 3, time = 1) @Fork(1) public class RleBenchmark { diff --git a/example-gradle/src/jmh/java/bench/SleepBenchmark.java b/examples/example-gradle/src/jmh/java/bench/SleepBenchmark.java similarity index 91% rename from example-gradle/src/jmh/java/bench/SleepBenchmark.java rename to examples/example-gradle/src/jmh/java/bench/SleepBenchmark.java index cd24234..2a8ef82 100644 --- a/example-gradle/src/jmh/java/bench/SleepBenchmark.java +++ b/examples/example-gradle/src/jmh/java/bench/SleepBenchmark.java @@ -6,8 +6,8 @@ @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS) @State(Scope.Benchmark) -@Warmup(iterations = 3, time = 1) -@Measurement(iterations = 5, time = 1) +@Warmup(iterations = 1, time = 1) +@Measurement(iterations = 3, time = 1) @Fork(1) public class SleepBenchmark { diff --git a/examples/example-gradle/src/jmh/java/com/thealgorithms/sorts/SortBenchmark.java b/examples/example-gradle/src/jmh/java/com/thealgorithms/sorts/SortBenchmark.java new file mode 100644 index 0000000..a942f4a --- /dev/null +++ b/examples/example-gradle/src/jmh/java/com/thealgorithms/sorts/SortBenchmark.java @@ -0,0 +1,89 @@ +package com.thealgorithms.sorts; + +import java.util.Arrays; +import java.util.Random; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.*; + +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@State(Scope.Benchmark) +@Warmup(iterations = 1, time = 1) +@Measurement(iterations = 3, time = 1) +@Fork(1) +public class SortBenchmark { + + @Param({"100", "1000", "10000"}) + private int size; + + private Integer[] data; + + private final QuickSort quickSort = new QuickSort(); + private final MergeSort mergeSort = new MergeSort(); + private final HeapSort heapSort = new HeapSort(); + private final InsertionSort insertionSort = new InsertionSort(); + private final BubbleSort bubbleSort = new BubbleSort(); + private final TimSort timSort = new TimSort(); + private final ShellSort shellSort = new ShellSort(); + private final SelectionSort selectionSort = new SelectionSort(); + private final DualPivotQuickSort dualPivotQuickSort = new DualPivotQuickSort(); + private final IntrospectiveSort introspectiveSort = new IntrospectiveSort(); + + @Setup(Level.Trial) + public void setup() { + Random rng = new Random(42); + data = new Integer[size]; + for (int i = 0; i < size; i++) { + data[i] = rng.nextInt(size * 10); + } + } + + private Integer[] copyData() { + return Arrays.copyOf(data, data.length); + } + + @Benchmark + public Integer[] quickSort() { + return quickSort.sort(copyData()); + } + + @Benchmark + public Integer[] mergeSort() { + return mergeSort.sort(copyData()); + } + + @Benchmark + public Integer[] heapSort() { + return heapSort.sort(copyData()); + } + + @Benchmark + public Integer[] timSort() { + return timSort.sort(copyData()); + } + + @Benchmark + public Integer[] shellSort() { + return shellSort.sort(copyData()); + } + + @Benchmark + public Integer[] selectionSort() { + return selectionSort.sort(copyData()); + } + + @Benchmark + public Integer[] insertionSort() { + return insertionSort.sort(copyData()); + } + + @Benchmark + public Integer[] dualPivotQuickSort() { + return dualPivotQuickSort.sort(copyData()); + } + + @Benchmark + public Integer[] introspectiveSort() { + return introspectiveSort.sort(copyData()); + } +} diff --git a/example-maven/pom.xml b/examples/example-maven/pom.xml similarity index 71% rename from example-maven/pom.xml rename to examples/example-maven/pom.xml index 1212bea..60509c1 100644 --- a/example-maven/pom.xml +++ b/examples/example-maven/pom.xml @@ -27,6 +27,16 @@ ${jmh.version} provided + + org.apache.commons + commons-lang3 + 3.20.0 + + + org.apache.commons + commons-collections4 + 4.5.0 + @@ -40,6 +50,25 @@ 21 + + org.codehaus.mojo + build-helper-maven-plugin + 3.6.0 + + + add-thealgorithms-sources + generate-sources + + add-source + + + + ../TheAlgorithms-Java/src/main/java + + + + + org.apache.maven.plugins maven-jar-plugin diff --git a/examples/example-maven/src/main/java/bench b/examples/example-maven/src/main/java/bench new file mode 120000 index 0000000..92566e4 --- /dev/null +++ b/examples/example-maven/src/main/java/bench @@ -0,0 +1 @@ +../../../../example-gradle/src/jmh/java/bench \ No newline at end of file diff --git a/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java b/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java index 1b57274..785bc38 100644 --- a/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java +++ b/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java @@ -68,6 +68,7 @@ public class Runner extends BaseRunner { private static final int TAIL_LINES_ON_ERROR = Integer.getInteger("jmh.tailLines", 20); private static final String JMH_LOCK_FILE = System.getProperty("java.io.tmpdir") + "/jmh.lock"; private static final Boolean JMH_LOCK_IGNORE = Boolean.getBoolean("jmh.ignoreLock"); + private static boolean codspeedForksWarningShown = false; private final BenchmarkList list; private int cpuCount; @@ -465,6 +466,17 @@ private BenchmarkParams newBenchmarkParams(BenchmarkListEntry benchmark, ActionM benchmark.getWarmupForks().orElse( Defaults.WARMUP_FORKS)); + // TODO(COD-1788): Remove this after process filtering is supported + if (InstrumentHooks.getInstance().isInstrumented() && !"1".equals(System.getenv("CODSPEED_JVM_ALLOW_FORKS"))) { + if ((forks > 0 || warmupForks > 0) && !codspeedForksWarningShown) { + out.println("# CodSpeed: forcing forks=0 to enable flamegraph support."); + out.println("# Set CODSPEED_JVM_ALLOW_FORKS=1 to disable this override."); + codspeedForksWarningShown = true; + } + forks = 0; + warmupForks = 0; + } + TimeUnit timeUnit = options.getTimeUnit().orElse( benchmark.getTimeUnit().orElse( Defaults.OUTPUT_TIMEUNIT)); diff --git a/settings.gradle.kts b/settings.gradle.kts index 0d6d99f..b8a4adf 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -1,4 +1,4 @@ rootProject.name = "codspeed-jvm" -include("example-gradle") +include("examples:example-gradle") includeBuild("jmh-fork") From 4265566bfec284e26df4770356da6236a48a3da0 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 12 Mar 2026 14:59:58 +0100 Subject: [PATCH 05/12] fix: support files and parameters in URI --- jmh-fork/jmh-core/build.gradle.kts | 1 + .../main/java/io/codspeed/BenchmarkUri.java | 154 ++++++++++++++++++ .../result/CodSpeedResultCollector.java | 20 +-- .../org/openjdk/jmh/runner/BaseRunner.java | 3 +- .../java/org/openjdk/jmh/runner/Runner.java | 3 +- .../java/io/codspeed/BenchmarkUriTest.java | 100 ++++++++++++ 6 files changed, 267 insertions(+), 14 deletions(-) create mode 100644 jmh-fork/jmh-core/src/main/java/io/codspeed/BenchmarkUri.java create mode 100644 jmh-fork/jmh-core/src/test/java/io/codspeed/BenchmarkUriTest.java diff --git a/jmh-fork/jmh-core/build.gradle.kts b/jmh-fork/jmh-core/build.gradle.kts index 7e9a940..01e0e9d 100644 --- a/jmh-fork/jmh-core/build.gradle.kts +++ b/jmh-fork/jmh-core/build.gradle.kts @@ -12,6 +12,7 @@ dependencies { api("net.sf.jopt-simple:jopt-simple:5.0.4") api("org.apache.commons:commons-math3:3.6.1") implementation("com.google.code.gson:gson:2.11.0") + testImplementation("junit:junit:4.13.2") } tasks.compileJava { diff --git a/jmh-fork/jmh-core/src/main/java/io/codspeed/BenchmarkUri.java b/jmh-fork/jmh-core/src/main/java/io/codspeed/BenchmarkUri.java new file mode 100644 index 0000000..869c783 --- /dev/null +++ b/jmh-fork/jmh-core/src/main/java/io/codspeed/BenchmarkUri.java @@ -0,0 +1,154 @@ +package io.codspeed; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.Collection; +import java.util.concurrent.ConcurrentHashMap; +import org.openjdk.jmh.infra.BenchmarkParams; + +/** Builds CodSpeed benchmark URIs in the format: {file_path}::{classQName}::{method}[{params}] */ +public class BenchmarkUri { + + private static volatile Path cachedGitRoot; + private static final ConcurrentHashMap sourceFileCache = + new ConcurrentHashMap<>(); + + /** + * Builds a CodSpeed URI from the given benchmark params. + * + * @param params the JMH benchmark params + * @return the URI string + */ + public static String fromBenchmarkParams(BenchmarkParams params) { + String benchmarkName = params.getBenchmark(); + int lastDot = benchmarkName.lastIndexOf('.'); + if (lastDot == -1) { + return benchmarkName; + } + String classQName = benchmarkName.substring(0, lastDot); + String method = benchmarkName.substring(lastDot + 1); + + String filePath = resolveSourceFile(classQName); + String benchName = buildBenchName(method, params); + + return filePath + "::" + classQName + "::" + benchName; + } + + static String buildBenchName(String method, BenchmarkParams params) { + Collection keys = params.getParamsKeys(); + if (keys == null || keys.isEmpty()) { + return method; + } + + StringBuilder sb = new StringBuilder(method); + sb.append('['); + boolean first = true; + for (String key : keys) { + if (!first) { + sb.append(", "); + } + sb.append(params.getParam(key)); + first = false; + } + sb.append(']'); + return sb.toString(); + } + + /** + * Resolves the source file path relative to the git root for a given fully qualified class name. + * Searches from the git root for a .java file matching the class's package structure. + * + *

Falls back to the package-derived relative path if the file can't be found on disk. + */ + static String resolveSourceFile(String classQName) { + return sourceFileCache.computeIfAbsent(classQName, BenchmarkUri::resolveSourceFileUncached); + } + + private static String resolveSourceFileUncached(String classQName) { + // Handle inner classes: com.example.Outer$Inner -> com.example.Outer + String outerClass = classQName; + int dollarIdx = outerClass.indexOf('$'); + if (dollarIdx != -1) { + outerClass = outerClass.substring(0, dollarIdx); + } + + String relativePath = outerClass.replace('.', '/') + ".java"; + Path gitRoot = findGitRoot(); + + Path found = findFile(gitRoot, relativePath); + if (found != null) { + // Normalize to forward slashes for consistent URIs across platforms + return gitRoot.relativize(found).toString().replace('\\', '/'); + } + + return relativePath; + } + + /** Walks up from the CWD to find the nearest .git directory, returns its parent. */ + static Path findGitRoot() { + Path cached = cachedGitRoot; + if (cached != null) { + return cached; + } + + Path current = Paths.get(System.getProperty("user.dir")).toAbsolutePath().normalize(); + while (current != null) { + if (Files.isDirectory(current.resolve(".git"))) { + cachedGitRoot = current; + return current; + } + current = current.getParent(); + } + // Fall back to CWD if no git root found + Path fallback = Paths.get(System.getProperty("user.dir")).toAbsolutePath().normalize(); + cachedGitRoot = fallback; + return fallback; + } + + private static Path findFile(Path root, String relativeSuffix) { + String suffix = "/" + relativeSuffix; + Path[] result = new Path[1]; + + try { + Files.walkFileTree( + root, + new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + if (file.toString().endsWith(suffix) || file.equals(root.resolve(relativeSuffix))) { + result[0] = file; + return FileVisitResult.TERMINATE; + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) { + String dirName = dir.getFileName() != null ? dir.getFileName().toString() : ""; + // Skip hidden dirs, build outputs, and VCS dirs + if (dirName.startsWith(".") + || dirName.equals("build") + || dirName.equals("target") + || dirName.equals("node_modules")) { + return FileVisitResult.SKIP_SUBTREE; + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException exc) { + return FileVisitResult.CONTINUE; + } + }); + } catch (IOException e) { + // Fall through to return null + } + + return result[0]; + } +} diff --git a/jmh-fork/jmh-core/src/main/java/io/codspeed/result/CodSpeedResultCollector.java b/jmh-fork/jmh-core/src/main/java/io/codspeed/result/CodSpeedResultCollector.java index 8a4b7ca..3f9dc17 100644 --- a/jmh-fork/jmh-core/src/main/java/io/codspeed/result/CodSpeedResultCollector.java +++ b/jmh-fork/jmh-core/src/main/java/io/codspeed/result/CodSpeedResultCollector.java @@ -2,6 +2,7 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import io.codspeed.BenchmarkUri; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -33,12 +34,13 @@ public static void collectAndWrite( Mode mode = params.getMode(); TimeUnit timeUnit = params.getTimeUnit(); String benchmarkName = params.getBenchmark(); + String uri = BenchmarkUri.fromBenchmarkParams(params); double nanosPerUnit = TimeUnit.NANOSECONDS.convert(1, timeUnit); if (mode == Mode.SampleTime) { - collectSampleTime(runResult, benchmarkName, nanosPerUnit, benchmarks); + collectSampleTime(runResult, benchmarkName, uri, nanosPerUnit, benchmarks); } else { - collectIterationBased(runResult, benchmarkName, mode, nanosPerUnit, benchmarks); + collectIterationBased(runResult, benchmarkName, uri, mode, nanosPerUnit, benchmarks); } } @@ -70,6 +72,7 @@ public static void collectAndWrite( private static void collectIterationBased( RunResult runResult, String benchmarkName, + String uri, Mode mode, double nanosPerUnit, List benchmarks) { @@ -101,11 +104,7 @@ private static void collectIterationBased( benchmarks.add( WalltimeBenchmark.fromRuntimeData( - benchmarkName, - benchmarkName, - toLongArray(itersPerRound), - toLongArray(timesPerRoundNs), - null)); + benchmarkName, uri, toLongArray(itersPerRound), toLongArray(timesPerRoundNs), null)); } /** @@ -118,6 +117,7 @@ private static void collectIterationBased( private static void collectSampleTime( RunResult runResult, String benchmarkName, + String uri, double nanosPerUnit, List benchmarks) { List itersPerRound = new ArrayList<>(); @@ -151,11 +151,7 @@ private static void collectSampleTime( benchmarks.add( WalltimeBenchmark.fromRuntimeData( - benchmarkName, - benchmarkName, - toLongArray(itersPerRound), - toLongArray(timesPerRoundNs), - null)); + benchmarkName, uri, toLongArray(itersPerRound), toLongArray(timesPerRoundNs), null)); } private static long[] toLongArray(List list) { diff --git a/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/BaseRunner.java b/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/BaseRunner.java index cc31d8d..03adf5b 100644 --- a/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/BaseRunner.java +++ b/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/BaseRunner.java @@ -32,6 +32,7 @@ import org.openjdk.jmh.results.IterationResult; import org.openjdk.jmh.runner.format.OutputFormat; import org.openjdk.jmh.runner.options.Options; +import io.codspeed.BenchmarkUri; import io.codspeed.instrument_hooks.InstrumentHooks; import org.openjdk.jmh.util.Multimap; import org.openjdk.jmh.util.TreeMultimap; @@ -126,7 +127,7 @@ public void acceptMeta(BenchmarkResultMetaData md) { if (mode.doMeasurement()) { hooks.stopBenchmark(); - hooks.setExecutedBenchmark((int) ProcessHandle.current().pid(), params.getBenchmark()); + hooks.setExecutedBenchmark((int) ProcessHandle.current().pid(), BenchmarkUri.fromBenchmarkParams(params)); } } diff --git a/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java b/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java index 785bc38..542b820 100644 --- a/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java +++ b/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java @@ -40,6 +40,7 @@ import org.openjdk.jmh.util.*; import org.openjdk.jmh.util.Optional; +import io.codspeed.BenchmarkUri; import io.codspeed.instrument_hooks.InstrumentHooks; import io.codspeed.result.CodSpeedResultCollector; @@ -739,7 +740,7 @@ private Multimap runSeparate(ActionPlan action InstrumentHooks hooks = InstrumentHooks.getInstance(); hooks.stopBenchmark(); - hooks.setExecutedBenchmark((int) server.getClientPid(), params.getBenchmark()); + hooks.setExecutedBenchmark((int) server.getClientPid(), BenchmarkUri.fromBenchmarkParams(params)); } catch (IOException e) { results.clear(); diff --git a/jmh-fork/jmh-core/src/test/java/io/codspeed/BenchmarkUriTest.java b/jmh-fork/jmh-core/src/test/java/io/codspeed/BenchmarkUriTest.java new file mode 100644 index 0000000..c0e0c35 --- /dev/null +++ b/jmh-fork/jmh-core/src/test/java/io/codspeed/BenchmarkUriTest.java @@ -0,0 +1,100 @@ +package io.codspeed; + +import static org.junit.Assert.assertEquals; + +import java.util.Collections; +import org.junit.Test; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.infra.BenchmarkParams; +import org.openjdk.jmh.infra.IterationParams; +import org.openjdk.jmh.runner.IterationType; +import org.openjdk.jmh.runner.WorkloadParams; +import org.openjdk.jmh.runner.options.TimeValue; + +public class BenchmarkUriTest { + + private static BenchmarkParams makeParams(String benchmark, WorkloadParams workloadParams) { + return new BenchmarkParams( + benchmark, + "generated.Target", + false, + 1, + new int[] {1}, + Collections.emptyList(), + 1, + 0, + new IterationParams(IterationType.WARMUP, 1, TimeValue.seconds(1), 1), + new IterationParams(IterationType.MEASUREMENT, 1, TimeValue.seconds(1), 1), + Mode.Throughput, + workloadParams, + java.util.concurrent.TimeUnit.SECONDS, + 1, + "java", + Collections.emptyList(), + "17", + "HotSpot", + "17.0.1", + "1.0", + TimeValue.seconds(10)); + } + + @Test + public void testBuildBenchNameNoParams() { + WorkloadParams wp = new WorkloadParams(); + BenchmarkParams params = makeParams("com.example.MyBenchmark.testMethod", wp); + String benchName = BenchmarkUri.buildBenchName("testMethod", params); + assertEquals("testMethod", benchName); + } + + @Test + public void testBuildBenchNameSingleParam() { + WorkloadParams wp = new WorkloadParams(); + wp.put("size", "1024", 0); + BenchmarkParams params = makeParams("com.example.MyBenchmark.testMethod", wp); + String benchName = BenchmarkUri.buildBenchName("testMethod", params); + assertEquals("testMethod[1024]", benchName); + } + + @Test + public void testBuildBenchNameMultipleParams() { + WorkloadParams wp = new WorkloadParams(); + wp.put("size", "1024", 0); + wp.put("mode", "fast", 0); + BenchmarkParams params = makeParams("com.example.MyBenchmark.testMethod", wp); + String benchName = BenchmarkUri.buildBenchName("testMethod", params); + // WorkloadParams uses TreeMap, so keys are sorted alphabetically: mode < size + assertEquals("testMethod[fast, 1024]", benchName); + } + + @Test + public void testResolveSourceFileFallback() { + // Class that doesn't exist on disk — should fall back to derived path + String path = BenchmarkUri.resolveSourceFile("com.nonexistent.FakeClass"); + assertEquals("com/nonexistent/FakeClass.java", path); + } + + @Test + public void testResolveSourceFileInnerClass() { + // Inner class should strip the $Inner part + String path = BenchmarkUri.resolveSourceFile("com.nonexistent.Outer$Inner"); + assertEquals("com/nonexistent/Outer.java", path); + } + + @Test + public void testFullUriNoParams() { + WorkloadParams wp = new WorkloadParams(); + BenchmarkParams params = makeParams("com.nonexistent.MyBenchmark.testMethod", wp); + String uri = BenchmarkUri.fromBenchmarkParams(params); + assertEquals("com/nonexistent/MyBenchmark.java::com.nonexistent.MyBenchmark::testMethod", uri); + } + + @Test + public void testFullUriWithParams() { + WorkloadParams wp = new WorkloadParams(); + wp.put("size", "65536", 0); + BenchmarkParams params = makeParams("com.nonexistent.MyBenchmark.encode", wp); + String uri = BenchmarkUri.fromBenchmarkParams(params); + assertEquals( + "com/nonexistent/MyBenchmark.java::com.nonexistent.MyBenchmark::encode[65536]", uri); + } +} From 19e1cbdaae14bcd8f2523f93d05365f4b7020c7b Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 12 Mar 2026 15:59:33 +0100 Subject: [PATCH 06/12] ci: enable java caching --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 83b7250..c3781cd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,7 @@ jobs: with: distribution: temurin java-version: 21 + cache: 'maven' - name: Setup Maven uses: stCarolas/setup-maven@v5 with: @@ -54,6 +55,7 @@ jobs: with: distribution: temurin java-version: 21 + cache: 'gradle' - name: Run Gradle example benchmarks run: ./gradlew :examples:example-gradle:jmh @@ -70,6 +72,7 @@ jobs: with: distribution: temurin java-version: 21 + cache: 'maven' - name: Setup Maven uses: stCarolas/setup-maven@v5 with: @@ -91,6 +94,7 @@ jobs: with: distribution: temurin java-version: 21 + cache: 'gradle' - name: Run the benchmarks uses: CodSpeedHQ/action@main with: From 29ab64fb2460a9a416f0e338fe15adb7dfd13fc4 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 13 Mar 2026 19:48:35 +0100 Subject: [PATCH 07/12] docs: add AGENTS.md --- AGENTS.md | 131 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ CLAUDE.md | 1 + 2 files changed, 132 insertions(+) create mode 100644 AGENTS.md create mode 120000 CLAUDE.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..182803e --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,131 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +**codspeed-jvm** is a CodSpeed integration for JVM-based projects that use JMH (Java Microbenchmark Harness). It provides walltime benchmarking support and performance tracking in CI. + +### Key Components + +- **`jmh-fork/`**: Forked OpenJDK JMH with CodSpeed walltime result collection. Maven multi-module project used as a composite build dependency. +- **`examples/example-gradle/`**: Gradle-based example benchmarks using the custom JMH fork +- **`examples/example-maven/`**: Maven-based example benchmarks +- **`examples/TheAlgorithms-Java/`**: TheAlgorithms submodule with additional benchmarks + +## Build & Development + +### Prerequisites + +- **Java**: JDK 21+ (for main project), JDK 11+ (for jmh-fork modules) +- **Maven**: 3.9.9+ (for jmh-fork) +- **Gradle**: Uses gradlew wrapper (no separate installation needed) + +### Common Commands + +#### Building + +```bash +# Build and run Gradle example benchmarks +./gradlew :examples:example-gradle:jmh + +# Build JAR artifact only (useful for local profiling) +./gradlew :examples:example-gradle:jmhJar + +# Build and install JMH fork to local Maven repo (required before Maven example) +cd jmh-fork && mvn clean install -DskipTests -q + +# Build Maven example (after installing JMH fork) +cd examples/example-maven && mvn package -q +java -jar examples/example-maven/target/example-maven-1.0-SNAPSHOT.jar +``` + +#### Testing + +```bash +# Run CodSpeed tests for the JMH fork integration +cd jmh-fork && mvn test -pl jmh-core -Dtest="io.codspeed.*.*" + +# Run with CodSpeed locally (walltime mode) +codspeed run --mode walltime --skip-upload -- ./gradlew :examples:example-gradle:jmh + +# Run with specific pattern filtering +CODSPEED_LOG=trace codspeed run --mode walltime --skip-upload -- java -jar examples/example-gradle/build/libs/example-gradle-1.0-SNAPSHOT-jmh.jar ".*BacktrackingBenchmark.*" +``` + +#### Code Quality + +```bash +# Run pre-commit hooks manually (linting and formatting) +pre-commit run --all-files +# Uses Google Java Formatter v1.25.2 +``` + +### Build Structure + +- **Root**: Gradle root project (`settings.gradle.kts`) includes: + - `examples:example-gradle` as a regular subproject + - `jmh-fork` as a composite build (allows independent versioning) +- **jmh-fork**: Maven multi-module project with submodules `jmh-core`, `jmh-generator-*`, etc. + +### Publishing JMH Fork + +The JMH fork uses versioning scheme `1.37.0-codspeed.N`. When updating versions: + +```bash +cd jmh-fork && mvn versions:set -DnewVersion=1.37.0-codspeed.2 +``` + +Then update the version reference in root `build.gradle.kts`. + +## Architecture & Key Patterns + +### Multi-Build Structure + +The repository combines Maven and Gradle via Gradle's composite build feature: +- **jmh-fork** (Maven multi-module) is built as a composite dependency for independent versioning +- Example projects depend on the forked JMH packages + +### CodSpeed Integration Points + +1. **Walltime Collection**: Modified JMH fork in `jmh-fork/jmh-core` collects walltime metrics during benchmark execution +2. **JNI Bindings**: Native code hooks (C) for precise instrumentation in `jmh-fork/jmh-core/src/main/java/io/codspeed/` +3. **CI Integration**: GitHub Actions workflows trigger CodSpeed measurement runs on `codspeed-macro` runners + +### Benchmark Architecture + +- Benchmarks use standard JMH annotations (`@Benchmark`, `@Fork`, etc.) +- The `gradle-jmh-plugin` (v0.7.2) generates benchmark JARs from sources +- Both Gradle (`me.champeau.jmh`) and Maven profiles support building executable benchmark JARs + +## CI/CD & Testing + +### GitHub Actions Workflows (`.github/workflows/ci.yml`) + +- **lint**: Pre-commit hooks (Java formatting, YAML validation, file endings) +- **test**: CodSpeed-specific unit tests (jmh-core integration tests) +- **build-and-run-gradle**: Matrix test across Linux, macOS, Windows +- **build-and-run-maven**: Matrix test for Maven example, including publishToMavenLocal step +- **walltime-benchmarks**: Runs on CodSpeed infrastructure for real performance tracking + +## Modifying JMH Fork + +1. Make changes in `jmh-fork/` +2. Bump version: `cd jmh-fork && mvn versions:set -DnewVersion=X.Y.Z-codspeed.N` +3. Publish locally: `cd jmh-fork && mvn clean install -DskipTests -q` +4. Update version reference in root `build.gradle.kts` if needed +5. Test both Gradle and Maven examples to verify compatibility + +## Profiling & Debugging + +Use `just` commands for quick profiling workflows (see `Justfile`): +- `just profile-codspeed` — walltime profiling via CodSpeed runner +- `just profile-perf` — Linux perf + flamegraph (requires Linux + JDK perf integration) +- `just profile-asprof` — async-profiler flamegraph + +## Notes + +- **Submodules**: Repository uses Git submodules (`.gitmodules`). Clone with `--recurse-submodules`. +- **Java Versions**: Root project uses Java 21; jmh-fork runs on Java 11+. +- **Pre-commit Exclusions**: jmh-fork is mostly excluded from pre-commit hooks except for `jmh-core/src/main/java/io/codspeed/`. +- **CodSpeed Credentials**: CI uses OIDC for authentication. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 0000000..47dc3e3 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file From bc9811ae6d1873ad7180c42f11de0089828d4be6 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 25 Mar 2026 10:18:37 +0100 Subject: [PATCH 08/12] chore: add build flags to remove instrument-hooks warnings --- jmh-fork/jmh-core/build.gradle.kts | 1 + 1 file changed, 1 insertion(+) diff --git a/jmh-fork/jmh-core/build.gradle.kts b/jmh-fork/jmh-core/build.gradle.kts index 01e0e9d..917f62e 100644 --- a/jmh-fork/jmh-core/build.gradle.kts +++ b/jmh-fork/jmh-core/build.gradle.kts @@ -45,6 +45,7 @@ tasks.register("compileNative") { commandLine( "gcc", "-shared", "-fPIC", "-O2", "-std=c11", + "-Wno-format", "-Wno-format-security", "-o", outputLib.absolutePath, "${nativeDir}/instrument_hooks_jni.c", "${instrumentHooksDir}/dist/core.c", From 137bd407e82a14258f1f2fd4fdb771d2c9359426 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 25 Mar 2026 11:48:01 +0100 Subject: [PATCH 09/12] feat: change JMH fork group ID to io.codspeed.jmh Prepares for MavenCentral publishing by using a unique group ID (`io.codspeed.jmh` instead of `org.openjdk.jmh`) and dropping the `-codspeed.N` version suffix. Refs: COD-2400 --- AGENTS.md | 6 +++--- CONTRIBUTING.md | 4 ++-- examples/example-gradle/build.gradle.kts | 2 +- examples/example-maven/pom.xml | 6 +++--- jmh-fork/build.gradle.kts | 4 ++-- .../jmh-groovy-benchmark-archetype/pom.xml | 4 ++-- .../main/resources/archetype-resources/pom.xml | 4 ++-- .../jmh-java-benchmark-archetype/pom.xml | 4 ++-- .../main/resources/archetype-resources/pom.xml | 4 ++-- .../jmh-kotlin-benchmark-archetype/pom.xml | 4 ++-- .../main/resources/archetype-resources/pom.xml | 4 ++-- .../jmh-scala-benchmark-archetype/pom.xml | 4 ++-- .../main/resources/archetype-resources/pom.xml | 4 ++-- jmh-fork/jmh-archetypes/pom.xml | 4 ++-- jmh-fork/jmh-core-benchmarks/pom.xml | 8 ++++---- jmh-fork/jmh-core-ct/pom.xml | 12 ++++++------ jmh-fork/jmh-core-it/pom.xml | 18 +++++++++--------- jmh-fork/jmh-core/pom.xml | 4 ++-- jmh-fork/jmh-generator-annprocess/pom.xml | 6 +++--- jmh-fork/jmh-generator-asm/pom.xml | 8 ++++---- jmh-fork/jmh-generator-bytecode/pom.xml | 10 +++++----- jmh-fork/jmh-generator-reflection/pom.xml | 6 +++--- jmh-fork/jmh-samples/pom.xml | 8 ++++---- jmh-fork/pom.xml | 4 ++-- settings.gradle.kts | 10 +++++++++- 25 files changed, 80 insertions(+), 72 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 182803e..58bcb44 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -70,10 +70,10 @@ pre-commit run --all-files ### Publishing JMH Fork -The JMH fork uses versioning scheme `1.37.0-codspeed.N`. When updating versions: +The JMH fork is published under group ID `io.codspeed.jmh`. When updating versions: ```bash -cd jmh-fork && mvn versions:set -DnewVersion=1.37.0-codspeed.2 +cd jmh-fork && mvn versions:set -DnewVersion=x.y.z ``` Then update the version reference in root `build.gradle.kts`. @@ -111,7 +111,7 @@ The repository combines Maven and Gradle via Gradle's composite build feature: ## Modifying JMH Fork 1. Make changes in `jmh-fork/` -2. Bump version: `cd jmh-fork && mvn versions:set -DnewVersion=X.Y.Z-codspeed.N` +2. Bump version: `cd jmh-fork && mvn versions:set -DnewVersion=X.Y.Z` 3. Publish locally: `cd jmh-fork && mvn clean install -DskipTests -q` 4. Update version reference in root `build.gradle.kts` if needed 5. Test both Gradle and Maven examples to verify compatibility diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c173660..4b7ead7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,10 +2,10 @@ ## Bumping the JMH fork version -The JMH fork uses a versioning scheme like `1.37.0-codspeed.N`. To bump the version across all modules: +The JMH fork is published under group ID `io.codspeed.jmh`. To bump the version across all modules: ```bash -cd jmh-fork && mvn versions:set -DnewVersion=1.37.0-codspeed.2 +cd jmh-fork && mvn versions:set -DnewVersion=x.y.z ``` This updates all `pom.xml` files in the multi-module project. After bumping, also update the version reference in `build.gradle.kts`. diff --git a/examples/example-gradle/build.gradle.kts b/examples/example-gradle/build.gradle.kts index 2aff705..235e6f0 100644 --- a/examples/example-gradle/build.gradle.kts +++ b/examples/example-gradle/build.gradle.kts @@ -7,7 +7,7 @@ group = "io.codspeed" version = "1.0-SNAPSHOT" jmh { - jmhVersion.set("1.37.0-codspeed.1") + jmhVersion.set("0.1.0") } sourceSets { diff --git a/examples/example-maven/pom.xml b/examples/example-maven/pom.xml index 60509c1..fa81560 100644 --- a/examples/example-maven/pom.xml +++ b/examples/example-maven/pom.xml @@ -12,17 +12,17 @@ UTF-8 21 21 - 1.37.0-codspeed.1 + 0.1.0 - org.openjdk.jmh + io.codspeed.jmh jmh-core ${jmh.version} - org.openjdk.jmh + io.codspeed.jmh jmh-generator-annprocess ${jmh.version} provided diff --git a/jmh-fork/build.gradle.kts b/jmh-fork/build.gradle.kts index 8c2f715..b2b4d17 100644 --- a/jmh-fork/build.gradle.kts +++ b/jmh-fork/build.gradle.kts @@ -2,8 +2,8 @@ subprojects { apply(plugin = "java-library") apply(plugin = "maven-publish") - group = "org.openjdk.jmh" - version = "1.37.0-codspeed.1" + group = "io.codspeed.jmh" + version = "0.1.0" repositories { mavenCentral() diff --git a/jmh-fork/jmh-archetypes/jmh-groovy-benchmark-archetype/pom.xml b/jmh-fork/jmh-archetypes/jmh-groovy-benchmark-archetype/pom.xml index 095fded..840476f 100644 --- a/jmh-fork/jmh-archetypes/jmh-groovy-benchmark-archetype/pom.xml +++ b/jmh-fork/jmh-archetypes/jmh-groovy-benchmark-archetype/pom.xml @@ -3,9 +3,9 @@ 4.0.0 - org.openjdk.jmh + io.codspeed.jmh jmh-archetypes - 1.37.0-codspeed.1 + 0.1.0 jmh-groovy-benchmark-archetype diff --git a/jmh-fork/jmh-archetypes/jmh-groovy-benchmark-archetype/src/main/resources/archetype-resources/pom.xml b/jmh-fork/jmh-archetypes/jmh-groovy-benchmark-archetype/src/main/resources/archetype-resources/pom.xml index ebea8b3..cfdfd3d 100644 --- a/jmh-fork/jmh-archetypes/jmh-groovy-benchmark-archetype/src/main/resources/archetype-resources/pom.xml +++ b/jmh-fork/jmh-archetypes/jmh-groovy-benchmark-archetype/src/main/resources/archetype-resources/pom.xml @@ -45,7 +45,7 @@ THE POSSIBILITY OF SUCH DAMAGE. - org.openjdk.jmh + io.codspeed.jmh jmh-core \${jmh.version} @@ -168,7 +168,7 @@ THE POSSIBILITY OF SUCH DAMAGE. - org.openjdk.jmh + io.codspeed.jmh jmh-generator-bytecode \${jmh.version} diff --git a/jmh-fork/jmh-archetypes/jmh-java-benchmark-archetype/pom.xml b/jmh-fork/jmh-archetypes/jmh-java-benchmark-archetype/pom.xml index f45a6b1..73d2407 100644 --- a/jmh-fork/jmh-archetypes/jmh-java-benchmark-archetype/pom.xml +++ b/jmh-fork/jmh-archetypes/jmh-java-benchmark-archetype/pom.xml @@ -28,9 +28,9 @@ questions. 4.0.0 - org.openjdk.jmh + io.codspeed.jmh jmh-archetypes - 1.37.0-codspeed.1 + 0.1.0 jmh-java-benchmark-archetype diff --git a/jmh-fork/jmh-archetypes/jmh-java-benchmark-archetype/src/main/resources/archetype-resources/pom.xml b/jmh-fork/jmh-archetypes/jmh-java-benchmark-archetype/src/main/resources/archetype-resources/pom.xml index 761842c..0b8ea68 100644 --- a/jmh-fork/jmh-archetypes/jmh-java-benchmark-archetype/src/main/resources/archetype-resources/pom.xml +++ b/jmh-fork/jmh-archetypes/jmh-java-benchmark-archetype/src/main/resources/archetype-resources/pom.xml @@ -47,12 +47,12 @@ THE POSSIBILITY OF SUCH DAMAGE. - org.openjdk.jmh + io.codspeed.jmh jmh-core \${jmh.version} - org.openjdk.jmh + io.codspeed.jmh jmh-generator-annprocess \${jmh.version} provided diff --git a/jmh-fork/jmh-archetypes/jmh-kotlin-benchmark-archetype/pom.xml b/jmh-fork/jmh-archetypes/jmh-kotlin-benchmark-archetype/pom.xml index fa46484..6092728 100644 --- a/jmh-fork/jmh-archetypes/jmh-kotlin-benchmark-archetype/pom.xml +++ b/jmh-fork/jmh-archetypes/jmh-kotlin-benchmark-archetype/pom.xml @@ -3,9 +3,9 @@ 4.0.0 - org.openjdk.jmh + io.codspeed.jmh jmh-archetypes - 1.37.0-codspeed.1 + 0.1.0 jmh-kotlin-benchmark-archetype diff --git a/jmh-fork/jmh-archetypes/jmh-kotlin-benchmark-archetype/src/main/resources/archetype-resources/pom.xml b/jmh-fork/jmh-archetypes/jmh-kotlin-benchmark-archetype/src/main/resources/archetype-resources/pom.xml index 2c8a043..3b78924 100644 --- a/jmh-fork/jmh-archetypes/jmh-kotlin-benchmark-archetype/src/main/resources/archetype-resources/pom.xml +++ b/jmh-fork/jmh-archetypes/jmh-kotlin-benchmark-archetype/src/main/resources/archetype-resources/pom.xml @@ -46,7 +46,7 @@ THE POSSIBILITY OF SUCH DAMAGE. - org.openjdk.jmh + io.codspeed.jmh jmh-core \${jmh.version} @@ -157,7 +157,7 @@ THE POSSIBILITY OF SUCH DAMAGE. - org.openjdk.jmh + io.codspeed.jmh jmh-generator-bytecode \${jmh.version} diff --git a/jmh-fork/jmh-archetypes/jmh-scala-benchmark-archetype/pom.xml b/jmh-fork/jmh-archetypes/jmh-scala-benchmark-archetype/pom.xml index 66bbbe3..29a5097 100644 --- a/jmh-fork/jmh-archetypes/jmh-scala-benchmark-archetype/pom.xml +++ b/jmh-fork/jmh-archetypes/jmh-scala-benchmark-archetype/pom.xml @@ -3,9 +3,9 @@ 4.0.0 - org.openjdk.jmh + io.codspeed.jmh jmh-archetypes - 1.37.0-codspeed.1 + 0.1.0 jmh-scala-benchmark-archetype diff --git a/jmh-fork/jmh-archetypes/jmh-scala-benchmark-archetype/src/main/resources/archetype-resources/pom.xml b/jmh-fork/jmh-archetypes/jmh-scala-benchmark-archetype/src/main/resources/archetype-resources/pom.xml index 2a040fb..637080e 100644 --- a/jmh-fork/jmh-archetypes/jmh-scala-benchmark-archetype/src/main/resources/archetype-resources/pom.xml +++ b/jmh-fork/jmh-archetypes/jmh-scala-benchmark-archetype/src/main/resources/archetype-resources/pom.xml @@ -45,7 +45,7 @@ THE POSSIBILITY OF SUCH DAMAGE. - org.openjdk.jmh + io.codspeed.jmh jmh-core \${jmh.version} @@ -174,7 +174,7 @@ THE POSSIBILITY OF SUCH DAMAGE. - org.openjdk.jmh + io.codspeed.jmh jmh-generator-bytecode \${jmh.version} diff --git a/jmh-fork/jmh-archetypes/pom.xml b/jmh-fork/jmh-archetypes/pom.xml index c4121b0..0cd218b 100644 --- a/jmh-fork/jmh-archetypes/pom.xml +++ b/jmh-fork/jmh-archetypes/pom.xml @@ -28,9 +28,9 @@ questions. 4.0.0 - org.openjdk.jmh + io.codspeed.jmh jmh-parent - 1.37.0-codspeed.1 + 0.1.0 JMH Archetypes diff --git a/jmh-fork/jmh-core-benchmarks/pom.xml b/jmh-fork/jmh-core-benchmarks/pom.xml index ae4e9a3..8642534 100644 --- a/jmh-fork/jmh-core-benchmarks/pom.xml +++ b/jmh-fork/jmh-core-benchmarks/pom.xml @@ -28,9 +28,9 @@ questions. 4.0.0 - org.openjdk.jmh + io.codspeed.jmh jmh-parent - 1.37.0-codspeed.1 + 0.1.0 JMH Core Benchmarks @@ -45,12 +45,12 @@ questions. - org.openjdk.jmh + io.codspeed.jmh jmh-core ${project.version} - org.openjdk.jmh + io.codspeed.jmh jmh-generator-annprocess ${project.version} provided diff --git a/jmh-fork/jmh-core-ct/pom.xml b/jmh-fork/jmh-core-ct/pom.xml index 4fa1d95..9585308 100644 --- a/jmh-fork/jmh-core-ct/pom.xml +++ b/jmh-fork/jmh-core-ct/pom.xml @@ -28,9 +28,9 @@ questions. 4.0.0 - org.openjdk.jmh + io.codspeed.jmh jmh-parent - 1.37.0-codspeed.1 + 0.1.0 JMH Core Compilation Tests @@ -45,22 +45,22 @@ questions. - org.openjdk.jmh + io.codspeed.jmh jmh-core ${project.version} - org.openjdk.jmh + io.codspeed.jmh jmh-generator-annprocess ${project.version} - org.openjdk.jmh + io.codspeed.jmh jmh-generator-reflection ${project.version} - org.openjdk.jmh + io.codspeed.jmh jmh-generator-asm ${project.version} diff --git a/jmh-fork/jmh-core-it/pom.xml b/jmh-fork/jmh-core-it/pom.xml index 62d8fce..7aef82c 100644 --- a/jmh-fork/jmh-core-it/pom.xml +++ b/jmh-fork/jmh-core-it/pom.xml @@ -28,9 +28,9 @@ questions. 4.0.0 - org.openjdk.jmh + io.codspeed.jmh jmh-parent - 1.37.0-codspeed.1 + 0.1.0 JMH Core Integration Tests @@ -45,7 +45,7 @@ questions. - org.openjdk.jmh + io.codspeed.jmh jmh-core ${project.version} @@ -107,7 +107,7 @@ questions. - org.openjdk.jmh + io.codspeed.jmh jmh-generator-annprocess ${project.version} provided @@ -133,7 +133,7 @@ questions. - org.openjdk.jmh + io.codspeed.jmh jmh-generator-annprocess ${project.version} provided @@ -163,7 +163,7 @@ questions. - org.openjdk.jmh + io.codspeed.jmh jmh-generator-annprocess ${project.version} provided @@ -193,7 +193,7 @@ questions. - org.openjdk.jmh + io.codspeed.jmh jmh-generator-annprocess ${project.version} provided @@ -223,7 +223,7 @@ questions. - org.openjdk.jmh + io.codspeed.jmh jmh-generator-bytecode ${project.version} @@ -307,7 +307,7 @@ questions. - org.openjdk.jmh + io.codspeed.jmh jmh-generator-bytecode ${project.version} diff --git a/jmh-fork/jmh-core/pom.xml b/jmh-fork/jmh-core/pom.xml index 04204fb..847f927 100644 --- a/jmh-fork/jmh-core/pom.xml +++ b/jmh-fork/jmh-core/pom.xml @@ -29,9 +29,9 @@ questions. 4.0.0 - org.openjdk.jmh + io.codspeed.jmh jmh-parent - 1.37.0-codspeed.1 + 0.1.0 JMH Core diff --git a/jmh-fork/jmh-generator-annprocess/pom.xml b/jmh-fork/jmh-generator-annprocess/pom.xml index dbf7bbc..94e119f 100644 --- a/jmh-fork/jmh-generator-annprocess/pom.xml +++ b/jmh-fork/jmh-generator-annprocess/pom.xml @@ -29,9 +29,9 @@ questions. 4.0.0 - org.openjdk.jmh + io.codspeed.jmh jmh-parent - 1.37.0-codspeed.1 + 0.1.0 JMH Generators: Annotation Processors @@ -44,7 +44,7 @@ questions. - org.openjdk.jmh + io.codspeed.jmh jmh-core ${project.version} diff --git a/jmh-fork/jmh-generator-asm/pom.xml b/jmh-fork/jmh-generator-asm/pom.xml index 79544f8..b576490 100644 --- a/jmh-fork/jmh-generator-asm/pom.xml +++ b/jmh-fork/jmh-generator-asm/pom.xml @@ -28,9 +28,9 @@ questions. 4.0.0 - org.openjdk.jmh + io.codspeed.jmh jmh-parent - 1.37.0-codspeed.1 + 0.1.0 JMH Generators: ASM @@ -43,12 +43,12 @@ questions. - org.openjdk.jmh + io.codspeed.jmh jmh-core ${project.version} - org.openjdk.jmh + io.codspeed.jmh jmh-generator-reflection ${project.version} diff --git a/jmh-fork/jmh-generator-bytecode/pom.xml b/jmh-fork/jmh-generator-bytecode/pom.xml index b950355..49f2234 100644 --- a/jmh-fork/jmh-generator-bytecode/pom.xml +++ b/jmh-fork/jmh-generator-bytecode/pom.xml @@ -28,9 +28,9 @@ questions. 4.0.0 - org.openjdk.jmh + io.codspeed.jmh jmh-parent - 1.37.0-codspeed.1 + 0.1.0 JMH Generators: Bytecode @@ -43,17 +43,17 @@ questions. - org.openjdk.jmh + io.codspeed.jmh jmh-core ${project.version} - org.openjdk.jmh + io.codspeed.jmh jmh-generator-reflection ${project.version} - org.openjdk.jmh + io.codspeed.jmh jmh-generator-asm ${project.version} diff --git a/jmh-fork/jmh-generator-reflection/pom.xml b/jmh-fork/jmh-generator-reflection/pom.xml index 0963199..0203fc0 100644 --- a/jmh-fork/jmh-generator-reflection/pom.xml +++ b/jmh-fork/jmh-generator-reflection/pom.xml @@ -28,9 +28,9 @@ questions. 4.0.0 - org.openjdk.jmh + io.codspeed.jmh jmh-parent - 1.37.0-codspeed.1 + 0.1.0 JMH Generators: Reflection @@ -43,7 +43,7 @@ questions. - org.openjdk.jmh + io.codspeed.jmh jmh-core ${project.version} diff --git a/jmh-fork/jmh-samples/pom.xml b/jmh-fork/jmh-samples/pom.xml index 331140b..e1b2b14 100644 --- a/jmh-fork/jmh-samples/pom.xml +++ b/jmh-fork/jmh-samples/pom.xml @@ -34,9 +34,9 @@ THE POSSIBILITY OF SUCH DAMAGE. 4.0.0 - org.openjdk.jmh + io.codspeed.jmh jmh-parent - 1.37.0-codspeed.1 + 0.1.0 JMH Samples @@ -51,12 +51,12 @@ THE POSSIBILITY OF SUCH DAMAGE. - org.openjdk.jmh + io.codspeed.jmh jmh-core ${project.version} - org.openjdk.jmh + io.codspeed.jmh jmh-generator-annprocess ${project.version} provided diff --git a/jmh-fork/pom.xml b/jmh-fork/pom.xml index 6e3d54a..0570add 100644 --- a/jmh-fork/pom.xml +++ b/jmh-fork/pom.xml @@ -27,10 +27,10 @@ questions. xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - org.openjdk.jmh + io.codspeed.jmh jmh-parent pom - 1.37.0-codspeed.1 + 0.1.0 Java Microbenchmark Harness Parent diff --git a/settings.gradle.kts b/settings.gradle.kts index b8a4adf..47d3ded 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -1,4 +1,12 @@ rootProject.name = "codspeed-jvm" include("examples:example-gradle") -includeBuild("jmh-fork") +includeBuild("jmh-fork") { + dependencySubstitution { + substitute(module("org.openjdk.jmh:jmh-core")).using(project(":jmh-core")) + substitute(module("org.openjdk.jmh:jmh-generator-annprocess")).using(project(":jmh-generator-annprocess")) + substitute(module("org.openjdk.jmh:jmh-generator-bytecode")).using(project(":jmh-generator-bytecode")) + substitute(module("org.openjdk.jmh:jmh-generator-reflection")).using(project(":jmh-generator-reflection")) + substitute(module("org.openjdk.jmh:jmh-generator-asm")).using(project(":jmh-generator-asm")) + } +} From 3c5083609a786561e5c06f2fcc9ab93cc3203c91 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 27 Mar 2026 11:23:43 +0100 Subject: [PATCH 10/12] fix: guard CodSpeed-specific behavior behind isInstrumented check Only show the "Running with CodSpeed" banner, write walltime results, and restrict benchmark modes when actually running under CodSpeed instrumentation. Restores standard JMH behavior when running locally. --- .../java/org/openjdk/jmh/runner/Runner.java | 51 ++++++++++++------- .../jmh/runner/format/TextReportFormat.java | 5 +- 2 files changed, 36 insertions(+), 20 deletions(-) diff --git a/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java b/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java index 542b820..f12ac1f 100644 --- a/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java +++ b/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java @@ -265,9 +265,10 @@ private Collection internalRun() throws RunnerException { // override the benchmark types; // this may yield new benchmark records if (!options.getBenchModes().isEmpty()) { - // CodSpeed: warn and pick first mode if multiple are specified via CLI (-bm flag) Collection benchModes = options.getBenchModes(); - if (benchModes.size() > 1) { + + // CodSpeed: warn and pick first mode if multiple are specified via CLI (-bm flag) + if (InstrumentHooks.getInstance().isInstrumented() && benchModes.size() > 1) { Mode firstMode = benchModes.iterator().next(); out.println("WARNING: CodSpeed does not support multiple benchmark modes. " + "The -bm flag specifies " + benchModes.size() + " modes: " + benchModes + ". " + @@ -288,18 +289,28 @@ private Collection internalRun() throws RunnerException { } // clone with all the modes - // CodSpeed: Mode.All would expand into multiple modes per benchmark, which we don't support. - // Instead, pick the first concrete mode and warn. { + boolean isInstrumented = InstrumentHooks.getInstance().isInstrumented(); List newBenchmarks = new ArrayList<>(); for (BenchmarkListEntry br : benchmarks) { - if (br.getMode() == Mode.All) { + if (br.getMode() != Mode.All) { + newBenchmarks.add(br); + continue; + } + + if (isInstrumented) { + // CodSpeed: Mode.All would expand into multiple modes per benchmark, + // which we don't support. Pick a single mode and warn. Mode firstMode = Mode.SampleTime; out.println("WARNING: CodSpeed does not support multiple benchmark modes. " + br.getUsername() + " uses Mode.All, using " + firstMode + " instead."); newBenchmarks.add(br.cloneWith(firstMode)); } else { - newBenchmarks.add(br); + // Standard JMH: expand Mode.All into all concrete modes + for (Mode m : Mode.values()) { + if (m == Mode.All) continue; + newBenchmarks.add(br.cloneWith(m)); + } } } @@ -596,20 +607,22 @@ private Collection runBenchmarks(SortedSet benchm SortedSet runResults = mergeRunResults(results); - // Collect CodSpeed walltime results - try { - String profileFolder = System.getenv("CODSPEED_PROFILE_FOLDER"); - if (profileFolder == null) { - profileFolder = "/tmp"; + // Collect CodSpeed walltime results (only when instrumented) + if (InstrumentHooks.getInstance().isInstrumented()) { + try { + String profileFolder = System.getenv("CODSPEED_PROFILE_FOLDER"); + if (profileFolder == null) { + profileFolder = "/tmp"; + } + int pid = (int) ProcessHandle.current().pid(); + // TODO: Get and set the actual version + CodSpeedResultCollector.collectAndWrite( + runResults, pid, "codspeed-jvm", "1.0.0", + Path.of(profileFolder).resolve("results") + ); + } catch (IOException e) { + out.println("WARNING: Failed to write CodSpeed results: " + e.getMessage()); } - int pid = (int) ProcessHandle.current().pid(); - // TODO: Get and set the actual version - CodSpeedResultCollector.collectAndWrite( - runResults, pid, "codspeed-jvm", "1.0.0", - Path.of(profileFolder).resolve("results") - ); - } catch (IOException e) { - out.println("WARNING: Failed to write CodSpeed results: " + e.getMessage()); } out.endRun(runResults); diff --git a/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/format/TextReportFormat.java b/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/format/TextReportFormat.java index 0736d5f..a61684f 100644 --- a/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/format/TextReportFormat.java +++ b/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/format/TextReportFormat.java @@ -38,6 +38,7 @@ import org.openjdk.jmh.runner.options.TimeValue; import org.openjdk.jmh.runner.options.VerboseMode; import org.openjdk.jmh.util.Utils; +import io.codspeed.instrument_hooks.InstrumentHooks; import java.io.PrintStream; import java.util.ArrayList; @@ -62,7 +63,9 @@ public void startBenchmark(BenchmarkParams params) { opts = ""; } - println("# Running with CodSpeed (mode: walltime)"); + if (InstrumentHooks.getInstance().isInstrumented()) { + println("# Running with CodSpeed (mode: walltime)"); + } println("# JMH version: " + params.getJmhVersion()); println("# VM version: JDK " + params.getJdkVersion() + ", " + params.getVmName() + ", " + params.getVmVersion()); From eb1a958dfed4e7c824dc1c79e1148e8edd8fc3c9 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 27 Mar 2026 11:25:34 +0100 Subject: [PATCH 11/12] fix: use actual JMH fork version in walltime result output --- .../jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java b/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java index f12ac1f..4a11281 100644 --- a/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java +++ b/jmh-fork/jmh-core/src/main/java/org/openjdk/jmh/runner/Runner.java @@ -615,9 +615,8 @@ private Collection runBenchmarks(SortedSet benchm profileFolder = "/tmp"; } int pid = (int) ProcessHandle.current().pid(); - // TODO: Get and set the actual version CodSpeedResultCollector.collectAndWrite( - runResults, pid, "codspeed-jvm", "1.0.0", + runResults, pid, "codspeed-jvm", Version.getPlainVersion(), Path.of(profileFolder).resolve("results") ); } catch (IOException e) { From b43be0b73e4b722ac2ab7ab92f4e73a9fd948227 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 27 Mar 2026 12:02:07 +0100 Subject: [PATCH 12/12] ci: skip upload for non-temurin distribution benchmarks --- .github/workflows/ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c3781cd..ae80a0b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,17 +86,22 @@ jobs: walltime-benchmarks: runs-on: codspeed-macro + strategy: + matrix: + distribution: [corretto, temurin, oracle, zulu, microsoft, semeru] steps: - uses: actions/checkout@v4 with: submodules: true - uses: actions/setup-java@v4 with: - distribution: temurin + distribution: ${{ matrix.distribution }} java-version: 21 cache: 'gradle' - name: Run the benchmarks uses: CodSpeedHQ/action@main + env: + CODSPEED_SKIP_UPLOAD: ${{ matrix.distribution != 'temurin' }} with: mode: walltime run: ./gradlew :examples:example-gradle:jmh