diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 7e3690a8be5..6fff787bedf 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -44,13 +44,13 @@ jobs: # TODO (jack-berg): Select or build appropriate benchmarks for other key areas: # - Log SDK record & export - # - Trace SDK record & export + # - Trace SDK export # - Metric SDK export # - Noop implementation - name: Run Benchmark run: | cd sdk/all/build - java -jar libs/opentelemetry-sdk-*-jmh.jar -rf json MetricRecordBenchmark + java -jar libs/opentelemetry-sdk-*-jmh.jar -rf json MetricRecordBenchmark SpanRecordBenchmark - name: Use CLA approved github bot run: .github/scripts/use-cla-approved-bot.sh diff --git a/sdk/all/src/jmh/java/io/opentelemetry/sdk/BenchmarkUtils.java b/sdk/all/src/jmh/java/io/opentelemetry/sdk/BenchmarkUtils.java new file mode 100644 index 00000000000..106f22afea9 --- /dev/null +++ b/sdk/all/src/jmh/java/io/opentelemetry/sdk/BenchmarkUtils.java @@ -0,0 +1,17 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.sdk; + +public class BenchmarkUtils { + + private BenchmarkUtils() {} + + /** + * The number of record operations per benchmark invocation. By using a constant across benchmarks + * of different signals, it's easier to compare benchmark results across signals. + */ + public static final int RECORDS_PER_INVOCATION = 1024 * 10; +} diff --git a/sdk/all/src/jmh/java/io/opentelemetry/sdk/MetricRecordBenchmark.java b/sdk/all/src/jmh/java/io/opentelemetry/sdk/MetricRecordBenchmark.java index d51c8171ea3..1a093e8fe5f 100644 --- a/sdk/all/src/jmh/java/io/opentelemetry/sdk/MetricRecordBenchmark.java +++ b/sdk/all/src/jmh/java/io/opentelemetry/sdk/MetricRecordBenchmark.java @@ -43,15 +43,28 @@ import org.openjdk.jmh.annotations.Warmup; /** - * Notes on interpreting the data: + * This benchmark measures the performance of recording metrics. It includes the following + * dimensions: * - *

The benchmark has two dimensions which partially overlap: cardinality and thread count. - * Cardinality dictates how many unique attribute sets (i.e. series) are recorded to, and thread - * count dictates how many threads are simultaneously recording to those series. In all cases, the - * record path needs to look up an aggregation handle for the series corresponding to the - * measurement's {@link Attributes} in a {@link java.util.concurrent.ConcurrentHashMap}. That will - * be the case until otel adds support for bound + *

+ * + *

Each operation consists of recording {@link MetricRecordBenchmark#RECORDS_PER_INVOCATION} + * measurements. + * + *

The cardinality and thread count dimensions partially overlap. Cardinality dictates how many + * unique attribute sets (i.e. series) are recorded to, and thread count dictates how many threads + * are simultaneously recording to those series. In all cases, the record path needs to look up an + * aggregation handle for the series corresponding to the measurement's {@link Attributes} in a + * {@link java.util.concurrent.ConcurrentHashMap}. That will be the case until otel adds support for + * bound * instruments. The cardinality dictates the size of this map, which has some impact on * performance. However, by far the dominant bottleneck is contention. That is, the number of * threads simultaneously trying to record to the same series. Increasing the threads increases @@ -72,10 +85,11 @@ public class MetricRecordBenchmark { private static final int INITIAL_SEED = 513423236; - private static final int RECORD_COUNT = 10 * 1024; + private static final int MAX_THREADS = 4; + private static final int RECORDS_PER_INVOCATION = BenchmarkUtils.RECORDS_PER_INVOCATION; @State(Scope.Benchmark) - public static class ThreadState { + public static class BenchmarkState { @Param InstrumentTypeAndAggregation instrumentTypeAndAggregation; @@ -154,8 +168,8 @@ public void setup() { } Collections.shuffle(attributesList); - measurements = new ArrayList<>(RECORD_COUNT); - for (int i = 0; i < RECORD_COUNT; i++) { + measurements = new ArrayList<>(RECORDS_PER_INVOCATION); + for (int i = 0; i < RECORDS_PER_INVOCATION; i++) { measurements.add((long) random.nextInt(2000)); } Collections.shuffle(measurements); @@ -175,25 +189,26 @@ public void tearDown() { @Fork(1) @Warmup(iterations = 5, time = 1) @Measurement(iterations = 5, time = 1) - public void record_1Thread(ThreadState threadState) { - record(threadState); + public void record_SingleThread(BenchmarkState benchmarkState) { + record(benchmarkState); } @Benchmark - @Group("threads4") - @GroupThreads(4) + @Group("threads" + MAX_THREADS) + @GroupThreads(MAX_THREADS) @Fork(1) @Warmup(iterations = 5, time = 1) @Measurement(iterations = 5, time = 1) - public void record_4Threads(ThreadState threadState) { - record(threadState); + public void record_MultipleThreads(BenchmarkState benchmarkState) { + record(benchmarkState); } - private static void record(ThreadState threadState) { - for (int i = 0; i < RECORD_COUNT; i++) { - Attributes attributes = threadState.attributesList.get(i % threadState.attributesList.size()); - long value = threadState.measurements.get(i % threadState.measurements.size()); - threadState.instrument.record(value, attributes); + private static void record(BenchmarkState benchmarkState) { + for (int i = 0; i < RECORDS_PER_INVOCATION; i++) { + Attributes attributes = + benchmarkState.attributesList.get(i % benchmarkState.attributesList.size()); + long value = benchmarkState.measurements.get(i % benchmarkState.measurements.size()); + benchmarkState.instrument.record(value, attributes); } } diff --git a/sdk/all/src/jmh/java/io/opentelemetry/sdk/SpanRecordBenchmark.java b/sdk/all/src/jmh/java/io/opentelemetry/sdk/SpanRecordBenchmark.java new file mode 100644 index 00000000000..d9b18fd518a --- /dev/null +++ b/sdk/all/src/jmh/java/io/opentelemetry/sdk/SpanRecordBenchmark.java @@ -0,0 +1,174 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.sdk; + +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.TraceFlags; +import io.opentelemetry.api.trace.TraceState; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.sdk.trace.IdGenerator; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; +import io.opentelemetry.sdk.trace.export.SpanExporter; +import io.opentelemetry.sdk.trace.samplers.Sampler; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Group; +import org.openjdk.jmh.annotations.GroupThreads; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; + +/** + * This benchmark measures the performance of recording spans. It includes the following dimensions: + * + *

+ * + *

Each operation consists of recording {@link SpanRecordBenchmark#RECORDS_PER_INVOCATION} spans. + * + *

In order to isolate the record path while remaining realistic, the benchmark uses a {@link + * BatchSpanProcessor} paired with a noop {@link SpanExporter}. In order to avoid quickly outpacing + * the batch processor queue and dropping spans, the processor is configured with a queue size of + * {@link SpanRecordBenchmark#RECORDS_PER_INVOCATION} * {@link SpanRecordBenchmark#MAX_THREADS} and + * is flushed after each invocation. + */ +public class SpanRecordBenchmark { + + private static final int RECORDS_PER_INVOCATION = BenchmarkUtils.RECORDS_PER_INVOCATION; + private static final int MAX_THREADS = 4; + private static final int QUEUE_SIZE = RECORDS_PER_INVOCATION * MAX_THREADS; + + @State(Scope.Benchmark) + public static class BenchmarkState { + + // Encode a variety of dimensions (# attributes, # events, # links) into a single enum to + // benchmark various shapes of spans without combinatorial explosion. + @Param SpanSize spanSize; + + SdkTracerProvider tracerProvider; + Tracer tracer; + List> attributeKeys; + List attributeValues; + List exceptions; + List linkContexts; + + @Setup + public void setup() { + tracerProvider = + SdkTracerProvider.builder() + // Configure a batch processor with a noop exporter (SpanExporter.composite() is a + // shortcut for a noop exporter). This allows testing the throughput / performance + // impact of BatchSpanProcessor, which is essential for real workloads, while avoiding + // noise from SpanExporters whose performance is subject to implementation and network + // details. + .addSpanProcessor( + BatchSpanProcessor.builder(SpanExporter.composite()) + .setMaxQueueSize(QUEUE_SIZE) + .build()) + .setSampler(Sampler.alwaysOn()) + .build(); + tracer = tracerProvider.get("benchmarkTracer"); + + attributeKeys = new ArrayList<>(spanSize.attributes); + attributeValues = new ArrayList<>(spanSize.attributes); + for (int i = 0; i < spanSize.attributes; i++) { + attributeKeys.add(AttributeKey.stringKey("key" + i)); + attributeValues.add("value" + i); + } + + exceptions = new ArrayList<>(spanSize.events); + for (int i = 0; i < spanSize.events; i++) { + exceptions.add(new Exception("test exception")); + } + + linkContexts = new ArrayList<>(spanSize.links); + for (int i = 0; i < spanSize.links; i++) { + linkContexts.add( + SpanContext.create( + IdGenerator.random().generateTraceId(), + IdGenerator.random().generateSpanId(), + TraceFlags.getDefault(), + TraceState.getDefault())); + } + } + + @TearDown(Level.Invocation) + public void flush() { + tracerProvider.forceFlush().join(10, TimeUnit.SECONDS); + } + + @TearDown + public void tearDown() { + tracerProvider.shutdown(); + } + } + + @Benchmark + @Group("threads1") + @GroupThreads(1) + @Fork(1) + @Warmup(iterations = 5, time = 1) + @Measurement(iterations = 5, time = 1) + public void record_SingleThread(BenchmarkState benchmarkState) { + record(benchmarkState); + } + + @Benchmark + @Group("threads" + MAX_THREADS) + @GroupThreads(MAX_THREADS) + @Fork(1) + @Warmup(iterations = 5, time = 1) + @Measurement(iterations = 5, time = 1) + public void record_MultipleThreads(BenchmarkState benchmarkState) { + record(benchmarkState); + } + + private static void record(BenchmarkState benchmarkState) { + for (int i = 0; i < RECORDS_PER_INVOCATION; i++) { + Span span = benchmarkState.tracer.spanBuilder("test span name").startSpan(); + for (int j = 0; j < benchmarkState.attributeKeys.size(); j++) { + span.setAttribute( + benchmarkState.attributeKeys.get(j), benchmarkState.attributeValues.get(j)); + } + for (int j = 0; j < benchmarkState.exceptions.size(); j++) { + span.recordException(benchmarkState.exceptions.get(j)); + } + for (int j = 0; j < benchmarkState.linkContexts.size(); j++) { + span.addLink(benchmarkState.linkContexts.get(j)); + } + span.end(); + } + } + + public enum SpanSize { + SMALL(0, 0, 0), + MEDIUM(10, 1, 0), + LARGE(100, 10, 5); + + private final int attributes; + private final int events; + private final int links; + + SpanSize(int attributes, int events, int links) { + this.attributes = attributes; + this.events = events; + this.links = links; + } + } +} diff --git a/sdk/trace/src/jmh/java/io/opentelemetry/sdk/trace/SpanBenchmark.java b/sdk/trace/src/jmh/java/io/opentelemetry/sdk/trace/SpanBenchmark.java deleted file mode 100644 index cf0cf3737c8..00000000000 --- a/sdk/trace/src/jmh/java/io/opentelemetry/sdk/trace/SpanBenchmark.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright The OpenTelemetry Authors - * SPDX-License-Identifier: Apache-2.0 - */ - -package io.opentelemetry.sdk.trace; - -import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.api.trace.Tracer; -import io.opentelemetry.sdk.resources.Resource; -import io.opentelemetry.sdk.trace.samplers.Sampler; -import java.util.concurrent.TimeUnit; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.Fork; -import org.openjdk.jmh.annotations.Level; -import org.openjdk.jmh.annotations.Measurement; -import org.openjdk.jmh.annotations.OutputTimeUnit; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.Threads; -import org.openjdk.jmh.annotations.Warmup; - -@State(Scope.Benchmark) -public class SpanBenchmark { - @SuppressWarnings("NonFinalStaticField") - private static SdkSpanBuilder sdkSpanBuilder; - - private final Resource serviceResource = - Resource.create( - Attributes.builder() - .put("service.name", "benchmark1") - .put("service.version", "123.456.89") - .put("service.instance.id", "123ab456-a123-12ab-12ab-12340a1abc12") - .build()); - - @Setup(Level.Trial) - public final void setup() { - SdkTracerProvider tracerProvider = - SdkTracerProvider.builder() - .setResource(serviceResource) - .setSampler(Sampler.alwaysOn()) - .build(); - - Tracer tracerSdk = tracerProvider.get("benchmarkTracer"); - sdkSpanBuilder = - (SdkSpanBuilder) - tracerSdk.spanBuilder("benchmarkSpanBuilder").setAttribute("longAttribute", 33L); - } - - @Benchmark - @Threads(value = 1) - @Fork(1) - @Warmup(iterations = 5, time = 1) - @Measurement(iterations = 10, time = 1) - @OutputTimeUnit(TimeUnit.MILLISECONDS) - public void simpleSpanStartAddEventEnd_01Thread() { - doSpanWork(); - } - - @Benchmark - @Threads(value = 5) - @Fork(1) - @Warmup(iterations = 5, time = 1) - @Measurement(iterations = 10, time = 1) - @OutputTimeUnit(TimeUnit.MILLISECONDS) - public void simpleSpanStartAddEventEnd_05Threads() { - doSpanWork(); - } - - @Benchmark - @Threads(value = 2) - @Fork(1) - @Warmup(iterations = 5, time = 1) - @Measurement(iterations = 10, time = 1) - @OutputTimeUnit(TimeUnit.MILLISECONDS) - public void simpleSpanStartAddEventEnd_02Threads() { - doSpanWork(); - } - - @Benchmark - @Threads(value = 10) - @Fork(1) - @Warmup(iterations = 5, time = 1) - @Measurement(iterations = 10, time = 1) - @OutputTimeUnit(TimeUnit.MILLISECONDS) - public void simpleSpanStartAddEventEnd_10Threads() { - doSpanWork(); - } - - private static void doSpanWork() { - Span span = sdkSpanBuilder.startSpan(); - span.addEvent("testEvent"); - span.end(); - } -}