Skip to content

Commit c0d50fe

Browse files
sravani-revurisravani-revuri
authored
HDDS-14961. Introduce configuration for tracing (#10029)
1 parent 8b4e3ce commit c0d50fe

7 files changed

Lines changed: 139 additions & 58 deletions

File tree

‎hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -413,6 +413,8 @@ private static void addDeprecatedKeys() {
413413
HddsConfigKeys.HDDS_METRICS_PERCENTILES_INTERVALS_KEY),
414414
new DeprecationDelta("hdds.recon.heartbeat.interval",
415415
HddsConfigKeys.HDDS_RECON_HEARTBEAT_INTERVAL),
416+
new DeprecationDelta("hdds.tracing.enabled",
417+
"ozone.tracing.enabled"),
416418
});
417419
}
418420

‎hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -469,9 +469,6 @@ public final class ScmConfigKeys {
469469
public static final String OZONE_SCM_NETWORK_TOPOLOGY_SCHEMA_FILE_DEFAULT =
470470
"network-topology-default.xml";
471471

472-
public static final String HDDS_TRACING_ENABLED = "hdds.tracing.enabled";
473-
public static final boolean HDDS_TRACING_ENABLED_DEFAULT = false;
474-
475472
public static final String OZONE_SCM_RATIS_PORT_KEY
476473
= "ozone.scm.ratis.port";
477474
public static final int OZONE_SCM_RATIS_PORT_DEFAULT

‎hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/SpanSampler.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ public SamplingResult shouldSample(
5959
// check if we have a valid parent span
6060
if (!parentSpan.getSpanContext().isValid()) {
6161
// Root span: always delegate to trace-level sampler
62-
// This ensures OTEL_TRACES_SAMPLER_ARG is respected
62+
// This ensures ozone.tracing.sampler is respected
6363
return rootSampler.shouldSample(parentContext, traceId, spanName,
6464
spanKind, attributes, parentLinks);
6565
}

‎hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingConfig.java‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.hadoop.hdds.tracing;
19+
20+
import org.apache.commons.lang3.StringUtils;
21+
import org.apache.hadoop.hdds.conf.Config;
22+
import org.apache.hadoop.hdds.conf.ConfigGroup;
23+
import org.apache.hadoop.hdds.conf.ConfigTag;
24+
import org.apache.hadoop.hdds.conf.ConfigType;
25+
import org.apache.hadoop.hdds.conf.PostConstruct;
26+
import org.apache.hadoop.hdds.conf.ReconfigurableConfig;
27+
28+
/**
29+
* OpenTelemetry tracing configuration for Ozone services.
30+
* Priority is given as such:
31+
* 1. Explicit configuration keys
32+
* 2. Environment variables
33+
* 3. Default values defined in this class
34+
*/
35+
@ConfigGroup(prefix = "ozone.tracing")
36+
public class TracingConfig extends ReconfigurableConfig {
37+
38+
private static final String OTEL_EXPORTER_OTLP_ENDPOINT = "OTEL_EXPORTER_OTLP_ENDPOINT";
39+
private static final String OTEL_TRACES_SAMPLER_ARG = "OTEL_TRACES_SAMPLER_ARG";
40+
private static final String OTEL_SPAN_SAMPLING_ARG = "OTEL_SPAN_SAMPLING_ARG";
41+
42+
@Config(
43+
key = "ozone.tracing.enabled",
44+
defaultValue = "false",
45+
type = ConfigType.BOOLEAN,
46+
tags = { ConfigTag.OZONE, ConfigTag.HDDS },
47+
description = "If true, tracing is initialized and spans may be exported (subject to sampling)."
48+
)
49+
private boolean tracingEnabled;
50+
51+
@Config(
52+
key = "ozone.tracing.endpoint",
53+
defaultValue = "",
54+
type = ConfigType.STRING,
55+
tags = { ConfigTag.OZONE, ConfigTag.HDDS },
56+
description = "OTLP gRPC receiver endpoint URL."
57+
)
58+
private String tracingEndpoint;
59+
60+
@Config(
61+
key = "ozone.tracing.sampler",
62+
defaultValue = "-1",
63+
type = ConfigType.DOUBLE,
64+
tags = { ConfigTag.OZONE, ConfigTag.HDDS },
65+
description = "Root trace sampling ratio (0.0 to 1.0)."
66+
)
67+
private double traceSamplerRatio;
68+
69+
@Config(
70+
key = "ozone.tracing.span.sampling",
71+
defaultValue = "",
72+
type = ConfigType.STRING,
73+
tags = { ConfigTag.OZONE, ConfigTag.HDDS },
74+
description = "Optional per-span sampling: comma-separated spanName:rate entries."
75+
)
76+
private String spanSampling;
77+
78+
public boolean isTracingEnabled() {
79+
return tracingEnabled;
80+
}
81+
82+
@PostConstruct
83+
public void validate() {
84+
if (tracingEndpoint.isEmpty()) {
85+
tracingEndpoint = System.getenv(OTEL_EXPORTER_OTLP_ENDPOINT);
86+
}
87+
if (StringUtils.isBlank(tracingEndpoint)) {
88+
tracingEndpoint = "http://localhost:4317";
89+
}
90+
91+
if (traceSamplerRatio < 0) {
92+
String envTraceRatio = System.getenv(OTEL_TRACES_SAMPLER_ARG);
93+
if (envTraceRatio != null) {
94+
try {
95+
traceSamplerRatio = Double.parseDouble(envTraceRatio.trim());
96+
} catch (NumberFormatException ignored) {
97+
}
98+
}
99+
}
100+
if (traceSamplerRatio < 0 || traceSamplerRatio > 1) {
101+
traceSamplerRatio = 1.0;
102+
}
103+
104+
if (spanSampling.isEmpty()) {
105+
String envSampling = System.getenv(OTEL_SPAN_SAMPLING_ARG);
106+
if (!StringUtils.isBlank(envSampling)) {
107+
spanSampling = envSampling;
108+
}
109+
}
110+
}
111+
112+
public String getTracingEndpoint() {
113+
return tracingEndpoint;
114+
}
115+
116+
public double getTraceSamplerRatio() {
117+
return traceSamplerRatio;
118+
}
119+
120+
public String getSpanSampling() {
121+
return spanSampling;
122+
}
123+
124+
}

‎hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingUtil.java‎

Lines changed: 11 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,6 @@
3737
import java.util.HashMap;
3838
import java.util.Map;
3939
import org.apache.hadoop.hdds.conf.ConfigurationSource;
40-
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
4140
import org.apache.ratis.util.function.CheckedRunnable;
4241
import org.apache.ratis.util.function.CheckedSupplier;
4342
import org.slf4j.Logger;
@@ -49,12 +48,6 @@
4948
public final class TracingUtil {
5049
private static final Logger LOG = LoggerFactory.getLogger(TracingUtil.class);
5150
private static final String NULL_SPAN_AS_STRING = "";
52-
private static final String OTEL_EXPORTER_OTLP_ENDPOINT = "OTEL_EXPORTER_OTLP_ENDPOINT";
53-
private static final String OTEL_EXPORTER_OTLP_ENDPOINT_DEFAULT = "http://localhost:4317";
54-
private static final String OTEL_TRACES_SAMPLER_ARG = "OTEL_TRACES_SAMPLER_ARG";
55-
private static final double OTEL_TRACES_SAMPLER_RATIO_DEFAULT = 1.0;
56-
private static final String OTEL_SPAN_SAMPLING_ARG = "OTEL_SPAN_SAMPLING_ARG";
57-
private static final String OTEL_TRACES_SAMPLER_CONFIG_DEFAULT = "";
5851

5952
private static volatile boolean isInit = false;
6053
private static Tracer tracer = OpenTelemetry.noop().getTracer("noop");
@@ -72,46 +65,23 @@ public static void initTracing(
7265
}
7366

7467
try {
75-
initialize(serviceName);
68+
initialize(serviceName, conf);
7669
isInit = true;
7770
LOG.info("Initialized tracing service: {}", serviceName);
7871
} catch (Exception e) {
7972
LOG.error("Failed to initialize tracing", e);
8073
}
8174
}
8275

83-
private static void initialize(String serviceName) {
84-
String otelEndPoint = System.getenv(OTEL_EXPORTER_OTLP_ENDPOINT);
85-
if (otelEndPoint == null || otelEndPoint.isEmpty()) {
86-
otelEndPoint = OTEL_EXPORTER_OTLP_ENDPOINT_DEFAULT;
87-
}
88-
89-
double samplerRatio = OTEL_TRACES_SAMPLER_RATIO_DEFAULT;
90-
try {
91-
String sampleStrRatio = System.getenv(OTEL_TRACES_SAMPLER_ARG);
92-
if (sampleStrRatio != null && !sampleStrRatio.isEmpty()) {
93-
samplerRatio = Double.parseDouble(System.getenv(OTEL_TRACES_SAMPLER_ARG));
94-
LOG.info("Sampling Trace Config = '{}'", samplerRatio);
95-
}
96-
} catch (NumberFormatException ex) {
97-
// log and use the default value.
98-
LOG.warn("Invalid value for {}: '{}'. Falling back to default: {}",
99-
OTEL_TRACES_SAMPLER_ARG, System.getenv(OTEL_TRACES_SAMPLER_ARG), OTEL_TRACES_SAMPLER_RATIO_DEFAULT, ex);
100-
}
76+
private static void initialize(String serviceName, ConfigurationSource conf) {
77+
//Fetch and log the right tracing parameters based on config, environment variable and default value priority.
78+
TracingConfig tracingConfig = conf.getObject(TracingConfig.class);
79+
String otelEndPoint = tracingConfig.getTracingEndpoint();
80+
double samplerRatio = tracingConfig.getTraceSamplerRatio();
81+
LOG.info("Sampling Trace Config = '{}'", samplerRatio);
82+
String spanSamplingConfig = tracingConfig.getSpanSampling();
83+
LOG.info("Sampling Span Config = '{}'", spanSamplingConfig);
10184

102-
String spanSamplingConfig = OTEL_TRACES_SAMPLER_CONFIG_DEFAULT;
103-
try {
104-
String spanStrConfig = System.getenv(OTEL_SPAN_SAMPLING_ARG);
105-
if (spanStrConfig != null && !spanStrConfig.isEmpty()) {
106-
spanSamplingConfig = spanStrConfig;
107-
}
108-
LOG.info("Sampling Span Config = '{}'", spanSamplingConfig);
109-
} catch (Exception ex) {
110-
// Log and use the default value.
111-
LOG.warn("Failed to process {}. Falling back to default configuration: {}",
112-
OTEL_SPAN_SAMPLING_ARG, OTEL_TRACES_SAMPLER_CONFIG_DEFAULT, ex);
113-
}
114-
// Pass the config to parseSpanSamplingConfig to get spans to be sampled.
11585
Map<String, LoopSampler> spanMap = parseSpanSamplingConfig(spanSamplingConfig);
11686

11787
Resource resource = Resource.create(Attributes.of(AttributeKey.stringKey("service.name"), serviceName));
@@ -201,11 +171,8 @@ public static <T> T createProxy(
201171
new TraceAllMethod<>(delegate, itf.getSimpleName())));
202172
}
203173

204-
public static boolean isTracingEnabled(
205-
ConfigurationSource conf) {
206-
return conf.getBoolean(
207-
ScmConfigKeys.HDDS_TRACING_ENABLED,
208-
ScmConfigKeys.HDDS_TRACING_ENABLED_DEFAULT);
174+
public static boolean isTracingEnabled(ConfigurationSource conf) {
175+
return conf.getObject(TracingConfig.class).isTracingEnabled();
209176
}
210177

211178
/**

‎hadoop-hdds/common/src/main/resources/ozone-default.xml‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3573,14 +3573,6 @@
35733573
If enabled, SCM DB Snapshot is taken by Recon.
35743574
</description>
35753575
</property>
3576-
<property>
3577-
<name>hdds.tracing.enabled</name>
3578-
<value>false</value>
3579-
<tag>OZONE, HDDS</tag>
3580-
<description>
3581-
If enabled, tracing information is sent to tracing server.
3582-
</description>
3583-
</property>
35843576
<property>
35853577
<name>ozone.recon.task.thread.count</name>
35863578
<value>1</value>

‎hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/tracing/TestTracingUtil.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@
2828
import java.io.IOException;
2929
import org.apache.hadoop.hdds.conf.InMemoryConfigurationForTesting;
3030
import org.apache.hadoop.hdds.conf.MutableConfigurationSource;
31-
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
3231
import org.apache.hadoop.hdds.tracing.TestTraceAllMethod.Service;
3332
import org.apache.hadoop.hdds.tracing.TestTraceAllMethod.ServiceImpl;
3433
import org.junit.jupiter.api.Test;
@@ -58,7 +57,7 @@ public void testInitTracing() {
5857

5958
private static MutableConfigurationSource tracingEnabled() {
6059
MutableConfigurationSource config = new InMemoryConfigurationForTesting();
61-
config.setBoolean(ScmConfigKeys.HDDS_TRACING_ENABLED, true);
60+
config.setBoolean("ozone.tracing.enabled", true);
6261
return config;
6362
}
6463

0 commit comments

Comments
 (0)