Report gRPC status code in client-computed stats

dd-java-agent/instrumentation/armeria/armeria-grpc-0.84/src/main/java/datadog/trace/instrumentation/armeria/grpc/client/GrpcClientDecorator.java

@@ -15,6 +15,7 @@
 import datadog.trace.api.datastreams.DataStreamsTags;
 import datadog.trace.api.naming.SpanNaming;
 import datadog.trace.bootstrap.instrumentation.api.AgentSpan;
+import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags;
 import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes;
 import datadog.trace.bootstrap.instrumentation.api.Tags;
 import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString;
@@ -116,6 +117,7 @@ public AgentSpan onClose(final AgentSpan span, final Status status) {
 
     span.setTag("status.code", status.getCode().name());
     span.setTag("grpc.status.code", status.getCode().name());
+    span.setTag(InstrumentationTags.GRPC_STATUS_CODE, status.getCode().value());
     span.setTag("status.description", status.getDescription());
 
     // TODO why is there a mismatch between client / server for calling the onError method?

dd-java-agent/instrumentation/armeria/armeria-grpc-0.84/src/main/java/datadog/trace/instrumentation/armeria/grpc/server/GrpcServerDecorator.java

@@ -9,6 +9,7 @@
 import datadog.trace.api.naming.SpanNaming;
 import datadog.trace.bootstrap.instrumentation.api.AgentSpan;
 import datadog.trace.bootstrap.instrumentation.api.ErrorPriorities;
+import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags;
 import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes;
 import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString;
 import datadog.trace.bootstrap.instrumentation.decorator.ServerDecorator;
@@ -97,6 +98,7 @@ public <RespT, ReqT> AgentSpan onCall(final AgentSpan span, ServerCall<ReqT, Res
   public AgentSpan onStatus(final AgentSpan span, final Status status) {
     span.setTag("status.code", status.getCode().name());
     span.setTag("grpc.status.code", status.getCode().name());
+    span.setTag(InstrumentationTags.GRPC_STATUS_CODE, status.getCode().value());
     span.setTag("status.description", status.getDescription());
     return span.setError(
         SERVER_ERROR_STATUSES.get(status.getCode().value()), ErrorPriorities.HTTP_SERVER_DECORATOR);

dd-java-agent/instrumentation/armeria/armeria-grpc-0.84/src/test/groovy/ArmeriaGrpcStreamingTest.groovy

@@ -7,6 +7,7 @@ import com.linecorp.armeria.server.grpc.GrpcService
 import com.linecorp.armeria.testing.junit4.server.ServerRule
 import datadog.trace.agent.test.naming.VersionedNamingTestBase
 import datadog.trace.api.DDSpanTypes
+import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags
 import datadog.trace.bootstrap.instrumentation.api.Tags
 import example.GreeterGrpc
 import example.Helloworld
@@ -178,6 +179,7 @@ abstract class ArmeriaGrpcStreamingTest extends VersionedNamingTestBase {
             "$Tags.RPC_SERVICE" "example.Greeter"
             "status.code" "OK"
             "grpc.status.code" "OK"
+            "$InstrumentationTags.GRPC_STATUS_CODE" 0
             "request.type" "example.Helloworld\$Response"
             "response.type" "example.Helloworld\$Response"
             peerServiceFrom(Tags.RPC_SERVICE)
@@ -215,6 +217,7 @@ abstract class ArmeriaGrpcStreamingTest extends VersionedNamingTestBase {
             "$Tags.SPAN_KIND" Tags.SPAN_KIND_SERVER
             "status.code" "OK"
             "grpc.status.code" "OK"
+            "$InstrumentationTags.GRPC_STATUS_CODE" 0
             defaultTags(true)
           }
         }

dd-java-agent/instrumentation/armeria/armeria-grpc-0.84/src/test/groovy/ArmeriaGrpcTest.groovy

@@ -18,6 +18,7 @@ import datadog.trace.api.gateway.RequestContext
 import datadog.trace.api.gateway.RequestContextSlot
 import datadog.trace.bootstrap.instrumentation.api.AgentPropagation
 import datadog.trace.bootstrap.instrumentation.api.AgentTracer
+import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags
 import datadog.trace.bootstrap.instrumentation.api.Tags
 import datadog.trace.core.datastreams.StatsGroup
 import datadog.trace.instrumentation.armeria.grpc.server.GrpcExtractAdapter
@@ -182,6 +183,7 @@ abstract class ArmeriaGrpcTest extends VersionedNamingTestBase {
             "$Tags.RPC_SERVICE" "example.Greeter"
             "status.code" "OK"
             "grpc.status.code" "OK"
+            "$InstrumentationTags.GRPC_STATUS_CODE" 0
             "request.type" "example.Helloworld\$Request"
             "response.type" "example.Helloworld\$Response"
             if ({ isDataStreamsEnabled() }) {
@@ -221,6 +223,7 @@ abstract class ArmeriaGrpcTest extends VersionedNamingTestBase {
             "$Tags.SPAN_KIND" Tags.SPAN_KIND_SERVER
             "status.code" "OK"
             "grpc.status.code" "OK"
+            "$InstrumentationTags.GRPC_STATUS_CODE" 0
             if ({ isDataStreamsEnabled() }) {
               "$DDTags.PATHWAY_HASH" { String }
             }
@@ -319,6 +322,7 @@ abstract class ArmeriaGrpcTest extends VersionedNamingTestBase {
             "$Tags.RPC_SERVICE" "example.Greeter"
             "status.code" "${status.code.name()}"
             "grpc.status.code" "${status.code.name()}"
+            "$InstrumentationTags.GRPC_STATUS_CODE" status.code.value()
             "status.description" description
             "request.type" "example.Helloworld\$Request"
             "response.type" "example.Helloworld\$Response"
@@ -343,6 +347,7 @@ abstract class ArmeriaGrpcTest extends VersionedNamingTestBase {
             "$Tags.SPAN_KIND" Tags.SPAN_KIND_SERVER
             "status.code" "${status.code.name()}"
             "grpc.status.code" "${status.code.name()}"
+            "$InstrumentationTags.GRPC_STATUS_CODE" status.code.value()
             "status.description" description
             "canceled" { true } // 1.0.0 handles cancellation incorrectly so accesting any value
             if (status.cause != null) {
@@ -432,6 +437,7 @@ abstract class ArmeriaGrpcTest extends VersionedNamingTestBase {
             "$Tags.RPC_SERVICE" "example.Greeter"
             "status.code" status.code.name()
             "grpc.status.code" status.code.name()
+            "$InstrumentationTags.GRPC_STATUS_CODE" status.code.value()
             if (status.description != null) {
               "status.description" status.description
             }
@@ -459,6 +465,7 @@ abstract class ArmeriaGrpcTest extends VersionedNamingTestBase {
             errorTags error.class, error.message
             "status.code" "${status.code.name()}"
             "grpc.status.code" "${status.code.name()}"
+            "$InstrumentationTags.GRPC_STATUS_CODE" status.code.value()
             "status.description"  { it == null || String}
             "canceled" { true } // 1.0.0 handles cancellation incorrectly so accesting any value
             if ({ isDataStreamsEnabled() }) {
@@ -574,6 +581,7 @@ abstract class ArmeriaGrpcTest extends VersionedNamingTestBase {
             "$Tags.SPAN_KIND" Tags.SPAN_KIND_SERVER
             "status.code" "OK"
             "grpc.status.code" "OK"
+            "$InstrumentationTags.GRPC_STATUS_CODE" 0
             if ({ isDataStreamsEnabled() }) {
               "$DDTags.PATHWAY_HASH" { String }
             }
@@ -650,6 +658,7 @@ abstract class ArmeriaGrpcTest extends VersionedNamingTestBase {
             "$Tags.RPC_SERVICE" "example.Greeter"
             "status.code" "OK"
             "grpc.status.code" "OK"
+            "$InstrumentationTags.GRPC_STATUS_CODE" 0
             "request.type" "example.Helloworld\$Request"
             "response.type" "example.Helloworld\$Response"
             if ({ isDataStreamsEnabled() }) {

dd-java-agent/instrumentation/google-pubsub-1.116/src/test/groovy/PubSubTest.groovy

@@ -27,6 +27,7 @@ import datadog.trace.api.DDSpanTypes
 import datadog.trace.api.DDTags
 import datadog.trace.api.config.GeneralConfig
 import datadog.trace.api.config.TraceInstrumentationConfig
+import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags
 import datadog.trace.bootstrap.instrumentation.api.Tags
 import datadog.trace.core.DDSpan
 import datadog.trace.core.datastreams.StatsGroup
@@ -284,6 +285,7 @@ abstract class PubSubTest extends VersionedNamingTestBase {
         "$Tags.RPC_SERVICE" { String }
         "status.code" { String }
         "grpc.status.code" { String }
+        "$InstrumentationTags.GRPC_STATUS_CODE" { Integer }
         if ({ isDataStreamsEnabled() }) {
           "$DDTags.PATHWAY_HASH" { String }
         }

dd-java-agent/instrumentation/grpc-1.5/src/main/java/datadog/trace/instrumentation/grpc/client/GrpcClientDecorator.java

@@ -16,6 +16,7 @@
 import datadog.trace.api.datastreams.DataStreamsContext;
 import datadog.trace.api.naming.SpanNaming;
 import datadog.trace.bootstrap.instrumentation.api.AgentSpan;
+import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags;
 import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes;
 import datadog.trace.bootstrap.instrumentation.api.Tags;
 import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString;
@@ -116,6 +117,7 @@ public <C> void injectContext(Context context, final C request, CarrierSetter<C>
   public AgentSpan onClose(final AgentSpan span, final Status status) {
     span.setTag("status.code", status.getCode().name());
     span.setTag("grpc.status.code", status.getCode().name());
+    span.setTag(InstrumentationTags.GRPC_STATUS_CODE, status.getCode().value());
     span.setTag("status.description", status.getDescription());
 
     // TODO why is there a mismatch between client / server for calling the onError method?

dd-java-agent/instrumentation/grpc-1.5/src/main/java/datadog/trace/instrumentation/grpc/server/GrpcServerDecorator.java

@@ -10,6 +10,7 @@
 import datadog.trace.api.naming.SpanNaming;
 import datadog.trace.bootstrap.instrumentation.api.AgentSpan;
 import datadog.trace.bootstrap.instrumentation.api.ErrorPriorities;
+import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags;
 import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes;
 import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString;
 import datadog.trace.bootstrap.instrumentation.decorator.ServerDecorator;
@@ -98,6 +99,7 @@ public <RespT, ReqT> AgentSpan onCall(final AgentSpan span, ServerCall<ReqT, Res
   public AgentSpan onStatus(final AgentSpan span, final Status status) {
     span.setTag("status.code", status.getCode().name());
     span.setTag("grpc.status.code", status.getCode().name());
+    span.setTag(InstrumentationTags.GRPC_STATUS_CODE, status.getCode().value());
     span.setTag("status.description", status.getDescription());
     return span.setError(
         SERVER_ERROR_STATUSES.get(status.getCode().value()), ErrorPriorities.HTTP_SERVER_DECORATOR);

dd-java-agent/instrumentation/grpc-1.5/src/test/groovy/GrpcStreamingTest.groovy

@@ -1,6 +1,7 @@
 import com.google.common.util.concurrent.MoreExecutors
 import datadog.trace.agent.test.naming.VersionedNamingTestBase
 import datadog.trace.api.DDSpanTypes
+import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags
 import datadog.trace.bootstrap.instrumentation.api.Tags
 import example.GreeterGrpc
 import example.Helloworld
@@ -162,6 +163,7 @@ abstract class GrpcStreamingTest extends VersionedNamingTestBase {
             "$Tags.RPC_SERVICE" "example.Greeter"
             "status.code" "OK"
             "grpc.status.code" "OK"
+            "$InstrumentationTags.GRPC_STATUS_CODE" 0
             "request.type" "example.Helloworld\$Response"
             "response.type" "example.Helloworld\$Response"
             peerServiceFrom(Tags.RPC_SERVICE)
@@ -198,6 +200,7 @@ abstract class GrpcStreamingTest extends VersionedNamingTestBase {
             "$Tags.SPAN_KIND" Tags.SPAN_KIND_SERVER
             "status.code" "OK"
             "grpc.status.code" "OK"
+            "$InstrumentationTags.GRPC_STATUS_CODE" 0
             defaultTags(true)
           }
         }

dd-java-agent/instrumentation/grpc-1.5/src/test/groovy/GrpcTest.groovy

@@ -13,6 +13,7 @@ import datadog.trace.api.gateway.RequestContext
 import datadog.trace.api.gateway.RequestContextSlot
 import datadog.trace.bootstrap.instrumentation.api.AgentPropagation
 import datadog.trace.bootstrap.instrumentation.api.AgentTracer
+import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags
 import datadog.trace.bootstrap.instrumentation.api.Tags
 import datadog.trace.core.datastreams.StatsGroup
 import datadog.trace.instrumentation.grpc.server.GrpcExtractAdapter
@@ -173,6 +174,7 @@ abstract class GrpcTest extends VersionedNamingTestBase {
             "$Tags.PEER_PORT" server.port
             "status.code" "OK"
             "grpc.status.code" "OK"
+            "$InstrumentationTags.GRPC_STATUS_CODE" 0
             "request.type" "example.Helloworld\$Request"
             "response.type" "example.Helloworld\$Response"
             if ({ isDataStreamsEnabled() }) {
@@ -212,6 +214,7 @@ abstract class GrpcTest extends VersionedNamingTestBase {
             "$Tags.SPAN_KIND" Tags.SPAN_KIND_SERVER
             "status.code" "OK"
             "grpc.status.code" "OK"
+            "$InstrumentationTags.GRPC_STATUS_CODE" 0
             if ({ isDataStreamsEnabled() }) {
               "$DDTags.PATHWAY_HASH" { String }
             }
@@ -317,6 +320,7 @@ abstract class GrpcTest extends VersionedNamingTestBase {
             "$Tags.RPC_SERVICE" "example.Greeter"
             "status.code" "${status.code.name()}"
             "grpc.status.code" "${status.code.name()}"
+            "$InstrumentationTags.GRPC_STATUS_CODE" status.code.value()
             "status.description" description
             "request.type" "example.Helloworld\$Request"
             "response.type" "example.Helloworld\$Response"
@@ -341,6 +345,7 @@ abstract class GrpcTest extends VersionedNamingTestBase {
             "$Tags.SPAN_KIND" Tags.SPAN_KIND_SERVER
             "status.code" "${status.code.name()}"
             "grpc.status.code" "${status.code.name()}"
+            "$InstrumentationTags.GRPC_STATUS_CODE" status.code.value()
             "status.description" description
             if (status.cause != null) {
               errorTags status.cause.class, status.cause.message
@@ -424,6 +429,7 @@ abstract class GrpcTest extends VersionedNamingTestBase {
             "$Tags.PEER_PORT" server.port
             "status.code" "UNKNOWN"
             "grpc.status.code" "UNKNOWN"
+            "$InstrumentationTags.GRPC_STATUS_CODE" 2
             "request.type" "example.Helloworld\$Request"
             "response.type" "example.Helloworld\$Response"
             "status.description" { it == null || String}
@@ -448,6 +454,7 @@ abstract class GrpcTest extends VersionedNamingTestBase {
             "$Tags.SPAN_KIND" Tags.SPAN_KIND_SERVER
             "status.code" "${status.code.name()}"
             "grpc.status.code" "${status.code.name()}"
+            "$InstrumentationTags.GRPC_STATUS_CODE" status.code.value()
             "status.description"  { it == null || String}
             errorTags error.class, error.message
             if ({ isDataStreamsEnabled() }) {
@@ -554,6 +561,7 @@ abstract class GrpcTest extends VersionedNamingTestBase {
             "$Tags.SPAN_KIND" Tags.SPAN_KIND_SERVER
             "status.code" "OK"
             "grpc.status.code" "OK"
+            "$InstrumentationTags.GRPC_STATUS_CODE" 0
             if ({ isDataStreamsEnabled() }) {
               "$DDTags.PATHWAY_HASH" { String }
             }
@@ -621,6 +629,7 @@ abstract class GrpcTest extends VersionedNamingTestBase {
             "$Tags.RPC_SERVICE" "example.Greeter"
             "status.code" "OK"
             "grpc.status.code" "OK"
+            "$InstrumentationTags.GRPC_STATUS_CODE" 0
             "request.type" "example.Helloworld\$Request"
             "response.type" "example.Helloworld\$Response"
             if ({ isDataStreamsEnabled() }) {

dd-trace-core/src/main/java/datadog/trace/common/metrics/ConflatingMetricsAggregator.java

@@ -1,6 +1,7 @@
 package datadog.trace.common.metrics;
 
 import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V06_METRICS_ENDPOINT;
+import static datadog.trace.api.DDSpanTypes.RPC;
 import static datadog.trace.api.DDTags.BASE_SERVICE;
 import static datadog.trace.api.Functions.UTF8_ENCODE;
 import static datadog.trace.bootstrap.instrumentation.api.Tags.HTTP_ENDPOINT;
@@ -29,6 +30,7 @@
 import datadog.trace.api.WellKnownTags;
 import datadog.trace.api.cache.DDCache;
 import datadog.trace.api.cache.DDCaches;
+import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags;
 import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString;
 import datadog.trace.common.metrics.SignalItem.ReportSignal;
 import datadog.trace.common.writer.ddagent.DDAgentApi;
@@ -326,21 +328,28 @@ private boolean publish(CoreSpan<?> span, boolean isTopLevel, CharSequence spanK
       httpEndpoint = httpEndpointObj != null ? httpEndpointObj.toString() : null;
     }
 
+    CharSequence spanType = span.getType();
+    String grpcStatusCode = null;
+    if (spanType != null && RPC.contentEquals(spanType)) {
+      Object grpcStatusObj = span.unsafeGetTag(InstrumentationTags.GRPC_STATUS_CODE);
+      grpcStatusCode = grpcStatusObj != null ? grpcStatusObj.toString() : null;
+    }
     MetricKey newKey =
         new MetricKey(
             span.getResourceName(),
             SERVICE_NAMES.computeIfAbsent(span.getServiceName(), UTF8_ENCODE),
             span.getOperationName(),
             span.getServiceNameSource(),
-            span.getType(),
+            spanType,
             span.getHttpStatusCode(),
             isSynthetic(span),
             span.getParentId() == 0,
             SPAN_KINDS.computeIfAbsent(
                 spanKind, UTF8BytesString::create), // save repeated utf8 conversions
             getPeerTags(span, spanKind.toString()),
             httpMethod,
-            httpEndpoint);
+            httpEndpoint,
+            grpcStatusCode);
     MetricKey key = keys.putIfAbsent(newKey, newKey);
     if (null == key) {
       key = newKey;

dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricKey.java

@@ -22,6 +22,8 @@ public final class MetricKey {
   static final DDCache<String, UTF8BytesString> HTTP_METHOD_CACHE = DDCaches.newFixedSizeCache(8);
   static final DDCache<String, UTF8BytesString> HTTP_ENDPOINT_CACHE =
       DDCaches.newFixedSizeCache(32);
+  static final DDCache<String, UTF8BytesString> GRPC_STATUS_CODE_CACHE =
+      DDCaches.newFixedSizeCache(32);
 
   private final UTF8BytesString resource;
   private final UTF8BytesString service;
@@ -36,6 +38,7 @@ public final class MetricKey {
   private final List<UTF8BytesString> peerTags;
   private final UTF8BytesString httpMethod;
   private final UTF8BytesString httpEndpoint;
+  private final UTF8BytesString grpcStatusCode;
 
   public MetricKey(
       CharSequence resource,
@@ -49,7 +52,8 @@ public MetricKey(
       CharSequence spanKind,
       List<UTF8BytesString> peerTags,
       CharSequence httpMethod,
-      CharSequence httpEndpoint) {
+      CharSequence httpEndpoint,
+      CharSequence grpcStatusCode) {
     this.resource = null == resource ? EMPTY : utf8(RESOURCE_CACHE, resource);
     this.service = null == service ? EMPTY : utf8(SERVICE_CACHE, service);
     this.serviceSource = null == serviceSource ? null : utf8(SERVICE_SOURCE_CACHE, serviceSource);
@@ -62,6 +66,8 @@ public MetricKey(
     this.peerTags = peerTags == null ? Collections.emptyList() : peerTags;
     this.httpMethod = httpMethod == null ? null : utf8(HTTP_METHOD_CACHE, httpMethod);
     this.httpEndpoint = httpEndpoint == null ? null : utf8(HTTP_ENDPOINT_CACHE, httpEndpoint);
+    this.grpcStatusCode =
+        grpcStatusCode == null ? null : utf8(GRPC_STATUS_CODE_CACHE, grpcStatusCode);
 
     int tmpHash = 0;
     tmpHash = HashingUtils.addToHash(tmpHash, this.isTraceRoot);
@@ -76,6 +82,7 @@ public MetricKey(
     tmpHash = HashingUtils.addToHash(tmpHash, this.serviceSource);
     tmpHash = HashingUtils.addToHash(tmpHash, this.httpEndpoint);
     tmpHash = HashingUtils.addToHash(tmpHash, this.httpMethod);
+    tmpHash = HashingUtils.addToHash(tmpHash, this.grpcStatusCode);
     this.hash = tmpHash;
   }
 
@@ -135,6 +142,10 @@ public UTF8BytesString getHttpEndpoint() {
     return httpEndpoint;
   }
 
+  public UTF8BytesString getGrpcStatusCode() {
+    return grpcStatusCode;
+  }
+
   @Override
   public boolean equals(Object o) {
     if (this == o) {
@@ -154,7 +165,8 @@ public boolean equals(Object o) {
           && peerTags.equals(metricKey.peerTags)
           && Objects.equals(serviceSource, metricKey.serviceSource)
           && Objects.equals(httpMethod, metricKey.httpMethod)
-          && Objects.equals(httpEndpoint, metricKey.httpEndpoint);
+          && Objects.equals(httpEndpoint, metricKey.httpEndpoint)
+          && Objects.equals(grpcStatusCode, metricKey.grpcStatusCode);
     }
     return false;
   }