hypertrace · JBAhire · Apr 15, 2021 · Apr 15, 2021 · Apr 15, 2021 · Apr 15, 2021
@@ -11,7 +11,7 @@ public class EnrichedSpanConstants {
   public static final String SPACE_IDS_ATTRIBUTE = "SPACE_IDS";
   public static final String API_EXIT_CALLS_ATTRIBUTE = "API_EXIT_CALLS";
   public static final String API_CALLEE_NAME_COUNT_ATTRIBUTE = "API_CALLEE_NAME_COUNT";
-
+  public static final String API_TRACE_ERROR_SPAN_COUNT_ATTRIBUTE = "API_TRACE_ERROR_SPAN_COUNT";
   /**
    * Returns the constant value for the given Enum.
    *

@@ -2,10 +2,12 @@
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Optional;
 import org.hypertrace.core.datamodel.Event;
 import org.hypertrace.core.datamodel.MetricValue;
 import org.hypertrace.core.datamodel.Metrics;
 import org.hypertrace.core.datamodel.StructuredTrace;
+import org.hypertrace.core.datamodel.shared.ApiNode;
 import org.hypertrace.core.datamodel.shared.trace.AttributeValueCreator;
 import org.hypertrace.core.datamodel.shared.trace.MetricValueCreator;
 import org.hypertrace.semantic.convention.utils.error.ErrorSemanticConventionUtils;
@@ -15,6 +17,7 @@
 import org.hypertrace.traceenricher.enrichedspan.constants.v1.CommonAttribute;
 import org.hypertrace.traceenricher.enrichedspan.constants.v1.ErrorMetrics;
 import org.hypertrace.traceenricher.enrichment.AbstractTraceEnricher;
+import org.hypertrace.traceenricher.trace.util.ApiTraceGraph;
 import org.hypertrace.traceenricher.util.Constants;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -64,11 +67,7 @@ private void enrichExceptionDetails(Event event) {
 
   private void enrichErrorDetails(Event event) {
     // Figure out if there are any errors in the event.
-    boolean hasError =
-        ErrorSemanticConventionUtils.checkForError(event)
-            || ErrorSemanticConventionUtils.checkForException(event)
-            || Constants.getEnrichedSpanConstant(ApiStatus.API_STATUS_FAIL)
-                .equals(EnrichedSpanUtils.getStatus(event));
+    boolean hasError = findIfEventHasError(event);
 
     if (hasError) {
       if (event.getMetrics() == null) {
@@ -86,12 +85,32 @@ private void enrichErrorDetails(Event event) {
     }
   }
 
+  private boolean findIfEventHasError(Event event) {
+    return ErrorSemanticConventionUtils.checkForError(event)
+        || ErrorSemanticConventionUtils.checkForException(event)
+        || Constants.getEnrichedSpanConstant(ApiStatus.API_STATUS_FAIL)
+            .equals(EnrichedSpanUtils.getStatus(event));
+  }
+
   @Override
   public void enrichTrace(StructuredTrace trace) {
     // TODO: There could be other cases where the client which is initiating this transaction
     //  has errored out but the entry span in transaction might be fine (server responded but
     //  client couldn't process it). Those cases should be handled in future.
 
+    ApiTraceGraph apiTraceGraph = new ApiTraceGraph(trace);
+    for (ApiNode<Event> apiNode : apiTraceGraph.getApiNodeList()) {
+      Optional<Event> entryEvent = apiNode.getEntryApiBoundaryEvent();
+      int apiTraceErrorCount =
+          (int) apiNode.getEvents().stream().filter(this::findIfEventHasError).count();
+      if (entryEvent.isPresent()) {
+        addEnrichedAttribute(
+            entryEvent.get(),
+            EnrichedSpanConstants.API_TRACE_ERROR_SPAN_COUNT_ATTRIBUTE,
+            AttributeValueCreator.create(apiTraceErrorCount));
+      }
+    }
+
     // Find the earliest Event from this trace and check if that's an ENTRY type.
     Event earliestEvent = getEarliestEvent(trace);
 
@@ -113,7 +132,6 @@ public void enrichTrace(StructuredTrace trace) {
                 AttributeValueCreator.create(true));
       }
     }
-
     // Count the no. of errors and exceptions in this trace overall. These need not have caused
     // the trace to error out but it's a good metric to track anyways.
     // Trace is considered to have an error if there is an error in the entry span only.

@@ -3,10 +3,16 @@
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-import com.google.common.collect.Lists;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.Map;
 import org.hypertrace.core.datamodel.AttributeValue;
+import org.hypertrace.core.datamodel.Attributes;
 import org.hypertrace.core.datamodel.Event;
+import org.hypertrace.core.datamodel.MetricValue;
+import org.hypertrace.core.datamodel.Metrics;
 import org.hypertrace.core.datamodel.StructuredTrace;
 import org.hypertrace.core.datamodel.shared.trace.AttributeValueCreator;
 import org.hypertrace.core.semantic.convention.constants.error.OTelErrorSemanticConventions;
@@ -29,15 +35,10 @@ public void noAttributes() {
     Event e = mock(Event.class);
     when(e.getAttributes()).thenReturn(null);
     enricher.enrichEvent(null, e);
-
     e = createMockEvent();
     enricher.enrichEvent(null, e);
 
-    StructuredTrace trace = mock(StructuredTrace.class);
-    when(trace.getAttributes()).thenReturn(null);
-    enricher.enrichTrace(trace);
-
-    trace = createMockStructuredTrace();
+    StructuredTrace trace = createStructuredTrace("test-id", createMockTestEvent());
     enricher.enrichTrace(trace);
   }
 
@@ -81,7 +82,7 @@ public void test_successStatus_shouldNotGetError() {
   @Test
   public void errorExists() {
     ErrorsAndExceptionsEnricher enricher = new ErrorsAndExceptionsEnricher();
-    Event e1 = createMockEvent();
+    Event e1 = createMockTestEvent();
     Map<String, AttributeValue> attributeValueMap = e1.getAttributes().getAttributeMap();
     attributeValueMap.put(
         Constants.getRawSpanConstant(Error.ERROR_ERROR), AttributeValueCreator.create(true));
@@ -93,7 +94,7 @@ public void errorExists() {
             .get(Constants.getEnrichedSpanConstant(ErrorMetrics.ERROR_METRICS_ERROR_COUNT))
             .getValue());
 
-    Event e2 = createMockEvent();
+    Event e2 = createMockTestEvent();
     attributeValueMap = e2.getAttributes().getAttributeMap();
     attributeValueMap.put(
         Constants.getRawSpanConstant(Error.ERROR_STACK_TRACE),
@@ -106,7 +107,7 @@ public void errorExists() {
             .get(Constants.getEnrichedSpanConstant(ErrorMetrics.ERROR_METRICS_EXCEPTION_COUNT))
             .getValue());
 
-    Event e3 = createMockEvent();
+    Event e3 = createMockTestEvent();
     attributeValueMap = e3.getAttributes().getAttributeMap();
     attributeValueMap.put(
         Constants.getRawSpanConstant(Error.ERROR_ERROR).toLowerCase(),
@@ -119,7 +120,7 @@ public void errorExists() {
             .get(Constants.getEnrichedSpanConstant(ErrorMetrics.ERROR_METRICS_ERROR_COUNT))
             .getValue());
 
-    Event e4 = createMockEvent();
+    Event e4 = createMockTestEvent();
     attributeValueMap = e4.getAttributes().getAttributeMap();
     attributeValueMap.put(
         Constants.getRawSpanConstant(Error.ERROR_ERROR).toLowerCase(),
@@ -132,7 +133,7 @@ public void errorExists() {
             .get(Constants.getEnrichedSpanConstant(ErrorMetrics.ERROR_METRICS_ERROR_COUNT))
             .getValue());
 
-    Event e5 = createMockEvent();
+    Event e5 = createMockTestEvent();
     attributeValueMap = e5.getAttributes().getAttributeMap();
     attributeValueMap.put(
         OTelErrorSemanticConventions.EXCEPTION_TYPE.getValue().toLowerCase(),
@@ -145,7 +146,7 @@ public void errorExists() {
             .get(Constants.getEnrichedSpanConstant(ErrorMetrics.ERROR_METRICS_ERROR_COUNT))
             .getValue());
 
-    Event e6 = createMockEvent();
+    Event e6 = createMockTestEvent();
     attributeValueMap = e6.getAttributes().getAttributeMap();
     attributeValueMap.put(
         OTelErrorSemanticConventions.EXCEPTION_STACKTRACE.getValue().toLowerCase(),
@@ -158,14 +159,15 @@ public void errorExists() {
             .get(Constants.getEnrichedSpanConstant(ErrorMetrics.ERROR_METRICS_EXCEPTION_COUNT))
             .getValue());
 
-    StructuredTrace trace = createMockStructuredTrace();
-    when(trace.getEventList()).thenReturn(Lists.newArrayList(e1, e2, e3, e4, e5, e6));
+    StructuredTrace trace = createStructuredTrace("test-id", e1, e2, e3, e4, e5, e6);
+
     enricher.enrichEvent(trace, e1);
     enricher.enrichEvent(trace, e2);
     enricher.enrichEvent(trace, e3);
     enricher.enrichEvent(trace, e4);
     enricher.enrichEvent(trace, e5);
     enricher.enrichEvent(trace, e6);
+    trace.setMetrics(Metrics.newBuilder().setMetricMap(new HashMap<>()).build());
     Assertions.assertEquals(
         1.0d,
         e4.getMetrics()
@@ -193,7 +195,7 @@ public void errorExists() {
             .getValue());
 
     // Trace itself doesn't have an error since there was no entry span.
-    Assertions.assertFalse(
+    Assertions.assertTrue(
         trace
             .getAttributes()
             .getAttributeMap()
@@ -237,4 +239,28 @@ public void errorExists() {
                     CommonAttribute.COMMON_ATTRIBUTE_TRANSACTION_HAS_ERROR))
             .getValue());
   }
+
+  private Event createMockTestEvent() {
+    Map<String, AttributeValue> map = new HashMap<>();
+    Map<String, MetricValue> metricMap = new HashMap<>();
+    map.put("span.kind", AttributeValue.newBuilder().setValue("server").build());
+    map.put("error", AttributeValue.newBuilder().setValue("true").build());
+    map.put("SPAN_TYPE", AttributeValue.newBuilder().setValue("ENTRY").build());
+    metricMap.put("Duration", MetricValue.newBuilder().setValue(4.0).build());
+    Event event =
+        Event.newBuilder()
+            .setCustomerId("customer1")
+            .setEventId(ByteBuffer.wrap("bdf03dfabf5c70f9".getBytes()))
+            .setEntityIdList(Arrays.asList("4bfca8f7-4974-36a4-9385-dd76bf5c8824"))
+            .setEnrichedAttributes(Attributes.newBuilder().setAttributeMap(map).build())
+            .setAttributes(Attributes.newBuilder().setAttributeMap(map).build())
+            .setEventName("test-event")
+            .setStartTimeMillis(1566869077746L)
+            .setEndTimeMillis(1566869077750L)
+            .setMetrics(Metrics.newBuilder().setMetricMap(metricMap).build())
+            .setEventRefList(Collections.emptyList())
+            .build();
+
+    return event;
+  }
 }
@@ -95,6 +95,14 @@
               "value_list": null,
               "value_map": null
             },
+            "API_TRACE_ERROR_SPAN_COUNT": {
+              "value": {
+                "string": "0"
+              },
+              "binary_value": null,
+              "value_list": null,
+              "value_map": null
+            },
             "SERVICE_NAME": {
               "value": {
                 "string": "api_01"

@@ -71,7 +71,7 @@ enricher {
 
   ErrorsAndExceptionsEnricher {
     class = "org.hypertrace.traceenricher.enrichment.enrichers.ErrorsAndExceptionsEnricher"
-    dependencies = ["ApiStatusEnricher"]
+    dependencies = ["ApiStatusEnricher", "ApiBoundaryTypeAttributeEnricher"]
   }
 
   BackendEntityEnricher {

@@ -17,7 +17,7 @@ pinot.controllerHost = pinot-controller
 pinot.controllerPort = 9000
 pinot.timeColumn = start_time_millis
 pinot.timeUnit = MILLISECONDS
-pinot.dimensionColumns = [tenant_id, span_id, span_kind, parent_span_id, trace_id, service_id, api_id, api_name, entry_api_id, protocol_name, tags__KEYS, tags__VALUES, status_code, start_time_millis, end_time_millis, duration_millis, api_trace_id, service_name, api_boundary_type, event_name, status_message, status, api_trace_count, display_entity_name, display_span_name, request_url, error_count, api_discovery_state, exception_count, space_ids, api_exit_calls, api_callee_name_count__KEYS, api_callee_name_count__VALUES]
+pinot.dimensionColumns = [tenant_id, span_id, span_kind, parent_span_id, trace_id, service_id, api_id, api_name, entry_api_id, protocol_name, tags__KEYS, tags__VALUES, status_code, start_time_millis, end_time_millis, duration_millis, api_trace_id, service_name, api_boundary_type, event_name, status_message, status, api_trace_count, display_entity_name, display_span_name, request_url, error_count, api_discovery_state, exception_count, space_ids, api_exit_calls, api_callee_name_count__KEYS, api_callee_name_count__VALUES, api_trace_error_span_count]
 pinot.columnsMaxLength={}
 pinot.metricColumns = []
 pinot.invertedIndexColumns= [tags__KEYS, tags__VALUES]

@@ -101,5 +101,7 @@ protocol SpanEventViewProtocol {
 
     // map of api callee name -> incoming call count
     map<string> api_callee_name_count = {};
+
+    int api_trace_error_span_count = 0;
   }
 }
@@ -238,6 +238,11 @@ private SpanEventView.Builder generateViewBuilder(
             SpanAttributeUtils.getStringAttributeWithDefault(
                 event, EnrichedSpanConstants.API_EXIT_CALLS_ATTRIBUTE, "0")));
 
+    builder.setApiTraceErrorSpanCount(
+        Integer.parseInt(
+            SpanAttributeUtils.getStringAttributeWithDefault(
+                event, EnrichedSpanConstants.API_TRACE_ERROR_SPAN_COUNT_ATTRIBUTE, "0")));
+
     return builder;
   }
 

@@ -217,4 +217,72 @@ public void testExitCallsInfo() {
     assertEquals(calleeNameCount, list.get(0).getApiCalleeNameCount());
     assertEquals(5, list.get(0).getApiExitCalls());
   }
+
+  @Test
+  public void testApiTraceErrorSpanCount() {
+    StructuredTrace.Builder traceBuilder = StructuredTrace.newBuilder();
+    traceBuilder
+        .setCustomerId("customer1")
+        .setTraceId(ByteBuffer.wrap("sample-trace-id".getBytes()))
+        .setEntityList(
+            Collections.singletonList(
+                Entity.newBuilder()
+                    .setCustomerId("customer1")
+                    .setEntityId("sample-entity-id")
+                    .setEntityName("sample-entity-name")
+                    .setEntityType("SERVICE")
+                    .build()))
+        .setEventList(
+            Collections.singletonList(
+                Event.newBuilder()
+                    .setCustomerId("customer1")
+                    .setEventId(ByteBuffer.wrap("sample-span-id".getBytes()))
+                    .setEventName("sample-span-name")
+                    .setEntityIdList(Collections.singletonList("sample-entity-id"))
+                    .setStartTimeMillis(System.currentTimeMillis())
+                    .setEndTimeMillis(System.currentTimeMillis())
+                    .setMetrics(Metrics.newBuilder().setMetricMap(new HashMap<>()).build())
+                    .setAttributesBuilder(Attributes.newBuilder().setAttributeMap(new HashMap<>()))
+                    .setEnrichedAttributesBuilder(
+                        Attributes.newBuilder().setAttributeMap(Maps.newHashMap()))
+                    .build()))
+        .setMetrics(Metrics.newBuilder().setMetricMap(new HashMap<>()).build())
+        .setEntityEdgeList(new ArrayList<>())
+        .setEventEdgeList(new ArrayList<>())
+        .setEntityEventEdgeList(new ArrayList<>())
+        .setStartTimeMillis(System.currentTimeMillis())
+        .setEndTimeMillis(System.currentTimeMillis());
+
+    StructuredTrace trace = traceBuilder.build();
+    SpanEventViewGenerator spanEventViewGenerator = new SpanEventViewGenerator();
+    List<SpanEventView> list = spanEventViewGenerator.process(trace);
+    assertEquals(0, list.get(0).getApiTraceErrorSpanCount());
+
+    Map<String, AttributeValue> spanAttributes = new HashMap<>();
+    spanAttributes.put(
+        EnrichedSpanConstants.API_TRACE_ERROR_SPAN_COUNT_ATTRIBUTE,
+        AttributeValue.newBuilder().setValue("5").build());
+
+    traceBuilder
+        .setEventList(
+            Collections.singletonList(
+                Event.newBuilder()
+                    .setCustomerId("customer1")
+                    .setEventId(ByteBuffer.wrap("sample-span-id".getBytes()))
+                    .setEventName("sample-span-name")
+                    .setEntityIdList(Collections.singletonList("sample-entity-id"))
+                    .setStartTimeMillis(System.currentTimeMillis())
+                    .setEndTimeMillis(System.currentTimeMillis())
+                    .setMetrics(Metrics.newBuilder().setMetricMap(new HashMap<>()).build())
+                    .setAttributesBuilder(Attributes.newBuilder().setAttributeMap(new HashMap<>()))
+                    .setEnrichedAttributesBuilder(
+                        Attributes.newBuilder().setAttributeMap(spanAttributes))
+                    .build()))
+        .build();
+
+    trace = traceBuilder.build();
+    spanEventViewGenerator = new SpanEventViewGenerator();
+    list = spanEventViewGenerator.process(trace);
+    assertEquals(5, list.get(0).getApiTraceErrorSpanCount());
+  }
 }