Merge pull request aws-observability#32 from bhautikpip/merge-conflic…

…t-trace-validator

TraceValidator

validator/build.gradle

@@ -41,6 +41,9 @@ dependencies {
     // yaml reader
     compile group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml', version: '2.11.1'
 
+    // json flattener
+    compile group: 'com.github.wnameless', name: 'json-flattener', version: '0.1.0'
+
     // command cli
     compile 'info.picocli:picocli:4.3.2'
 

validator/src/main/java/com/amazon/aoc/callers/HttpCaller.java

@@ -35,6 +35,7 @@ public class HttpCaller implements ICaller {
 
   public HttpCaller(String endpoint, String path) {
     this.url = endpoint + path;
+    log.info("validator is trying to hit this {} endpoint", this.url);
   }
 
   @Override
@@ -44,7 +45,7 @@ public SampleAppResponse callSampleApp() throws Exception {
 
     AtomicReference<SampleAppResponse> sampleAppResponseAtomicReference = new AtomicReference<>();
     RetryHelper.retry(
-        60,
+        30,
         () -> {
           try (Response response = client.newCall(request).execute()) {
             String responseBody = response.body().string();

validator/src/main/java/com/amazon/aoc/exception/ExceptionCode.java

@@ -25,10 +25,12 @@ public enum ExceptionCode {
 
   // validating errors
   TRACE_ID_NOT_MATCHED(50001, "trace id not matched"),
+  DATA_MODEL_NOT_MATCHED(50006, "trace id not matched"),
   TRACE_SPAN_LIST_NOT_MATCHED(50002, "trace span list has different length"),
   TRACE_SPAN_NOT_MATCHED(50003, "trace span not matched"),
   TRACE_LIST_NOT_MATCHED(50004, "trace list has different length"),
   DATA_EMITTER_UNAVAILABLE(50005, "the data emitter is unavailable to ping"),
+  EMPTY_LIST(50007, "list is empty or null"),
 
   // build validator
   VALIDATION_TYPE_NOT_EXISTED(60001, "validation type not existed"),

validator/src/main/java/com/amazon/aoc/fileconfigs/ExpectedTrace.java

@@ -20,6 +20,10 @@
 @Getter
 public enum ExpectedTrace implements FileConfig {
   DEFAULT_EXPECTED_TRACE("/expected-data-template/defaultExpectedTrace.mustache"),
+  OTEL_SDK_AWSSDK_EXPECTED_TRACE("/expected-data-template/otelSDKexpectedAWSSDKTrace.mustache"),
+  OTEL_SDK_HTTP_EXPECTED_TRACE("/expected-data-template/otelSDKexpectedHTTPTrace.mustache"),
+  XRAY_SDK_AWSSDK_EXPECTED_TRACE("/expected-data-template/xraySDKexpectedAWSSDKTrace.mustache"),
+  XRAY_SDK_HTTP_EXPECTED_TRACE("/expected-data-template/xraySDKexpectedHTTPTrace.mustache"),
   ;
 
   private String path;

validator/src/main/java/com/amazon/aoc/helpers/RetryHelper.java

@@ -37,11 +37,16 @@ public static void retry(
       throws Exception {
     while (retryCount-- > 0) {
       try {
-        log.info("still can retry for {} times", retryCount);
+        log.info("retry attempt left : {} ", retryCount);
         retryable.execute();
         return;
       } catch (Exception ex) {
-        log.error("exception during retry, you may ignore it", ex);
+        log.info("retrying after {} seconds", TimeUnit.MILLISECONDS.toSeconds(sleepInMilliSeconds));
+
+        if (retryCount == 0) {
+          log.error("retries exhausted, possible exception: ", ex);
+          break;
+        }
         TimeUnit.MILLISECONDS.sleep(sleepInMilliSeconds);
       }
     }

validator/src/main/java/com/amazon/aoc/validators/TraceValidator.java

@@ -26,93 +26,107 @@
 import com.amazon.aoc.models.ValidationConfig;
 import com.amazon.aoc.services.XRayService;
 import com.amazonaws.services.xray.model.Trace;
+import com.github.wnameless.json.flattener.JsonFlattener;
 import lombok.extern.log4j.Log4j2;
 
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.List;
-import java.util.stream.Collectors;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicReference;
 
 @Log4j2
 public class TraceValidator implements IValidator {
-  private MustacheHelper mustacheHelper = new MustacheHelper();
-  private static int MAX_RETRY_COUNT = 60;
-  private Context context;
-  private XRayService xrayService;
-  private ICaller caller;
-  private FileConfig expectedTrace;
-
-  @Override
-  public void init(
-      Context context, ValidationConfig validationConfig, ICaller caller, FileConfig expectedTrace)
-      throws Exception {
-    this.context = context;
-    this.xrayService = new XRayService(context.getRegion());
-    this.caller = caller;
-    this.expectedTrace = expectedTrace;
-  }
-
-  @Override
-  public void validate() throws Exception {
-    List<Trace> expectedTraceList = this.getExpectedTrace();
-    expectedTraceList.sort(Comparator.comparing(Trace::getId));
-    RetryHelper.retry(
-        MAX_RETRY_COUNT,
-        () -> {
-          List<Trace> traceList =
-              xrayService.listTraceByIds(
-                  expectedTraceList.stream()
-                      .map(trace -> trace.getId())
-                      .collect(Collectors.toList()));
-
-          traceList.sort(Comparator.comparing(Trace::getId));
-
-          log.info("expectedTraceList: {}", expectedTraceList);
-          log.info("traceList got from backend: {}", traceList);
-          if (expectedTraceList.size() != traceList.size()) {
-            throw new BaseException(ExceptionCode.TRACE_LIST_NOT_MATCHED);
-          }
-
-          for (int i = 0; i != expectedTraceList.size(); ++i) {
-            Trace trace = traceList.get(i);
-            compareTwoTraces(expectedTraceList.get(i), trace);
-          }
+    private MustacheHelper mustacheHelper = new MustacheHelper();
+    private XRayService xrayService;
+    private ICaller caller;
+    private Context context;
+    private FileConfig expectedTrace;
+
+    @Override
+    public void init(Context context, ValidationConfig validationConfig, ICaller caller, FileConfig expectedTrace) throws Exception {
+        this.xrayService = new XRayService(context.getRegion());
+        this.caller = caller;
+        this.context = context;
+        this.expectedTrace = expectedTrace;
+    }
+
+    @Override
+    public void validate() throws Exception {
+        // get stored trace
+        Map<String, Object> storedTrace = this.getStoredTrace();
+
+        // create trace id list to retrieve trace from x-ray service
+        String traceId = (String) storedTrace.get("trace_id");
+        List<String> traceIdList = Collections.singletonList(traceId);
+
+        // get retrieved trace from x-ray service
+        Map<String, Object> retrievedTrace = this.getRetrievedTrace(traceIdList);
+
+        // validation of trace id
+        if (!storedTrace.get("trace_id").equals(retrievedTrace.get("trace_id"))) {
+            log.error("trace id validation failed");
+            throw new BaseException(ExceptionCode.TRACE_ID_NOT_MATCHED);
+        }
+
+        // data model validation of other fields of segment document
+        for (Map.Entry<String, Object> entry : storedTrace.entrySet()) {
+            if (!entry.getValue().equals(retrievedTrace.get(entry.getKey()))) {
+                log.error("data model validation failed");
+                log.info("mis matched data model field list");
+                log.info("value of stored trace map: {}", entry.getValue());
+                log.info("value of retrieved map: {}",retrievedTrace.get(entry.getKey()));
+                log.info("==========================================");
+                throw new BaseException(ExceptionCode.DATA_MODEL_NOT_MATCHED);
+            }
+        }
+    }
+
+    // this method will hit get trace from x-ray service and get retrieved trace
+    private Map<String, Object> getRetrievedTrace(List<String> traceIdList) throws Exception {
+        Map<String, Object> flattenedJsonMapForRetrievedTrace = null;
+        AtomicReference<List<Trace>> retrieveTraceListAtomicReference = new AtomicReference<>();
+        int MAX_RETRY_COUNT = 3;
+
+        RetryHelper.retry(MAX_RETRY_COUNT, () -> {
+            List<Trace> retrieveTraceList = null;
+            retrieveTraceList = xrayService.listTraceByIds(traceIdList);
+            retrieveTraceListAtomicReference.set(retrieveTraceList);
+
+            if (retrieveTraceList == null || retrieveTraceList.isEmpty()) {
+                throw new BaseException(ExceptionCode.EMPTY_LIST);
+            }
         });
-  }
 
-  private void compareTwoTraces(Trace trace1, Trace trace2) throws BaseException {
-    // check trace id
-    if (!trace1.getId().equals(trace2.getId())) {
-      throw new BaseException(ExceptionCode.TRACE_ID_NOT_MATCHED);
+        // flattened JSON object to a map
+        if (retrieveTraceListAtomicReference.get() != null && !retrieveTraceListAtomicReference.get().isEmpty()) {
+            try {
+                flattenedJsonMapForRetrievedTrace = JsonFlattener.flattenAsMap(retrieveTraceListAtomicReference.get().get(0).getSegments().get(0).getDocument());
+            } catch (Exception e) {
+                log.error("exception while flattening the retrieved trace: " + e.getMessage());
+                e.printStackTrace();
+            }
+        } else {
+            log.error("retrieved trace list is empty or null");
+            throw new BaseException(ExceptionCode.EMPTY_LIST);
+        }
+
+        return flattenedJsonMapForRetrievedTrace;
     }
 
-    /*
-    if (trace1.getSegments().size() != trace2.getSegments().size()) {
-      throw new BaseException(ExceptionCode.TRACE_SPAN_LIST_NOT_MATCHED);
+    // this method will hit a http endpoints of sample web apps and get stored trace
+    private Map<String, Object> getStoredTrace() throws Exception {
+        Map<String, Object> flattenedJsonMapForStoredTraces = null;
+
+        SampleAppResponse sampleAppResponse = this.caller.callSampleApp();
+
+        String jsonExpectedTrace = mustacheHelper.render(this.expectedTrace, context);
+
+        try {
+            // flattened JSON object to a map
+            flattenedJsonMapForStoredTraces = JsonFlattener.flattenAsMap(jsonExpectedTrace);
+            flattenedJsonMapForStoredTraces.put("trace_id", sampleAppResponse.getTraceId());
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+
+        return flattenedJsonMapForStoredTraces;
     }
-    trace1.getSegments().sort(Comparator.comparing(Segment::getId));
-    trace2.getSegments().sort(Comparator.comparing(Segment::getId));
-
-    for (int i = 0; i != trace1.getSegments().size(); ++i) {
-      // check span id
-      if (!trace1.getSegments().get(i).getId()
-          .equals(trace2.getSegments().get(i).getId())) {
-        throw new BaseException(ExceptionCode.TRACE_SPAN_NOT_MATCHED);
-      }
-    }*/
-  }
-
-  // this endpoint will be a http endpoint including the path with get method
-  private List<Trace> getExpectedTrace() throws Exception {
-    SampleAppResponse sampleAppResponse = this.caller.callSampleApp();
-
-    // convert the trace data into xray format
-    Trace trace = new Trace();
-    trace.setId(sampleAppResponse.getTraceId());
-
-    // todo: construct the trace data from the template file
-
-    // we can support multi expected trace id to validate if need
-    return Arrays.asList(trace);
-  }
 }

validator/src/main/resources/expected-data-template/otelSDKexpectedAWSSDKTrace.mustache

@@ -0,0 +1,58 @@
+{
+  "name": "/aws-sdk-call",
+  "fault": false,
+  "error": false,
+  "throttle": false,
+  "http": {
+    "request": {
+      "url": "{{endpoint}}/aws-sdk-call",
+      "method": "GET"
+    },
+    "response": {
+      "status": 200
+    }
+  },
+  "subsegments": [
+    {
+      "fault": false,
+      "error": false,
+      "throttle": false,
+      "subsegments": [
+        {
+          "name": "S3",
+          "fault": false,
+          "error": false,
+          "throttle": false,
+          "http": {
+            "request": {
+              "url": "https://s3.{{region}}.amazonaws.com/",
+              "method": "GET"
+            },
+            "response": {
+              "status": 200
+            }
+          },
+          "namespace": "aws",
+          "subsegments": [
+            {
+              "name": "s3.{{region}}.amazonaws.com",
+              "fault": false,
+              "error": false,
+              "throttle": false,
+              "http": {
+                "request": {
+                  "url": "https://s3.{{region}}.amazonaws.com/",
+                  "method": "GET"
+                },
+                "response": {
+                  "status": 200
+                }
+              },
+              "namespace": "remote"
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}

validator/src/main/resources/expected-data-template/otelSDKexpectedHTTPTrace.mustache

@@ -0,0 +1,40 @@
+{
+  "name": "/outgoing-http-call",
+  "fault": false,
+  "error": false,
+  "throttle": false,
+  "http": {
+    "request": {
+      "url": "{{endpoint}}/outgoing-http-call",
+      "method": "GET"
+    },
+    "response": {
+      "status": 200
+    }
+  },
+  "subsegments": [
+    {
+      "fault": false,
+      "error": false,
+      "throttle": false,
+      "subsegments": [
+        {
+          "name": "aws.amazon.com",
+          "fault": false,
+          "error": false,
+          "throttle": false,
+          "http": {
+            "request": {
+              "url": "https://aws.amazon.com/",
+              "method": "GET"
+            },
+            "response": {
+              "status": 200
+            }
+          },
+          "namespace": "remote"
+        }
+      ]
+    }
+  ]
+}

validator/src/main/resources/expected-data-template/xraySDKexpectedAWSSDKTrace.mustache

@@ -0,0 +1,27 @@
+{
+  "in_progress": false,
+  "http": {
+    "request": {
+      "url": "{{endpoint}}/aws-sdk-call",
+      "method": "GET"
+    },
+    "response": {
+      "status": 200
+    }
+  },
+  "subsegments": [
+    {
+      "name": "S3",
+      "in_progress": false,
+      "http": {
+        "response": {
+          "status": 200
+        }
+      },
+      "aws": {
+        "operation": "ListBuckets"
+      },
+      "namespace": "aws"
+    }
+  ]
+}