Merge pull request opensearch-project#36 from khushbr/feature/rca-fra…

…mework-crash-metric Adding metric emission + UT for RCA_FRAMEWORK_CRASH
peterzhuamazon · Jul 30, 2021 · b5c348a · b5c348a
2 parents 80d39f1 + a268d99
commit b5c348a
Show file tree

Hide file tree

Showing 4 changed files with 25 additions and 15 deletions.
diff --git a/src/main/java/org/opensearch/performanceanalyzer/rca/RcaController.java b/src/main/java/org/opensearch/performanceanalyzer/rca/RcaController.java
@@ -63,6 +63,7 @@
 import org.opensearch.performanceanalyzer.rca.framework.core.RcaConf;
 import org.opensearch.performanceanalyzer.rca.framework.core.Stats;
 import org.opensearch.performanceanalyzer.rca.framework.core.ThresholdMain;
+import org.opensearch.performanceanalyzer.rca.framework.metrics.ExceptionsAndErrors;
 import org.opensearch.performanceanalyzer.rca.framework.metrics.RcaRuntimeMetrics;
 import org.opensearch.performanceanalyzer.rca.framework.util.InstanceDetails;
 import org.opensearch.performanceanalyzer.rca.framework.util.RcaConsts;
@@ -338,6 +339,8 @@ public void run() {
                     Thread.sleep(rcaStateCheckIntervalMillis - duration);
                 }
             } catch (InterruptedException ie) {
+                PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat(
+                        ExceptionsAndErrors.RCA_FRAMEWORK_CRASH, "", 1);
                 if (deliberateInterrupt) {
                     // This should only happen in case of tests. So, its okay for this log level to
                     // be info.

diff --git a/src/test/java/org/opensearch/performanceanalyzer/rca/RcaControllerTest.java b/src/test/java/org/opensearch/performanceanalyzer/rca/RcaControllerTest.java
@@ -63,6 +63,7 @@
 import org.opensearch.performanceanalyzer.rca.framework.core.ConnectedComponent;
 import org.opensearch.performanceanalyzer.rca.framework.core.RcaConf;
 import org.opensearch.performanceanalyzer.rca.framework.core.Stats;
+import org.opensearch.performanceanalyzer.rca.framework.metrics.ExceptionsAndErrors;
 import org.opensearch.performanceanalyzer.rca.framework.util.RcaConsts;
 import org.opensearch.performanceanalyzer.rca.scheduler.RCAScheduler;
 import org.opensearch.performanceanalyzer.rca.scheduler.RcaSchedulerState;
@@ -203,6 +204,12 @@ public void tearDown() throws InterruptedException {
         }
     }
 
+    @Test
+    public void rcaFrameworkCrash() throws InterruptedException {
+        controllerThread.interrupt();
+        Assert.assertTrue(RcaTestHelper.verify(ExceptionsAndErrors.RCA_FRAMEWORK_CRASH));
+    }
+
     @Test
     public void readRcaEnabledFromConf() throws IOException {
         changeRcaRunState(RcaState.STOP);

diff --git a/src/test/java/org/opensearch/performanceanalyzer/rca/RcaTestHelper.java b/src/test/java/org/opensearch/performanceanalyzer/rca/RcaTestHelper.java
@@ -50,9 +50,11 @@
 import org.apache.logging.log4j.core.appender.FileAppender;
 import org.jooq.tools.json.JSONObject;
 import org.opensearch.performanceanalyzer.AppContext;
+import org.opensearch.performanceanalyzer.PerformanceAnalyzerApp;
 import org.opensearch.performanceanalyzer.metrics.AllMetrics;
 import org.opensearch.performanceanalyzer.rca.framework.core.ConnectedComponent;
 import org.opensearch.performanceanalyzer.rca.framework.core.Node;
+import org.opensearch.performanceanalyzer.rca.stats.measurements.MeasurementSet;
 import org.opensearch.performanceanalyzer.reader.ClusterDetailsEventProcessor;
 import org.opensearch.performanceanalyzer.reader_writer_shared.Event;
 
@@ -199,4 +201,16 @@ public static void updateConfFileForMutedComponents(
         ((ObjectNode) configObject).putArray(componentKey).addAll(array);
         mapper.writeValue(new FileOutputStream(rcaConfPath), configObject);
     }
+
+    public static boolean verify(MeasurementSet measurementSet) throws InterruptedException {
+        final int MAX_TIME_TO_WAIT_MILLIS = 10_000;
+        int waited_for_millis = 0;
+        while (waited_for_millis++ < MAX_TIME_TO_WAIT_MILLIS) {
+            if (PerformanceAnalyzerApp.RCA_STATS_REPORTER.isMeasurementCollected(measurementSet)) {
+                return true;
+            }
+            Thread.sleep(1);
+        }
+        return false;
+    }
 }
diff --git a/...pensearch/performanceanalyzer/rca/listener/MisbehavingGraphOperateMethodListenerTest.java b/...pensearch/performanceanalyzer/rca/listener/MisbehavingGraphOperateMethodListenerTest.java
@@ -36,7 +36,6 @@
 import org.junit.Assert;
 import org.junit.Test;
 import org.opensearch.performanceanalyzer.AppContext;
-import org.opensearch.performanceanalyzer.PerformanceAnalyzerApp;
 import org.opensearch.performanceanalyzer.collectors.StatsCollector;
 import org.opensearch.performanceanalyzer.rca.RcaTestHelper;
 import org.opensearch.performanceanalyzer.rca.framework.api.AnalysisGraph;
@@ -55,7 +54,6 @@
 import org.opensearch.performanceanalyzer.rca.framework.util.RcaUtil;
 import org.opensearch.performanceanalyzer.rca.scheduler.RCASchedulerTask;
 import org.opensearch.performanceanalyzer.rca.spec.MetricsDBProviderTestHelper;
-import org.opensearch.performanceanalyzer.rca.stats.measurements.MeasurementSet;
 
 public class MisbehavingGraphOperateMethodListenerTest {
     class FaultyAnalysisGraph extends AnalysisGraph {
@@ -121,7 +119,7 @@ public void rcaMutedForThrowingExceptions() throws Exception {
 
         for (int i = 0; i <= MisbehavingGraphOperateMethodListener.TOLERANCE_LIMIT; i++) {
             rcaSchedulerTask.run();
-            Assert.assertTrue(verify(ExceptionsAndErrors.EXCEPTION_IN_OPERATE));
+            Assert.assertTrue(RcaTestHelper.verify(ExceptionsAndErrors.EXCEPTION_IN_OPERATE));
         }
 
         Assert.assertEquals(1, Stats.getInstance().getMutedGraphNodesCount());
@@ -130,18 +128,6 @@ public void rcaMutedForThrowingExceptions() throws Exception {
                         .isNodeMuted(FaultyAnalysisGraph.HighCpuSymptom.class.getSimpleName()));
     }
 
-    private boolean verify(MeasurementSet measurementSet) throws InterruptedException {
-        final int MAX_TIME_TO_WAIT_MILLIS = 10_000;
-        int waited_for_millis = 0;
-        while (waited_for_millis++ < MAX_TIME_TO_WAIT_MILLIS) {
-            if (PerformanceAnalyzerApp.RCA_STATS_REPORTER.isMeasurementCollected(measurementSet)) {
-                return true;
-            }
-            Thread.sleep(1);
-        }
-        return false;
-    }
-
     @After
     public void cleanup() {
         // RcaTestHelper.cleanUpLogs();