Improve test verification (#1777)

* Improve test verification

* Fail application availability check early

* Address a review comment

Author: doxiao

new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiDomain.java

@@ -410,7 +410,7 @@ public void testPatchAppV2() {
         } catch (InterruptedException ie) {
           // do nothing
         }
- 
+
         // check the application availability data that we have collected, and see if
         // the application has been available all the time since the beginning of this test method
         logger.info("Verify that V2 application was available when domain {0} was being patched with image {1}",
@@ -1045,7 +1045,7 @@ private static void collectAppAvaiability(
       String appPath
   ) {
     boolean v2AppAvailable = false;
- 
+
     // Access the pod periodically to check application's availability across the duration
     // of patching the domain with newer version of the application.
     while (!v2AppAvailable)  {
@@ -1074,6 +1074,7 @@ private static void collectAppAvaiability(
 
       if (count == 0) {
         logger.info("XXXXXXXXXXX: application not available XXXXXXXX");
+        break;
       } else {
         logger.fine("YYYYYYYYYYY: application available YYYYYYYY count = " + count);   
       }

new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItOperatorRestartWhenPodRoll.java

@@ -34,9 +34,8 @@
 import static oracle.weblogic.kubernetes.actions.TestActions.deletePod;
 import static oracle.weblogic.kubernetes.actions.TestActions.getOperatorPodName;
 import static oracle.weblogic.kubernetes.actions.TestActions.getPodCreationTimestamp;
-import static oracle.weblogic.kubernetes.assertions.TestAssertions.operatorIsReady;
+import static oracle.weblogic.kubernetes.assertions.TestAssertions.isOperatorPodRestarted;
 import static oracle.weblogic.kubernetes.assertions.TestAssertions.verifyRollingRestartOccurred;
-import static oracle.weblogic.kubernetes.assertions.impl.Kubernetes.isPodRestarted;
 import static oracle.weblogic.kubernetes.utils.CommonMiiTestUtils.createMiiDomainAndVerify;
 import static oracle.weblogic.kubernetes.utils.CommonPatchTestUtils.checkPodRestartVersionUpdated;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createSecretWithUsernamePassword;
@@ -235,18 +234,14 @@ private void restartOperatorAndVerify() {
             opNamespace,
             condition.getElapsedTimeInMS(),
             condition.getRemainingTimeInMS()))
-        .until(assertDoesNotThrow(() -> operatorIsReady(opNamespace),
-          "operatorIsReady failed with ApiException"));
+        .until(assertDoesNotThrow(() -> isOperatorPodRestarted(opNamespace, opPodCreationTime),
+            "Failed to check if the operator is restarted with ApiException"));
 
-    String opPodNameNew = 
+    String opPodNameNew =
         assertDoesNotThrow(() -> getOperatorPodName(TestConstants.OPERATOR_RELEASE_NAME, opNamespace),
-        "Failed to get the name of the operator pod");
+            "Failed to get the name of the operator pod");
 
     assertFalse(opPodNameNew.equals(opPodName),
         "The operator names before and after a restart should be different");
-
-    assertTrue(assertDoesNotThrow(() -> isPodRestarted(opPodNameNew, opNamespace, opPodCreationTime),
-        "Failed to check the operator for its new creation time with ApiException"),
-        "Operator restart failed");
   }
 }

new-integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/TestAssertions.java

@@ -446,6 +446,22 @@ public static Callable<Boolean> isPodRestarted(
     };
   }
 
+  /**
+   * Check if the oeprator pod in a given namespace is restarted based on podCreationTimestamp.
+   *
+   * @param namespace in which the pod is running
+   * @param timestamp the initial podCreationTimestamp
+   * @return true if the pod new timestamp is not equal to initial PodCreationTimestamp otherwise false
+   */
+  public static Callable<Boolean> isOperatorPodRestarted(
+      String namespace,
+      DateTime timestamp
+  ) {
+    return () -> {
+      return Kubernetes.isOperatorPodRestarted(namespace, timestamp);
+    };
+  }
+
   /**
    * Verify the pod state is not changed.
    *

new-integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/impl/Kubernetes.java

@@ -776,4 +776,31 @@ public static void copyFileToPod(String namespace,
     Copy copy = new Copy(apiClient);
     copy.copyFileToPod(namespace, pod, container, srcPath, destPath);
   }
+
+  /**
+   * Check if the operator pod in the given namespace is restarted based on podCreationTimestamp.
+   *
+   * @param namespace in which the operator pod is running
+   * @param timestamp the initial podCreationTimestamp
+   * @return true if the pod's creation timestamp is later than the initial PodCreationTimestamp
+   * @throws ApiException when query fails
+   */
+  public static Boolean isOperatorPodRestarted(String namespace, DateTime timestamp) throws ApiException {
+    String labelSelector = String.format("weblogic.operatorName in (%s)", namespace);
+    V1Pod pod = getPod(namespace, labelSelector, "weblogic-operator-");
+    if (pod != null) {
+      // get the podCondition with the 'Ready' type field
+      V1PodCondition v1PodReadyCondition = pod.getStatus().getConditions().stream()
+          .filter(v1PodCondition -> "Ready".equals(v1PodCondition.getType()))
+          .findAny()
+          .orElse(null);
+
+      if (v1PodReadyCondition != null
+          && v1PodReadyCondition.getStatus().equalsIgnoreCase("true")) {
+        String podName = pod.getMetadata().getName();
+        return isPodRestarted(podName, namespace, timestamp);
+      }
+    }
+    return false;
+  }
 }