Owls85476 attempt to fix intermittent integration test issues in nightlies (#2037)

* switch to kubectl and improve the test code

* change testAddSecondApp to use kubectl as well

* more changes

* check main thread done as well

* minor update

* minor change

Author: doxiao

integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiDomain.java

@@ -12,7 +12,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.Callable;
 
 import io.kubernetes.client.custom.V1Patch;
 import io.kubernetes.client.openapi.models.V1ConfigMap;
@@ -44,6 +44,7 @@
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.MethodSource;
 
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static java.util.concurrent.TimeUnit.MINUTES;
 import static java.util.concurrent.TimeUnit.SECONDS;
 import static oracle.weblogic.kubernetes.TestConstants.ADMIN_PASSWORD_DEFAULT;
@@ -122,6 +123,7 @@ class ItMiiDomain {
   private String miiImageAddSecondApp = null;
   private String miiImage = null;
   private static LoggingFacade logger = null;
+  private static volatile boolean mainThreadDone = false;
 
   /**
    * Install Operator.
@@ -137,9 +139,9 @@ public static void initAll(@Namespaces(3) List<String> namespaces) {
         .atMost(6, MINUTES).await();
 
     // create a reusable quick retry policy
-    withQuickRetryPolicy = with().pollDelay(0, SECONDS)
-        .and().with().pollInterval(4, SECONDS)
-        .atMost(10, SECONDS).await();
+    withQuickRetryPolicy = with().pollDelay(1, SECONDS)
+        .and().with().pollInterval(2, SECONDS)
+        .atMost(15, SECONDS).await();
 
     // get a new unique opNamespace
     logger.info("Creating unique namespace for Operator");
@@ -334,7 +336,7 @@ public void testPatchAppV2() {
     accountingThread =
         new Thread(
             () -> {
-              collectAppAvaiability(
+              collectAppAvailability(
                   domainNamespace,
                   appAvailability,
                   managedServerPrefix,
@@ -394,7 +396,7 @@ public void testPatchAppV2() {
             MII_APP_RESPONSE_V2 + i);
       } 
     } finally {
-    
+      mainThreadDone = true;
       if (accountingThread != null) {
         try {
           accountingThread.join();
@@ -1060,56 +1062,74 @@ private void checkPodImagePatched(
                namespace)));
   }
 
-  private static void collectAppAvaiability(
+  private static void collectAppAvailability(
       String namespace,
       List<Integer> appAvailability,
       String managedServerPrefix,
       int replicaCount,
       String internalPort,
       String appPath
   ) {
-    boolean v2AppAvailable = false;
+    with().pollDelay(2, SECONDS)
+        .and().with().pollInterval(200, MILLISECONDS)
+        .atMost(15, MINUTES)
+        .await()
+        .conditionEvaluationListener(
+            condition -> logger.info("Waiting for patched application running on all managed servers in namespace {1} "
+                + "(elapsed time {2}ms, remaining time {3}ms)",
+            namespace,
+            condition.getElapsedTimeInMS(),
+            condition.getRemainingTimeInMS()))
+        .until(assertDoesNotThrow(() -> checkContinuousAvailability(
+            namespace, appAvailability, managedServerPrefix, replicaCount, internalPort, appPath),
+            String.format(
+                "App is not available on all managed servers in namespace %s.",
+                namespace)));
 
-    // Access the pod periodically to check application's availability across the duration
-    // of patching the domain with newer version of the application.
-    while (!v2AppAvailable)  {
-      v2AppAvailable = true;
+  }
+
+  private static Callable<Boolean> checkContinuousAvailability(
+      String namespace,
+      List<Integer> appAvailability,
+      String managedServerPrefix,
+      int replicaCount,
+      String internalPort,
+      String appPath) {
+    return () -> {
+      boolean v2AppAvailable = true;
+      
       for (int i = 1; i <= replicaCount; i++) {
         v2AppAvailable = v2AppAvailable && appAccessibleInPod(
                             namespace,
-                            managedServerPrefix + i, 
-                            internalPort, 
-                            appPath, 
+                            managedServerPrefix + i,
+                            internalPort,
+                            appPath,
                             MII_APP_RESPONSE_V2 + i);
       }
 
       int count = 0;
       for (int i = 1; i <= replicaCount; i++) {
         if (appAccessibleInPod(
             namespace,
-            managedServerPrefix + i, 
-            internalPort, 
-            appPath, 
-            "Hello World")) {  
+            managedServerPrefix + i,
+            internalPort,
+            appPath,
+            "Hello World")) {
           count++;
         }
       }
       appAvailability.add(count);
-      
+
       if (count == 0) {
-        logger.info("XXXXXXXXXXX: application not available XXXXXXXX");
-        break;
-      } else {
-        logger.fine("YYYYYYYYYYY: application available YYYYYYYY count = " + count);   
+        logger.info("NNNNNNNNNNN: application not available NNNNNNNN");
+        return true;
       }
-      try {
-        TimeUnit.MILLISECONDS.sleep(200);
-      } catch (InterruptedException ie) {
-        // do nothing
-      }
-    }
+
+      logger.fine("YYYYYYYYYYY: application available YYYYYYYY count = " + count);
+      return v2AppAvailable || mainThreadDone;
+    };
   }
-  
+
   private static boolean appAlwaysAvailable(List<Integer> appAvailability) {
     for (Integer count: appAvailability) {
       if (count == 0) {

integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/TestAssertions.java

@@ -431,8 +431,7 @@ public static Callable<Boolean> credentialsNotValid(
   }
 
   /**
-   * Check if an application is accessible inside a WebLogic server pod using
-   * Kubernetes Java client API.
+   * Check if an application is accessible inside a WebLogic server pod.
    *
    * @param namespace Kubernetes namespace where the WebLogic server pod is running
    * @param podName name of the WebLogic server pod
@@ -447,27 +446,6 @@ public static boolean appAccessibleInPod(
       String port,
       String appPath,
       String expectedResponse
-  ) {
-    return Application.appAccessibleInPod(namespace, podName, port, appPath, expectedResponse);
-  }
-
-  /**
-   * Check if an application is accessible inside a WebLogic server pod using
-   * "kubectl exec" command.
-   *
-   * @param namespace Kubernetes namespace where the WebLogic server pod is running
-   * @param podName name of the WebLogic server pod
-   * @param port internal port of the managed server running in the pod
-   * @param appPath path to access the application
-   * @param expectedResponse the expected response from the application
-   * @return true if the command succeeds
-   */
-  public static boolean appAccessibleInPodKubectl(
-      String namespace,
-      String podName,
-      String port,
-      String appPath,
-      String expectedResponse
   ) {
     return Application.appAccessibleInPodKubectl(namespace, podName, port, appPath, expectedResponse);
   }
@@ -489,7 +467,7 @@ public static boolean appNotAccessibleInPod(
       String appPath,
       String expectedResponse
   ) {
-    return !Application.appAccessibleInPod(namespace, podName, port, appPath, expectedResponse);
+    return !Application.appAccessibleInPodKubectl(namespace, podName, port, appPath, expectedResponse);
   }
 
   /**

integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/impl/Application.java

@@ -83,7 +83,7 @@ public static boolean appAccessibleInPod(
   }
 
   /**
-   * Check if an application is accessible inside a WebLogic server pod.
+   * Check if an application is accessible inside a WebLogic server pod using "kubectl exec" command.
    *
    * @param namespace Kubernetes namespace where the WebLogic server pod is running
    * @param podName name of the WebLogic server pod