check operator image name after upgrade

new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItOperatorUpgrade.java

@@ -10,6 +10,7 @@
 import java.util.Map;
 import java.util.concurrent.Callable;
 
+import oracle.weblogic.kubernetes.actions.impl.OperatorParams;
 import oracle.weblogic.kubernetes.actions.impl.primitive.Command;
 import oracle.weblogic.kubernetes.actions.impl.primitive.CommandParams;
 import oracle.weblogic.kubernetes.actions.impl.primitive.HelmParams;
@@ -31,11 +32,15 @@
 import static java.util.concurrent.TimeUnit.SECONDS;
 import static oracle.weblogic.kubernetes.TestConstants.ADMIN_PASSWORD_DEFAULT;
 import static oracle.weblogic.kubernetes.TestConstants.ADMIN_USERNAME_DEFAULT;
+import static oracle.weblogic.kubernetes.TestConstants.OPERATOR_CHART_DIR;
 import static oracle.weblogic.kubernetes.TestConstants.OPERATOR_GITHUB_CHART_REPO_URL;
+import static oracle.weblogic.kubernetes.TestConstants.OPERATOR_RELEASE_NAME;
 import static oracle.weblogic.kubernetes.TestConstants.RESULTS_ROOT;
 import static oracle.weblogic.kubernetes.TestConstants.WDT_BASIC_IMAGE_NAME;
 import static oracle.weblogic.kubernetes.TestConstants.WDT_BASIC_IMAGE_TAG;
 import static oracle.weblogic.kubernetes.actions.ActionConstants.RESOURCE_DIR;
+import static oracle.weblogic.kubernetes.actions.TestActions.getOperatorContainerImageName;
+import static oracle.weblogic.kubernetes.actions.TestActions.getOperatorImageName;
 import static oracle.weblogic.kubernetes.actions.TestActions.getServiceNodePort;
 import static oracle.weblogic.kubernetes.actions.TestActions.shutdownDomain;
 import static oracle.weblogic.kubernetes.actions.TestActions.startDomain;
@@ -73,7 +78,8 @@ public class ItOperatorUpgrade {
   private String adminServerPodName = domainUid + "-admin-server";
   private String managedServerPodNamePrefix = domainUid + "-managed-server";
   private int replicaCount = 2;
-  List<String> namespaces;
+  private List<String> namespaces;
+  private String latestOperatorImageName;
 
   /**
    * Does some initialization of logger, conditionfactory, etc common
@@ -83,7 +89,7 @@ public class ItOperatorUpgrade {
   public static void init() {
     logger = getLogger();
     // create standard, reusable retry/backoff policy
-    withStandardRetryPolicy = with().pollDelay(2, SECONDS)
+    withStandardRetryPolicy = with().pollDelay(10, SECONDS)
         .and().with().pollInterval(10, SECONDS)
         .atMost(5, MINUTES).await();
 
@@ -131,7 +137,6 @@ public void testOperatorUpgradeFrom3_0_0(@Namespaces(3) List<String> namespaces)
     upgradeOperator("3.0.0", true);
   }
 
-
   private void upgradeOperator(String operatorVersion, boolean useHelmUpgrade) {
     logger.info("Assign a unique namespace for operator {0}", operatorVersion);
     assertNotNull(namespaces.get(0), "Namespace is null");
@@ -142,6 +147,7 @@ private void upgradeOperator(String operatorVersion, boolean useHelmUpgrade) {
     logger.info("Assign a unique namespace for domain");
     assertNotNull(namespaces.get(2), "Namespace is null");
     String domainNamespace = namespaces.get(2);
+    latestOperatorImageName = getOperatorImageName();
 
     // delete existing CRD
     new Command()
@@ -168,17 +174,45 @@ private void upgradeOperator(String operatorVersion, boolean useHelmUpgrade) {
     // create domain
     createDomainHomeInImageAndVerify(domainNamespace, operatorVersion);
 
-    externalRestHttpsPort = getNextFreePort(31001, 31201);
     if (useHelmUpgrade) {
       // upgrade to latest operator
-      upgradeAndVerifyOperator(opNamespace, domainNamespace);
+      HelmParams upgradeHelmParams = new HelmParams()
+          .releaseName(OPERATOR_RELEASE_NAME)
+          .namespace(opNamespace)
+          .chartDir(OPERATOR_CHART_DIR)
+          .repoUrl(null)
+          .chartVersion(null)
+          .chartName(null);
+
+      // operator chart values
+      OperatorParams opParams = new OperatorParams()
+          .helmParams(upgradeHelmParams)
+          .image(latestOperatorImageName)
+          .externalRestEnabled(true);
+
+      upgradeAndVerifyOperator(opNamespace, opParams);
+
+      // check operator image name after upgrade
+      logger.info("Checking image name in operator container ");
+      withStandardRetryPolicy
+          .conditionEvaluationListener(
+              condition -> logger.info("Checking operator image name in namespace {0} after upgrade "
+                      + "(elapsed time {1}ms, remaining time {2}ms)",
+                  opNamespace1,
+                  condition.getElapsedTimeInMS(),
+                  condition.getRemainingTimeInMS()))
+          .until(assertDoesNotThrow(() -> getOpContainerImageName(opNamespace1),
+              "Exception while getting the operator image name"));
+
     } else {
       opNamespace = opNamespace2;
       opServiceAccount = opNamespace2 + "-sa";
+
       // uninstall operator 2.5.0/2.6.0
       uninstallOperator(opHelmParams);
 
       // install latest operator
+      externalRestHttpsPort = getNextFreePort(31001, 31201);
       installAndVerifyOperator(opNamespace, opServiceAccount, true, externalRestHttpsPort, domainNamespace);
     }
     // check CRD version is updated
@@ -223,11 +257,13 @@ private void upgradeOperator(String operatorVersion, boolean useHelmUpgrade) {
    */
   @AfterEach
   public void tearDown() {
-    CleanupUtil.cleanup(namespaces);
-    new Command()
-        .withParams(new CommandParams()
-            .command("kubectl delete crd domains.weblogic.oracle --ignore-not-found"))
-        .execute();
+    if (System.getenv("SKIP_CLEANUP") == null) {
+      CleanupUtil.cleanup(namespaces);
+      new Command()
+          .withParams(new CommandParams()
+              .command("kubectl delete crd domains.weblogic.oracle --ignore-not-found"))
+          .execute();
+    }
   }
 
   private void createDomainHomeInImageAndVerify(String domainNamespace, String operatorVersion) {
@@ -329,4 +365,21 @@ private void checkDomainStarted(String domainUid, String domainNamespace) {
     }
   }
 
+  private Callable<Boolean> getOpContainerImageName(String namespace) {
+    return () -> {
+      String imageName = getOperatorContainerImageName(namespace);
+      if (imageName != null) {
+        if (!imageName.equals(latestOperatorImageName)) {
+          logger.info("Operator image name {0} doesn't match with latest image {1}",
+              imageName, latestOperatorImageName);
+          return false;
+        } else {
+          logger.info("Operator image name {0}", imageName);
+          return true;
+        }
+      }
+      return false;
+    };
+  }
+
 }

new-integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/TestActions.java

@@ -123,6 +123,17 @@ public static String getOperatorImageName() {
   public static boolean buildOperatorImage(String image) {
     return Operator.buildImage(image);
   }
+
+  /**
+   * Ge the image name used in the Operator container.
+   * @param namespace namespace of the operator
+   * @return image name
+   * @throws ApiException if Kubernetes client API call fails
+   */
+  public static String getOperatorContainerImageName(String namespace) throws ApiException {
+    return Operator.getOperatorContainerImage(namespace);
+  }
+
   // ----------------------   domain  -----------------------------------
 
   /**

new-integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/Operator.java

@@ -5,6 +5,7 @@
 
 import java.util.Optional;
 
+import io.kubernetes.client.openapi.ApiException;
 import oracle.weblogic.kubernetes.actions.impl.primitive.Command;
 import oracle.weblogic.kubernetes.actions.impl.primitive.CommandParams;
 import oracle.weblogic.kubernetes.actions.impl.primitive.Helm;
@@ -15,6 +16,7 @@
 import static oracle.weblogic.kubernetes.TestConstants.IMAGE_NAME_OPERATOR;
 import static oracle.weblogic.kubernetes.TestConstants.OPERATOR_DOCKER_BUILD_SCRIPT;
 import static oracle.weblogic.kubernetes.TestConstants.REPO_NAME;
+import static oracle.weblogic.kubernetes.actions.impl.primitive.Kubernetes.getContainerImage;
 
 /**
  * Action class with implementation methods for Operator.
@@ -101,4 +103,15 @@ public static boolean buildImage(String image) {
             .command(command))
         .execute();
   }
+
+  /**
+   * Get the container's image in the pod.
+   * @param namespace name of the pod's namespace
+   * @return image used for the container
+   * @throws ApiException if Kubernetes client API call fails
+   */
+  public static String getOperatorContainerImage(String namespace) throws ApiException {
+    return getContainerImage(namespace, "weblogic-operator-",
+        String.format("weblogic.operatorName in (%s)", namespace), null);
+  }
 }

new-integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/primitive/Helm.java

@@ -186,7 +186,7 @@ public static boolean addRepo(String repoName, String repoUrl) {
    */
   private static String valuesToString(Map<String, Object> helmValues) {
     StringBuffer valuesString = new StringBuffer("");
-
+    getLogger().info("Helm values {0}", helmValues);
     // values can be Map or String
     for (Map.Entry<String,Object> entry : helmValues.entrySet()) {
       if (entry.getValue() instanceof Map) {

new-integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/primitive/Kubernetes.java

@@ -43,6 +43,7 @@
 import io.kubernetes.client.openapi.models.V1ClusterRoleList;
 import io.kubernetes.client.openapi.models.V1ConfigMap;
 import io.kubernetes.client.openapi.models.V1ConfigMapList;
+import io.kubernetes.client.openapi.models.V1Container;
 import io.kubernetes.client.openapi.models.V1ContainerStatus;
 import io.kubernetes.client.openapi.models.V1Deployment;
 import io.kubernetes.client.openapi.models.V1DeploymentList;
@@ -480,7 +481,7 @@ public static boolean deletePod(String name, String namespace) {
   public static V1Pod getPod(String namespace, String labelSelector, String podName) throws ApiException {
     V1PodList pods = listPods(namespace, labelSelector);
     for (var pod : pods.getItems()) {
-      if (podName.equals(pod.getMetadata().getName())) {
+      if (pod.getMetadata().getName().contains(podName)) {
         return pod;
       }
     }
@@ -557,6 +558,36 @@ public static int getContainerRestartCount(
     return 0;
   }
 
+  /**
+   * Get the container's image in the pod.
+   * @param namespace name of the pod's namespace
+   * @param labelSelector in the format "weblogic.operatorName in (%s)"
+   * @param podName name of the pod
+   * @param containerName name of the container, null if there is only one container
+   * @return image used for the container
+   * @throws ApiException if Kubernetes client API call fails
+   */
+  public static String getContainerImage(String namespace, String podName,
+                                         String labelSelector, String containerName) throws ApiException {
+    V1Pod pod = getPod(namespace, labelSelector, podName);
+    if (pod != null) {
+      List<V1Container> containerList = pod.getSpec().getContainers();
+      if (containerName == null && containerList.size() >= 1) {
+        return containerList.get(0).getImage();
+      } else {
+        for (V1Container container : containerList) {
+          if (containerName.equals(container.getName())) {
+            return container.getImage();
+          }
+        }
+        getLogger().info("Container {0} doesn't exist in pod {1} namespace {2}",
+            containerName, podName, namespace);
+      }
+    } else {
+      getLogger().severe("Pod " + podName + " doesn't exist in namespace " + namespace);
+    }
+    return null;
+  }
 
   /**
    * Get the weblogic.domainRestartVersion label from a given pod.

new-integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java

@@ -351,7 +351,6 @@ public static HelmParams installAndVerifyOperator(String opNamespace,
    */
   public static boolean upgradeAndVerifyOperator(String opNamespace,
                                                  String... domainNamespace) {
-    LoggingFacade logger = getLogger();
     // Helm upgrade parameters
     HelmParams opHelmParams = new HelmParams()
         .releaseName(OPERATOR_RELEASE_NAME)
@@ -363,6 +362,20 @@ public static boolean upgradeAndVerifyOperator(String opNamespace,
         .helmParams(opHelmParams)
         .domainNamespaces(Arrays.asList(domainNamespace));
 
+    return upgradeAndVerifyOperator(opNamespace, opParams);
+  }
+
+  /**
+   * Upgrade WebLogic operator with the helm values provided.
+   *
+   * @param opNamespace the operator namespace in which the operator will be upgraded
+   * @param opParams operator parameters to use in the upgrade
+   * @return true if successful
+   */
+  public static boolean upgradeAndVerifyOperator(String opNamespace, OperatorParams opParams) {
+
+    LoggingFacade logger = getLogger();
+
     // upgrade operator
     logger.info("Upgrading operator in namespace {0}", opNamespace);
     if (!upgradeOperator(opParams)) {