initial changes to add introspect version to pod labels

operator/src/main/java/oracle/kubernetes/operator/DomainProcessorImpl.java

@@ -887,7 +887,7 @@ Step createDomainUpPlan(DomainPresenceInfo info) {
     Step domainUpStrategy =
         Step.chain(
             domainIntrospectionSteps(info),
-            DomainValidationSteps.createAfterIntrospectValidationSteps(info.getDomainUid()),
+            DomainValidationSteps.createAfterIntrospectValidationSteps(),
             new DomainStatusStep(info, null),
             bringAdminServerUp(info, delegate.getPodAwaiterStepFactory(info.getNamespace())),
             managedServerStrategy);

operator/src/main/java/oracle/kubernetes/operator/helpers/AnnotationHelper.java

@@ -3,14 +3,17 @@
 
 package oracle.kubernetes.operator.helpers;
 
+import java.util.Collections;
 import java.util.Map;
 import java.util.Optional;
 import java.util.function.Function;
+import java.util.stream.Stream;
 
 import io.kubernetes.client.openapi.models.V1ObjectMeta;
 import io.kubernetes.client.openapi.models.V1Pod;
 import io.kubernetes.client.openapi.models.V1Service;
 import io.kubernetes.client.util.Yaml;
+import oracle.kubernetes.operator.LabelConstants;
 import org.apache.commons.codec.digest.DigestUtils;
 
 /** Annotates pods, services with details about the Domain instance and checks these annotations. */
@@ -51,7 +54,19 @@ private static V1Pod addHashAndDebug(V1Pod pod) {
   }
 
   private static V1Pod addHash(V1Pod pod) {
+    Map<String, String> labels =
+        Optional.ofNullable(pod).map(V1Pod::getMetadata).map(V1ObjectMeta::getLabels).orElse(Collections.emptyMap());
+    String introspectVersion = null;
+    boolean restoreNeeded = false;
+    if (!labels.isEmpty() && labels.containsKey(LabelConstants.INTROSPECTION_STATE_LABEL)) {
+      restoreNeeded = true;
+      introspectVersion = pod.getMetadata().getLabels().remove(LabelConstants.INTROSPECTION_STATE_LABEL);
+    }
+
     pod.getMetadata().putAnnotationsItem(SHA256_ANNOTATION, HASH_FUNCTION.apply(pod));
+    if (restoreNeeded) {
+      pod.getMetadata().getLabels().put(LabelConstants.INTROSPECTION_STATE_LABEL, introspectVersion);
+    }
     return pod;
   }
 

operator/src/main/java/oracle/kubernetes/operator/helpers/DomainValidationSteps.java

@@ -48,8 +48,8 @@ public static Step createAdditionalDomainValidationSteps(V1PodSpec podSpec) {
     return new DomainAdditionalValidationStep(podSpec);
   }
 
-  public static Step createAfterIntrospectValidationSteps(String domainUid) {
-    return new DomainAfterIntrospectValidationStep(domainUid);
+  public static Step createAfterIntrospectValidationSteps() {
+    return new DomainAfterIntrospectValidationStep();
   }
 
   private static Step createListSecretsStep(String domainNamespace) {
@@ -230,11 +230,6 @@ private boolean hasMatchingMetadata(V1ObjectMeta metadata, String name, String n
   }
 
   private static class DomainAfterIntrospectValidationStep extends Step {
-    private String domainUid;
-
-    public DomainAfterIntrospectValidationStep(String domainUid) {
-      this.domainUid = domainUid;
-    }
 
     @Override
     public NextAction apply(Packet packet) {

operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java

@@ -305,6 +305,10 @@ String getPodPatchedMessageKey() {
       return MessageKeys.ADMIN_POD_PATCHED;
     }
 
+    @Override
+    String getPodUpdatedMessageKey() {
+      return MessageKeys.ADMIN_POD_UPDATED;
+    }
     @Override
     String getPodReplacedMessageKey() {
       return MessageKeys.ADMIN_POD_REPLACED;
@@ -455,6 +459,11 @@ String getPodPatchedMessageKey() {
       return MessageKeys.MANAGED_POD_PATCHED;
     }
 
+    @Override
+    String getPodUpdatedMessageKey() {
+      return MessageKeys.MANAGED_POD_UPDATED;
+    }
+
     @Override
     protected String getPodReplacedMessageKey() {
       return MessageKeys.MANAGED_POD_REPLACED;

operator/src/main/java/oracle/kubernetes/operator/helpers/PodStepContext.java

@@ -7,6 +7,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -58,6 +59,8 @@
 import oracle.kubernetes.weblogic.domain.model.Shutdown;
 import org.apache.commons.lang3.builder.EqualsBuilder;
 
+import static oracle.kubernetes.operator.LabelConstants.INTROSPECTION_STATE_LABEL;
+
 public abstract class PodStepContext extends BasePodStepContext {
 
   private static final LoggingFacade LOGGER = LoggingFactory.getLogger("Operator", "Operator");
@@ -364,6 +367,24 @@ protected Step patchPod(V1Pod currentPod, Step next) {
             new V1Patch(patchBuilder.build().toString()), patchResponse(next));
   }
 
+  private Step updateCurrentPod(V1Pod currentPod, Step next) {
+    return createProgressingStep(updatePod(currentPod, next));
+  }
+
+  protected Step updatePod(V1Pod currentPod, Step next) {
+    JsonPatchBuilder patchBuilder = Json.createPatchBuilder();
+    String baseName = "/metadata/labels/";
+    if (!getPodLabels().containsKey(INTROSPECTION_STATE_LABEL)) {
+      patchBuilder.add(baseName + INTROSPECTION_STATE_LABEL, getDomain().getSpec().getIntrospectVersion());
+    } else {
+      patchBuilder.replace(baseName + INTROSPECTION_STATE_LABEL, getDomain().getSpec().getIntrospectVersion());
+    }
+
+    return new CallBuilder()
+        .patchPodAsync(getPodName(), getNamespace(), getDomainUid(),
+            new V1Patch(patchBuilder.build().toString()), updateResponse(next));
+  }
+
   private Map<String, String> getLabels(V1Pod pod) {
     return Optional.ofNullable(pod.getMetadata()).map(V1ObjectMeta::getLabels).orElseGet(Collections::emptyMap);
   }
@@ -384,6 +405,10 @@ private void logPodPatched() {
     LOGGER.info(getPodPatchedMessageKey(), getDomainUid(), getServerName());
   }
 
+  private void logPodUpdated() {
+    LOGGER.info(getPodUpdatedMessageKey(), getDomainUid(), getServerName(), getDomain().getIntrospectVersion());
+  }
+
   private void logPodReplaced() {
     LOGGER.info(getPodReplacedMessageKey(), getDomainUid(), getServerName());
   }
@@ -394,6 +419,8 @@ private void logPodReplaced() {
 
   abstract String getPodPatchedMessageKey();
 
+  abstract String getPodUpdatedMessageKey();
+
   abstract String getPodReplacedMessageKey();
 
   Step createCyclePodStep(Step next) {
@@ -439,6 +466,10 @@ private ResponseStep<V1Pod> patchResponse(Step next) {
     return new PatchPodResponseStep(next);
   }
 
+  private ResponseStep<V1Pod> updateResponse(Step next) {
+    return new UpdatePodResponseStep(next);
+  }
+
   V1Pod createPodModel() {
     return withNonHashedElements(AnnotationHelper.withSha256Hash(createPodRecipe()));
   }
@@ -544,6 +575,8 @@ LabelConstants.CLUSTERRESTARTVERSION_LABEL, getServerSpec().getClusterRestartVer
         .putLabelsItem(
             LabelConstants.SERVERRESTARTVERSION_LABEL, getServerSpec().getServerRestartVersion());
 
+    Optional.ofNullable(getDomain().getSpec().getIntrospectVersion())
+        .ifPresent(version -> addIntrospectVersionLabel(metadata, version));
     Optional.ofNullable(miiDomainZipHash)
           .ifPresent(hash -> addHashLabel(metadata, LabelConstants.MODEL_IN_IMAGE_DOMAINZIP_HASH, hash));
     Optional.ofNullable(miiModelSecretsHash)
@@ -556,6 +589,10 @@ LabelConstants.CLUSTERRESTARTVERSION_LABEL, getServerSpec().getClusterRestartVer
     return metadata;
   }
 
+  private void addIntrospectVersionLabel(V1ObjectMeta metadata, String introspectVersion) {
+    metadata.putLabelsItem(INTROSPECTION_STATE_LABEL, introspectVersion);
+  }
+
   private void addHashLabel(V1ObjectMeta metadata, String label, String hash) {
     metadata.putLabelsItem(label, formatHashLabel(hash));
   }
@@ -847,6 +884,10 @@ public NextAction apply(Packet packet) {
         return doNext(patchCurrentPod(currentPod, getNext()), packet);
       } else {
         logPodExists();
+        if (!Objects.equals(currentPod.getMetadata().getLabels().get(INTROSPECTION_STATE_LABEL),
+            getDomain().getSpec().getIntrospectVersion())) {
+          return doNext(updateCurrentPod(currentPod, getNext()), packet);
+        }
         return doNext(packet);
       }
     }
@@ -965,4 +1006,25 @@ public NextAction onSuccess(Packet packet, CallResponse<V1Pod> callResponse) {
     }
   }
 
+  private class UpdatePodResponseStep extends BaseResponseStep {
+    private final Step next;
+
+    UpdatePodResponseStep(Step next) {
+      super(next);
+      this.next = next;
+    }
+
+    @Override
+    public NextAction onSuccess(Packet packet, CallResponse<V1Pod> callResponse) {
+
+      V1Pod newPod = callResponse.getResult();
+      logPodUpdated();
+      if (newPod != null) {
+        setRecordedPod(newPod);
+      }
+
+      return doNext(next, packet);
+    }
+  }
+
 }

operator/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java

@@ -133,6 +133,8 @@ public class MessageKeys {
   public static final String INTROSPECTOR_JOB_FAILED = "WLSKO-0175";
   public static final String INTROSPECTOR_JOB_FAILED_DETAIL = "WLSKO-0176";
   public static final String INTROSPECTOR_POD_FAILED = "WLSKO-0177";
+  public static final String ADMIN_POD_UPDATED = "WLSKO-0178";
+  public static final String MANAGED_POD_UPDATED = "WLSKO-0179";
 
   // domain status messages
   public static final String DUPLICATE_SERVER_NAME_FOUND = "WLSDO-0001";

operator/src/main/resources/Operator.properties

@@ -187,6 +187,8 @@ WLSKO-0175=Job {0} in namespace {1} failed with status {2}. Check log messages \
   copied from the introspector pod {3} log for additional information.
 WLSKO-0176=Job {1} in namespace {0} failed, job details are {2}
 WLSKO-0177=Pod {0} in namespace {1} failed, the pod status is {2}
+WLSKO-0178=Patching administration server Pod with ''weblogic.introspectVersion'' {2} for WebLogic domain with UID: {0}. Administration server name: {1}.
+WLSKO-0179=Patching managed server Pod with ''weblogic.introspectVersion'' {2} for WebLogic domain with UID: {0}. Managed server name: {1}.
 
 # Domain status messages
 

operator/src/test/java/oracle/kubernetes/operator/DomainProcessorTest.java

@@ -414,6 +414,91 @@ private Stream<V1ConfigMap> getConfigMaps() {
     return testSupport.<V1ConfigMap>getResources(CONFIG_MAP).stream();
   }
 
+  @Test
+  public void afterInitialIntrospection_serverPodsHaveInitialIntrospectVersionLabel() throws Exception {
+    domainConfigurator.withIntrospectVersion(OLD_INTROSPECTION_STATE);
+    testSupport.doOnCreate(POD, p -> recordPodCreation((V1Pod) p));
+    domainConfigurator.configureCluster(CLUSTER).withReplicas(MIN_REPLICAS);
+    DomainPresenceInfo info = new DomainPresenceInfo(newDomain);
+    processor.createMakeRightOperation(info).withExplicitRecheck().execute();
+    List<V1Pod> runningPods = getRunningPods();
+
+    //one introspector pod, one admin server pod and two managed server pods
+    assertThat(runningPods.size(), equalTo(4));
+    for (V1Pod pod: runningPods) {
+      if (!pod.getMetadata().getName().contains(LegalNames.getIntrospectorJobNameSuffix())) {
+        assertThat(getServerPodIntrospectionVersion(pod), equalTo(OLD_INTROSPECTION_STATE));
+      }
+    }
+  }
+
+  @Test
+  public void afterIntrospection_serverPodsHaveUpToDateIntrospectVersionLabel() throws Exception {
+    establishPreviousIntrospection(null);
+    domainConfigurator.withIntrospectVersion(NEW_INTROSPECTION_STATE);
+    DomainPresenceInfo info = new DomainPresenceInfo(newDomain);
+    processor.createMakeRightOperation(info).withExplicitRecheck().execute();
+
+    assertThat(job, notNullValue());
+
+    List<V1Pod> runningPods = getRunningPods();
+
+    //one introspector pod, one admin server pod and two managed server pods
+    assertThat(runningPods.size(), equalTo(4));
+
+    for (V1Pod pod: runningPods) {
+      if (!pod.getMetadata().getName().contains(LegalNames.getIntrospectorJobNameSuffix())) {
+        assertThat(getServerPodIntrospectionVersion(pod), equalTo(NEW_INTROSPECTION_STATE));
+      }
+    }
+  }
+
+  @Test
+  public void afterScaleupClusterIntrospection_serverPodsHaveUpToDateIntrospectVersionLabel() throws Exception {
+    establishPreviousIntrospection(null);
+    domainConfigurator.configureCluster(CLUSTER).withReplicas(3);
+    domainConfigurator.withIntrospectVersion("after-scaleup");
+    DomainPresenceInfo info = new DomainPresenceInfo(newDomain);
+    processor.createMakeRightOperation(info).withExplicitRecheck().execute();
+    List<V1Pod> runningPods = getRunningPods();
+
+    //one introspector pod, one admin server pod and three managed server pods
+    assertThat(runningPods.size(), equalTo(5));
+
+    for (V1Pod pod: runningPods) {
+      if (!pod.getMetadata().getName().contains(LegalNames.getIntrospectorJobNameSuffix())) {
+        assertThat(getServerPodIntrospectionVersion(pod), equalTo("after-scaleup"));
+      }
+    }
+  }
+
+  @Test
+  public void afterScaledownClusterIntrospection_serverPodsHaveUpToDateIntrospectVersionLabel() throws Exception {
+    establishPreviousIntrospection(null);
+    domainConfigurator.configureCluster(CLUSTER).withReplicas(1);
+    domainConfigurator.withIntrospectVersion("after-scaledown");
+    DomainPresenceInfo info = new DomainPresenceInfo(newDomain);
+    processor.createMakeRightOperation(info).withExplicitRecheck().execute();
+    List<V1Pod> runningPods = getRunningPods();
+
+    //one introspector pod, one admin server pod and one managed server pod
+    assertThat(runningPods.size(), equalTo(3));
+
+    for (V1Pod pod: runningPods) {
+      if (!pod.getMetadata().getName().contains(LegalNames.getIntrospectorJobNameSuffix())) {
+        assertThat(getServerPodIntrospectionVersion(pod), equalTo("after-scaledown"));
+      }
+    }
+  }
+
+  private String getServerPodIntrospectionVersion(V1Pod pod) {
+    return Optional.ofNullable(pod)
+        .map(V1Pod::getMetadata)
+        .map(V1ObjectMeta::getLabels)
+        .map(m -> m.get(INTROSPECTION_STATE_LABEL))
+        .orElse(null);
+  }
+
   private boolean isIntrospectorMeta(@Nullable V1ObjectMeta meta) {
     return meta != null && NS.equals(meta.getNamespace()) && INTROSPECTOR_MAP_NAME.equals(meta.getName());
   }

operator/src/test/java/oracle/kubernetes/operator/helpers/PodHelperTestBase.java

@@ -765,11 +765,21 @@ void initializeExistingPod() {
     initializeExistingPod(createPodModel());
   }
 
+  void initializeExistingPodWithIntrospectVersion(String introspectVersion) {
+    initializeExistingPodWithIntrospectVersion(createPodModel(), introspectVersion);
+  }
+
   void initializeExistingPod(V1Pod pod) {
     testSupport.defineResources(pod);
     domainPresenceInfo.setServerPod(getServerName(), pod);
   }
 
+  void initializeExistingPodWithIntrospectVersion(V1Pod pod, String introspectVersion) {
+    testSupport.defineResources(pod);
+    pod.getMetadata().getLabels().put(LabelConstants.INTROSPECTION_STATE_LABEL, introspectVersion);
+    domainPresenceInfo.setServerPod(getServerName(), pod);
+  }
+
   private V1Pod createPodModel() {
     return createPod(testSupport.getPacket());
   }
@@ -952,6 +962,15 @@ public void whenServerConfigurationAddsIntrospectionVersion_dontReplacePod() {
     verifyPodNotReplaced();
   }
 
+  @Test
+  public void whenServerConfigurationIntrospectionVersionTheSame_dontReplacePod() {
+    initializeExistingPodWithIntrospectVersion("123");
+
+    configurator.withIntrospectVersion("123");
+
+    verifyPodNotReplaced();
+  }
+
   @Test
   public void whenServerListenPortChanged_replacePod() {
     initializeExistingPod();