Fix for ready replicas and server state after the domain is shut down.

operator/src/main/java/oracle/kubernetes/operator/DomainProcessorImpl.java

@@ -118,7 +118,7 @@ public DomainProcessorImpl(DomainProcessorDelegate delegate, SemanticVersion pro
     this.productVersion = productVersion;
   }
 
-  private static DomainPresenceInfo getExistingDomainPresenceInfo(String ns, String domainUid) {
+  static DomainPresenceInfo getExistingDomainPresenceInfo(String ns, String domainUid) {
     return DOMAINS.computeIfAbsent(ns, k -> new ConcurrentHashMap<>()).get(domainUid);
   }
 
@@ -419,7 +419,6 @@ private void processServerPodWatch(V1Pod pod, String watchType) {
           createMakeRightOperation(info).interrupt().withExplicitRecheck().execute();
         }
         break;
-
       case "ERROR":
       default:
     }

operator/src/main/java/oracle/kubernetes/operator/DomainStatusUpdater.java

@@ -31,6 +31,7 @@
 import oracle.kubernetes.operator.helpers.DomainPresenceInfo.ServerStartupInfo;
 import oracle.kubernetes.operator.helpers.EventHelper;
 import oracle.kubernetes.operator.helpers.EventHelper.EventData;
+import oracle.kubernetes.operator.helpers.LastKnownStatus;
 import oracle.kubernetes.operator.helpers.PodHelper;
 import oracle.kubernetes.operator.helpers.ResponseStep;
 import oracle.kubernetes.operator.logging.LoggingFacade;
@@ -57,6 +58,7 @@
 import oracle.kubernetes.weblogic.domain.model.ServerStatus;
 import org.jetbrains.annotations.NotNull;
 
+import static oracle.kubernetes.operator.DomainProcessorImpl.getExistingDomainPresenceInfo;
 import static oracle.kubernetes.operator.LabelConstants.CLUSTERNAME_LABEL;
 import static oracle.kubernetes.operator.MIINonDynamicChangesMethod.CommitUpdateOnly;
 import static oracle.kubernetes.operator.ProcessingConstants.DOMAIN_TOPOLOGY;
@@ -71,6 +73,7 @@
 import static oracle.kubernetes.operator.ProcessingConstants.SERVER_STATE_MAP;
 import static oracle.kubernetes.operator.WebLogicConstants.RUNNING_STATE;
 import static oracle.kubernetes.operator.WebLogicConstants.SHUTDOWN_STATE;
+import static oracle.kubernetes.operator.WebLogicConstants.SHUTTING_DOWN_STATE;
 import static oracle.kubernetes.operator.helpers.EventHelper.EventItem.DOMAIN_PROCESSING_ABORTED;
 import static oracle.kubernetes.operator.helpers.EventHelper.EventItem.DOMAIN_PROCESSING_STARTING;
 import static oracle.kubernetes.weblogic.domain.model.DomainConditionType.Available;
@@ -420,7 +423,7 @@ boolean isStatusUnchanged(DomainStatus newStatus) {
       return newStatus.equals(getStatus());
     }
 
-    private String getNamespace() {
+    String getNamespace() {
       return getMetadata().getNamespace();
     }
 
@@ -567,7 +570,7 @@ private boolean stillHasPodPendingRestart(DomainStatus status) {
       }
 
       private V1Pod getServerPod(ServerStatus serverStatus) {
-        return getInfo().getServerPod(serverStatus.getServerName());
+        return getCachedInfo().getServerPod(serverStatus.getServerName());
       }
 
       private Map<String, String> getLabels(V1Pod pod) {
@@ -585,19 +588,24 @@ private boolean allIntendedServersRunning() {
       }
 
       private Stream<ServerStartupInfo> getServerStartupInfos() {
-        return Optional.ofNullable(getInfo().getServerStartupInfo()).stream().flatMap(Collection::stream);
+        return Optional.ofNullable(getCachedInfo()
+                .getServerStartupInfo()).stream().flatMap(Collection::stream);
       }
 
       private Optional<WlsDomainConfig> getDomainConfig() {
         return Optional.ofNullable(config).or(this::getScanCacheDomainConfig);
       }
 
       private Optional<WlsDomainConfig> getScanCacheDomainConfig() {
-        DomainPresenceInfo info = getInfo();
+        DomainPresenceInfo info = getCachedInfo();
         Scan scan = ScanCache.INSTANCE.lookupScan(info.getNamespace(), info.getDomainUid());
         return Optional.ofNullable(scan).map(Scan::getWlsDomainConfig);
       }
 
+      private DomainPresenceInfo getCachedInfo() {
+        return Optional.ofNullable(getExistingDomainPresenceInfo(getNamespace(), getDomainUid())).orElse(getInfo());
+      }
+
       private boolean shouldBeRunning(ServerStartupInfo startupInfo) {
         return startupInfo.isNotServiceOnly() && RUNNING_STATE.equals(startupInfo.getDesiredState());
       }
@@ -607,15 +615,17 @@ private boolean isNotRunning(@Nonnull String serverName) {
       }
 
       private boolean isHasFailedPod() {
-        return getInfo().getServerPods().anyMatch(PodHelper::isFailed);
+        return getCachedInfo().getServerPods().anyMatch(PodHelper::isFailed);
       }
 
       private boolean hasServerPod(String serverName) {
-        return Optional.ofNullable(getInfo().getServerPod(serverName)).isPresent();
+        return Optional.ofNullable(getCachedInfo().getServerPod(serverName)).isPresent();
       }
 
       private boolean hasReadyServerPod(String serverName) {
-        return Optional.ofNullable(getInfo().getServerPod(serverName)).filter(PodHelper::getReadyStatus).isPresent();
+        return (!getCachedInfo().isServerPodBeingDeleted(serverName))
+                && Optional.ofNullable(getCachedInfo().getServerPod(serverName))
+                .filter(PodHelper::getReadyStatus).isPresent();
       }
 
       Map<String, ServerStatus> getServerStatuses(final String adminServerName) {
@@ -637,7 +647,27 @@ private ServerStatus createServerStatus(String serverName, boolean isAdminServer
       }
 
       private String getRunningState(String serverName) {
-        return Optional.ofNullable(serverState).map(m -> m.get(serverName)).orElse(null);
+        String state = Optional.ofNullable(serverState).map(m -> m.get(serverName)).orElse(null);
+        if (getCachedInfo().getServerPod(serverName) == null) {
+          return SHUTDOWN_STATE;
+        } else if (isDeleting(serverName)) {
+          return SHUTTING_DOWN_STATE;
+        }
+        return state;
+      }
+
+      private boolean isDeleting(String serverName) {
+        return Optional.ofNullable(getCachedInfo().getServerPod(serverName))
+                .map(p -> PodHelper.isDeleting(p)).orElse(false);
+      }
+
+      private boolean isStateShutdown(String serverName) {
+        return isStateShutdown(getCachedInfo().getLastKnownServerStatus(serverName), SHUTDOWN_STATE);
+      }
+
+      private boolean isStateShutdown(LastKnownStatus lastKnownStatus, String shutdownState) {
+        String status = Optional.ofNullable(lastKnownStatus).map(s -> s.getStatus()).orElse(null);
+        return Objects.equals(status, shutdownState);
       }
 
       private String getDesiredState(String serverName, String clusterName, boolean isAdminServer) {
@@ -697,7 +727,7 @@ private ClusterStatus createClusterStatus(String clusterName) {
 
 
       private String getNodeName(String serverName) {
-        return Optional.ofNullable(getInfo().getServerPod(serverName))
+        return Optional.ofNullable(getCachedInfo().getServerPod(serverName))
             .map(V1Pod::getSpec)
             .map(V1PodSpec::getNodeName)
             .orElse(null);

operator/src/main/java/oracle/kubernetes/operator/JobWatcher.java

@@ -28,6 +28,7 @@
 import oracle.kubernetes.operator.builders.WatchBuilder;
 import oracle.kubernetes.operator.calls.CallResponse;
 import oracle.kubernetes.operator.helpers.CallBuilder;
+import oracle.kubernetes.operator.helpers.DomainPresenceInfo;
 import oracle.kubernetes.operator.helpers.KubernetesUtils;
 import oracle.kubernetes.operator.helpers.ResponseStep;
 import oracle.kubernetes.operator.logging.LoggingFacade;
@@ -241,9 +242,8 @@ private WaitForJobReadyStep(V1Job job, Step next) {
           metadata.getCreationTimestamp());
     }
 
-    // A job is considered ready once it has either successfully completed, or been marked as failed.
     @Override
-    boolean isReady(V1Job job) {
+    boolean isReady(V1Job job, DomainPresenceInfo info, String serverName) {
       return isComplete(job) || isFailed(job);
     }
 
@@ -255,8 +255,8 @@ boolean onReadNotFoundForCachedResource(V1Job cachedJob, boolean isNotFoundOnRea
     // Ignore modified callbacks from different jobs (identified by having different creation times) or those
     // where the job is not yet ready.
     @Override
-    boolean shouldProcessCallback(V1Job job) {
-      return hasExpectedCreationTime(job) && isReady(job);
+    boolean shouldProcessCallback(V1Job job, Packet packet) {
+      return hasExpectedCreationTime(job) && isReady(job, null, null);
     }
 
     private boolean hasExpectedCreationTime(V1Job job) {
@@ -317,7 +317,7 @@ protected DefaultResponseStep<V1Job> resumeIfReady(Callback callback) {
       return new DefaultResponseStep<>(null) {
         @Override
         public NextAction onSuccess(Packet packet, CallResponse<V1Job> callResponse) {
-          if (isReady(callResponse.getResult()) || callback.didResumeFiber()) {
+          if (isReady(callResponse.getResult(), null, null) || callback.didResumeFiber()) {
             callback.proceedFromWait(callResponse.getResult());
             return doNext(packet);
           }

operator/src/main/java/oracle/kubernetes/operator/PodWatcher.java

@@ -353,7 +353,7 @@ public NextAction onSuccess(Packet packet, CallResponse<V1Pod> callResponse) {
             }
           }
 
-          if (isReady(callResponse.getResult()) || callback.didResumeFiber()) {
+          if (isReady(callResponse.getResult(), info, serverName) || callback.didResumeFiber()) {
             callback.proceedFromWait(callResponse.getResult());
             return null;
           }
@@ -407,16 +407,17 @@ private WaitForPodReadyStep(String podName, Step next) {
       super(podName, next);
     }
 
-    // A pod is ready if it is not being deleted and has the ready status.
     @Override
-    protected boolean isReady(V1Pod result) {
+    protected boolean isReady(V1Pod result, DomainPresenceInfo info, String serverName) {
       return result != null && !PodHelper.isDeleting(result) && PodHelper.isReady(result);
     }
 
     // Pods should be processed if ready.
     @Override
-    boolean shouldProcessCallback(V1Pod resource) {
-      return isReady(resource);
+    boolean shouldProcessCallback(V1Pod resource, Packet packet) {
+      DomainPresenceInfo info = packet.getSpi(DomainPresenceInfo.class);
+      String serverName = (String)packet.get(SERVER_NAME);
+      return isReady(resource, info, serverName);
     }
 
     @Override
@@ -452,9 +453,11 @@ protected boolean onReadNotFoundForCachedResource(V1Pod cachedPod, boolean isNot
       return false;
     }
 
-    // A pod is considered deleted when reading its value from Kubernetes returns null.
     @Override
-    protected boolean isReady(V1Pod result) {
+    protected boolean isReady(V1Pod result, DomainPresenceInfo info, String serverName) {
+      if (result == null) {
+        Optional.ofNullable(info).ifPresent(i -> i.setServerPod(serverName, null));
+      }
       return result == null;
     }
 

operator/src/main/java/oracle/kubernetes/operator/WaitForReadyStep.java

@@ -23,6 +23,7 @@
 import oracle.kubernetes.weblogic.domain.model.Domain;
 
 import static oracle.kubernetes.operator.ProcessingConstants.MAKE_RIGHT_DOMAIN_OPERATION;
+import static oracle.kubernetes.operator.ProcessingConstants.SERVER_NAME;
 import static oracle.kubernetes.operator.helpers.KubernetesUtils.getDomainUidLabel;
 
 /**
@@ -79,9 +80,11 @@ static int getWatchBackstopRecheckCount() {
   /**
    * Returns true if the specified resource is deemed "ready." Different steps may define readiness in different ways.
    * @param resource the resource to check
+   * @param info domain presence info
+   * @param serverName Server name
    * @return true if processing can proceed
    */
-  abstract boolean isReady(T resource);
+  abstract boolean isReady(T resource, DomainPresenceInfo info, String serverName);
 
   /**
    * Returns true if the cached resource is not found during periodic listing.
@@ -100,7 +103,7 @@ static int getWatchBackstopRecheckCount() {
    * @param resource the resource to check
    * @return true if the resource is expected
    */
-  boolean shouldProcessCallback(T resource) {
+  boolean shouldProcessCallback(T resource, Packet packet) {
     return true;
   }
 
@@ -174,9 +177,11 @@ void logWaiting(String name) {
 
   @Override
   public final NextAction apply(Packet packet) {
+    String serverName = (String)packet.get(SERVER_NAME);
+    DomainPresenceInfo info = packet.getSpi(DomainPresenceInfo.class);
     if (shouldTerminateFiber(initialResource)) {
       return doTerminate(createTerminationException(initialResource), packet);
-    } else if (isReady(initialResource)) {
+    } else if (isReady(initialResource, packet.getSpi(DomainPresenceInfo.class), serverName)) {
       return doNext(packet);
     }
 
@@ -283,7 +288,7 @@ class Callback implements Consumer<T> {
 
     @Override
     public void accept(T resource) {
-      boolean shouldProcessCallback = shouldProcessCallback(resource);
+      boolean shouldProcessCallback = shouldProcessCallback(resource, packet);
       if (shouldProcessCallback) {
         proceedFromWait(resource);
       }

operator/src/main/java/oracle/kubernetes/operator/steps/ReadHealthStep.java

@@ -26,6 +26,8 @@
 import oracle.kubernetes.operator.Pair;
 import oracle.kubernetes.operator.ProcessingConstants;
 import oracle.kubernetes.operator.WebLogicConstants;
+import oracle.kubernetes.operator.calls.CallResponse;
+import oracle.kubernetes.operator.helpers.CallBuilder;
 import oracle.kubernetes.operator.helpers.DomainPresenceInfo;
 import oracle.kubernetes.operator.helpers.SecretHelper;
 import oracle.kubernetes.operator.http.HttpResponseStep;
@@ -227,10 +229,42 @@ public NextAction onSuccess(Packet packet, HttpResponse<String> response) {
 
     @Override
     public NextAction onFailure(Packet packet, HttpResponse<String> response) {
-      new HealthResponseProcessing(packet, response).recordFailedStateAndHealth();
-      return doNext(packet);
+      DomainPresenceInfo info = packet.getSpi(DomainPresenceInfo.class);
+      String serverName = (String) packet.get(ProcessingConstants.SERVER_NAME);
+      return doNext(listPodStep(serverName, info, response, getNext()), packet);
+    }
+
+    private Step listPodStep(String name, DomainPresenceInfo info, HttpResponse httpResponse, Step next) {
+      return new CallBuilder().readPodAsync(name, info.getNamespace(), info.getDomainUid(),
+              new ReadPodResponseStep(name, httpResponse, next));
     }
 
+    private class ReadPodResponseStep extends DefaultResponseStep<V1Pod> {
+      private final HttpResponse httpResponse;
+      private final String serverName;
+
+      ReadPodResponseStep(String serverName, HttpResponse httpResponse, Step next) {
+        super(next);
+        this.httpResponse = httpResponse;
+        this.serverName = serverName;
+      }
+
+      @Override
+      public NextAction onFailure(Packet packet, CallResponse<V1Pod> callResponse) {
+        if (callResponse.getStatusCode() == CallBuilder.NOT_FOUND) {
+          DomainPresenceInfo info = packet.getSpi(DomainPresenceInfo.class);
+          info.setServerPod(serverName, null);
+          return doNext(packet);
+        }
+        return onFailure(packet, callResponse);
+      }
+
+      @Override
+      public NextAction onSuccess(Packet packet, CallResponse<V1Pod> callResponse) {
+        new HealthResponseProcessing(packet, httpResponse).recordFailedStateAndHealth();
+        return doNext(packet);
+      }
+    }
 
     static class HealthResponseProcessing {
       private final String serverName;

operator/src/main/java/oracle/kubernetes/operator/steps/ServerDownIteratorStep.java

@@ -9,11 +9,13 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Queue;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
+import oracle.kubernetes.operator.PodAwaiterStepFactory;
 import oracle.kubernetes.operator.ProcessingConstants;
 import oracle.kubernetes.operator.helpers.DomainPresenceInfo;
 import oracle.kubernetes.operator.helpers.ServiceHelper;
@@ -52,10 +54,21 @@ public NextAction apply(Packet packet) {
     getShutdownClusteredServersStepFactories(getServerShutdownInfos(), packet).values()
             .forEach(factory -> shutdownDetails.addAll(factory.getServerShutdownStepAndPackets(info)));
 
+    Collection<StepAndPacket> shutdownWaiters =
+            getServerShutdownInfos().stream()
+                    .map(ssi -> createServerDownWaiters(packet, ssi)).collect(Collectors.toList());
+    shutdownDetails.addAll(shutdownWaiters);
     return doNext((new ShutdownManagedServersStep(shutdownDetails, getNext())), packet);
 
   }
 
+  private StepAndPacket createServerDownWaiters(Packet packet, DomainPresenceInfo.ServerShutdownInfo ssi) {
+    DomainPresenceInfo info = packet.getSpi(DomainPresenceInfo.class);
+    return new StepAndPacket(Optional.ofNullable(packet.getSpi(PodAwaiterStepFactory.class))
+            .map(p -> p.waitForDelete(info.getServerPod(ssi.getServerName()), null)).orElse(null),
+            createPacketForServer(packet, ssi));
+  }
+
   // pre-conditions: DomainPresenceInfo SPI
   // "principal"
   // "serverScan"

operator/src/main/java/oracle/kubernetes/weblogic/domain/model/ClusterStatus.java

@@ -104,7 +104,7 @@ public ClusterStatus withReplicas(Integer replicas) {
     return this;
   }
 
-  Integer getReadyReplicas() {
+  public Integer getReadyReplicas() {
     return readyReplicas;
   }