Cache secret for a configurable time period and reset on auth/authz failure

operator/src/main/java/oracle/kubernetes/operator/JobWatcher.java

@@ -34,6 +34,7 @@
 import oracle.kubernetes.operator.watcher.WatchListener;
 import oracle.kubernetes.operator.work.Packet;
 import oracle.kubernetes.operator.work.Step;
+import oracle.kubernetes.utils.SystemClock;
 
 /** Watches for Jobs to become Ready or leave Ready state. */
 public class JobWatcher extends Watcher<V1Job> implements WatchListener<V1Job>, JobAwaiterStepFactory {
@@ -322,7 +323,7 @@ public String toString() {
 
     private long getJobStartedSeconds() {
       if (job.getStatus() != null && job.getStatus().getStartTime() != null) {
-        return ChronoUnit.SECONDS.between(job.getStatus().getStartTime(), OffsetDateTime.now());
+        return ChronoUnit.SECONDS.between(job.getStatus().getStartTime(), SystemClock.now());
       }
       return -1;
     }

operator/src/main/java/oracle/kubernetes/operator/Main.java

@@ -57,6 +57,7 @@
 import oracle.kubernetes.operator.work.Packet;
 import oracle.kubernetes.operator.work.Step;
 import oracle.kubernetes.operator.work.ThreadFactorySingleton;
+import oracle.kubernetes.utils.SystemClock;
 import oracle.kubernetes.weblogic.domain.model.DomainList;
 
 import static oracle.kubernetes.operator.helpers.NamespaceHelper.getOperatorNamespace;
@@ -74,7 +75,7 @@ public class Main {
   private static final ScheduledExecutorService wrappedExecutorService =
       Engine.wrappedExecutorService("operator", container);
   private static final AtomicReference<OffsetDateTime> lastFullRecheck =
-      new AtomicReference<>(OffsetDateTime.now());
+      new AtomicReference<>(SystemClock.now());
   private static final Semaphore shutdownSignal = new Semaphore(0);
   private static final int DEFAULT_STUCK_POD_RECHECK_SECONDS = 30;
 

operator/src/main/java/oracle/kubernetes/operator/ServerStatusReader.java

@@ -6,7 +6,6 @@
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.Reader;
-import java.time.OffsetDateTime;
 import java.util.Collection;
 import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
@@ -38,6 +37,7 @@
 import oracle.kubernetes.operator.work.Packet;
 import oracle.kubernetes.operator.work.Step;
 import oracle.kubernetes.utils.OperatorUtils;
+import oracle.kubernetes.utils.SystemClock;
 import oracle.kubernetes.weblogic.domain.model.ServerHealth;
 
 import static oracle.kubernetes.operator.KubernetesConstants.CONTAINER_NAME;
@@ -149,7 +149,7 @@ public NextAction apply(Packet packet) {
       if (lastKnownStatus != null
           && !WebLogicConstants.UNKNOWN_STATE.equals(lastKnownStatus.getStatus())
           && lastKnownStatus.getUnchangedCount() >= main.unchangedCountToDelayStatusRecheck) {
-        if (OffsetDateTime.now()
+        if (SystemClock.now()
             .isBefore(lastKnownStatus.getTime().plusSeconds((int) main.eventualLongDelay))) {
           String state = lastKnownStatus.getStatus();
           serverStateMap.put(serverName, state);

operator/src/main/java/oracle/kubernetes/operator/TuningParameters.java

@@ -40,6 +40,7 @@ class MainTuning {
     public final int stuckPodRecheckSeconds;
     public final long initialShortDelay;
     public final long eventualLongDelay;
+    public final int weblogicCredentialsSecretRereadIntervalSeconds;
 
     /**
      * create main tuning.
@@ -53,6 +54,7 @@ class MainTuning {
      * @param stuckPodRecheckSeconds time between checks for stuck pods
      * @param initialShortDelay initial short delay
      * @param eventualLongDelay eventual long delay
+     * @param weblogicCredentialsSecretRereadIntervalSeconds credentials secret reread interval
      */
     public MainTuning(
           int initializationRetryDelaySeconds,
@@ -64,7 +66,8 @@ public MainTuning(
           int unchangedCountToDelayStatusRecheck,
           int stuckPodRecheckSeconds,
           long initialShortDelay,
-          long eventualLongDelay) {
+          long eventualLongDelay,
+          int weblogicCredentialsSecretRereadIntervalSeconds) {
       this.initializationRetryDelaySeconds = initializationRetryDelaySeconds;
       this.domainPresenceFailureRetrySeconds = domainPresenceFailureRetrySeconds;
       this.domainPresenceFailureRetryMaxCount = domainPresenceFailureRetryMaxCount;
@@ -75,6 +78,7 @@ public MainTuning(
       this.stuckPodRecheckSeconds = stuckPodRecheckSeconds;
       this.initialShortDelay = initialShortDelay;
       this.eventualLongDelay = eventualLongDelay;
+      this.weblogicCredentialsSecretRereadIntervalSeconds = weblogicCredentialsSecretRereadIntervalSeconds;
     }
 
     @Override
@@ -88,6 +92,7 @@ public String toString() {
           .append("unchangedCountToDelayStatusRecheck", unchangedCountToDelayStatusRecheck)
           .append("initialShortDelay", initialShortDelay)
           .append("eventualLongDelay", eventualLongDelay)
+          .append("weblogicCredentialsSecretRereadIntervalSeconds", weblogicCredentialsSecretRereadIntervalSeconds)
           .toString();
     }
 
@@ -102,6 +107,7 @@ public int hashCode() {
           .append(unchangedCountToDelayStatusRecheck)
           .append(initialShortDelay)
           .append(eventualLongDelay)
+          .append(weblogicCredentialsSecretRereadIntervalSeconds)
           .toHashCode();
     }
 
@@ -123,6 +129,7 @@ public boolean equals(Object o) {
           .append(unchangedCountToDelayStatusRecheck, mt.unchangedCountToDelayStatusRecheck)
           .append(initialShortDelay, mt.initialShortDelay)
           .append(eventualLongDelay, mt.eventualLongDelay)
+          .append(weblogicCredentialsSecretRereadIntervalSeconds, mt.weblogicCredentialsSecretRereadIntervalSeconds)
           .isEquals();
     }
   }

operator/src/main/java/oracle/kubernetes/operator/TuningParametersImpl.java

@@ -57,7 +57,8 @@ private void update() {
             (int) readTuningParameter("statusUpdateUnchangedCountToDelayStatusRecheck", 10),
             (int) readTuningParameter("stuckPodRecheckSeconds", 30),
             readTuningParameter("statusUpdateInitialShortDelay", 5),
-            readTuningParameter("statusUpdateEventualLongDelay", 30));
+            readTuningParameter("statusUpdateEventualLongDelay", 30),
+            (int) readTuningParameter("weblogicCredentialsSecretRereadIntervalSeconds", 120));
 
     CallBuilderTuning callBuilder =
         new CallBuilderTuning(

operator/src/main/java/oracle/kubernetes/operator/helpers/AuthorizationHeaderFactory.java → operator/src/main/java/oracle/kubernetes/operator/helpers/AuthorizationSource.java

@@ -4,27 +4,26 @@
 package oracle.kubernetes.operator.helpers;
 
 import java.util.Base64;
-import javax.annotation.Nonnull;
 
-public class AuthorizationHeaderFactory {
-  private final byte[] encodedUsername;
-  private final byte[] encodedPassword;
+public interface AuthorizationSource {
 
-  public AuthorizationHeaderFactory(@Nonnull byte[] userName, @Nonnull byte[] password) {
-    encodedUsername = encode(userName);
-    encodedPassword = encode(password);
-  }
+  byte[] getUserName();
 
-  private byte[] encode(byte[] source) {
-    return Base64.getEncoder().encode(source);
-  }
+  byte[] getPassword();
 
-  public String createBasicAuthorizationString() {
-    return "Basic " + createEncodedBasicCredentials(decode(encodedUsername), decode(encodedPassword));
+  /**
+   * Create an HTTP basic authorization header using the credentials.
+   * @return Basic authorization header
+   */
+  default String createBasicAuthorizationString() {
+    return "Basic " + createEncodedBasicCredentials(getUserName(), getPassword());
   }
 
-  private byte[] decode(byte[] source) {
-    return Base64.getDecoder().decode(source);
+  /**
+   * Notification that the credentials, when used, resulted in an authentication or authorization failure.
+   */
+  default void onFailure() {
+    // no-op
   }
 
   // Create encoded credentials from username and password.
@@ -35,10 +34,4 @@ private static String createEncodedBasicCredentials(final byte[] username, final
     System.arraycopy(password, 0, usernameAndPassword, username.length + 1, password.length);
     return Base64.getEncoder().encodeToString(usernameAndPassword);
   }
-
-  public static class SecretDataMissingException extends RuntimeException {
-
-    SecretDataMissingException() {
-    }
-  }
 }

operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java

@@ -6,7 +6,6 @@
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.StringReader;
-import java.time.OffsetDateTime;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -45,6 +44,7 @@
 import oracle.kubernetes.operator.work.NextAction;
 import oracle.kubernetes.operator.work.Packet;
 import oracle.kubernetes.operator.work.Step;
+import oracle.kubernetes.utils.SystemClock;
 import oracle.kubernetes.weblogic.domain.model.Domain;
 import org.apache.commons.lang3.builder.HashCodeBuilder;
 import org.yaml.snakeyaml.Yaml;
@@ -551,7 +551,7 @@ boolean isTopologyNotValid() {
 
     private void updatePacket() {
       ScanCache.INSTANCE.registerScan(
-            info.getNamespace(), info.getDomainUid(), new Scan(wlsDomainConfig, OffsetDateTime.now()));
+            info.getNamespace(), info.getDomainUid(), new Scan(wlsDomainConfig, SystemClock.now()));
       packet.put(ProcessingConstants.DOMAIN_TOPOLOGY, wlsDomainConfig);
 
       copyFileToPacketIfPresent(DOMAINZIP_HASH, DOMAINZIP_HASH);
@@ -867,7 +867,7 @@ private void recordTopology(Packet packet, DomainPresenceInfo info, DomainTopolo
       ScanCache.INSTANCE.registerScan(
           info.getNamespace(),
           info.getDomainUid(),
-          new Scan(domainTopology.getDomain(), OffsetDateTime.now()));
+          new Scan(domainTopology.getDomain(), SystemClock.now()));
 
       packet.put(ProcessingConstants.DOMAIN_TOPOLOGY, domainTopology.getDomain());
     }

operator/src/main/java/oracle/kubernetes/operator/helpers/DomainPresenceInfo.java

@@ -3,6 +3,7 @@
 
 package oracle.kubernetes.operator.helpers;
 
+import java.time.OffsetDateTime;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -14,6 +15,8 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.function.Predicate;
 import java.util.stream.Stream;
 import javax.annotation.Nonnull;
@@ -22,11 +25,14 @@
 import io.kubernetes.client.openapi.models.V1EnvVar;
 import io.kubernetes.client.openapi.models.V1ObjectMeta;
 import io.kubernetes.client.openapi.models.V1Pod;
+import io.kubernetes.client.openapi.models.V1Secret;
 import io.kubernetes.client.openapi.models.V1Service;
 import io.kubernetes.client.openapi.models.V1beta1PodDisruptionBudget;
+import oracle.kubernetes.operator.TuningParameters;
 import oracle.kubernetes.operator.WebLogicConstants;
 import oracle.kubernetes.operator.wlsconfig.WlsServerConfig;
 import oracle.kubernetes.operator.work.Packet;
+import oracle.kubernetes.utils.SystemClock;
 import oracle.kubernetes.weblogic.domain.model.Domain;
 import oracle.kubernetes.weblogic.domain.model.ServerSpec;
 import org.apache.commons.lang3.builder.EqualsBuilder;
@@ -55,6 +61,9 @@ public class DomainPresenceInfo {
   private final ConcurrentMap<String, ServerKubernetesObjects> servers = new ConcurrentHashMap<>();
   private final ConcurrentMap<String, V1Service> clusters = new ConcurrentHashMap<>();
   private final ConcurrentMap<String, V1beta1PodDisruptionBudget> podDisruptionBudgets = new ConcurrentHashMap<>();
+  private final ReadWriteLock webLogicCredentialsSecretLock = new ReentrantReadWriteLock();
+  private V1Secret webLogicCredentialsSecret;
+  private OffsetDateTime webLogicCredentialsSecretLastSet;
 
   private final List<String> validationWarnings = Collections.synchronizedList(new ArrayList<>());
   private EventItem lastEventItem;
@@ -454,6 +463,43 @@ boolean deleteClusterServiceFromEvent(String clusterName, V1Service event) {
         s -> !KubernetesUtils.isFirstNewer(getMetadata(s), getMetadata(event)));
   }
 
+  /**
+   * Retrieve the WebLogic credentials secret, if cached. This cached value will be automatically cleared
+   * after a configured time period.
+   * @return Cached secret value
+   */
+  public V1Secret getWebLogicCredentialsSecret() {
+    webLogicCredentialsSecretLock.readLock().lock();
+    try {
+      if (webLogicCredentialsSecretLastSet == null
+          || webLogicCredentialsSecretLastSet.isAfter(
+              SystemClock.now().minusSeconds(
+                  TuningParameters.getInstance().getMainTuning().weblogicCredentialsSecretRereadIntervalSeconds))) {
+        return webLogicCredentialsSecret;
+      }
+    } finally {
+      webLogicCredentialsSecretLock.readLock().unlock();
+    }
+
+    // time to clear
+    setWebLogicCredentialsSecret(null);
+    return null;
+  }
+
+  /**
+   * Cache the WebLogic credentials secret.
+   * @param webLogicCredentialsSecret Secret value
+   */
+  public void setWebLogicCredentialsSecret(V1Secret webLogicCredentialsSecret) {
+    webLogicCredentialsSecretLock.writeLock().lock();
+    try {
+      webLogicCredentialsSecretLastSet = (webLogicCredentialsSecret != null) ? SystemClock.now() : null;
+      this.webLogicCredentialsSecret = webLogicCredentialsSecret;
+    } finally {
+      webLogicCredentialsSecretLock.writeLock().unlock();
+    }
+  }
+
   private V1Service getNewerService(V1Service first, V1Service second) {
     return KubernetesUtils.isFirstNewer(getMetadata(first), getMetadata(second)) ? first : second;
   }

operator/src/main/java/oracle/kubernetes/operator/helpers/EventHelper.java

@@ -24,6 +24,7 @@
 import oracle.kubernetes.operator.work.NextAction;
 import oracle.kubernetes.operator.work.Packet;
 import oracle.kubernetes.operator.work.Step;
+import oracle.kubernetes.utils.SystemClock;
 import oracle.kubernetes.weblogic.domain.model.Domain;
 
 import static oracle.kubernetes.operator.DomainProcessorImpl.getEventK8SObjects;
@@ -632,7 +633,7 @@ public String getMessage(EventData eventData) {
     }
 
     OffsetDateTime getCurrentTimestamp() {
-      return OffsetDateTime.now();
+      return SystemClock.now();
     }
 
     void addLabels(V1ObjectMeta metadata, EventData eventData) {

operator/src/main/java/oracle/kubernetes/operator/helpers/LastKnownStatus.java

@@ -5,6 +5,7 @@
 
 import java.time.OffsetDateTime;
 
+import oracle.kubernetes.utils.SystemClock;
 import org.apache.commons.lang3.builder.EqualsBuilder;
 import org.apache.commons.lang3.builder.HashCodeBuilder;
 import org.apache.commons.lang3.builder.ToStringBuilder;
@@ -26,7 +27,7 @@ public LastKnownStatus(String status) {
   public LastKnownStatus(String status, int unchangedCount) {
     this.status = status;
     this.unchangedCount = unchangedCount;
-    this.time = OffsetDateTime.now();
+    this.time = SystemClock.now();
   }
 
   public String getStatus() {