OWLS-88569 add events for rolling restart (#2364)

* add scopes to compatibility check logic, and add three roll events and log messages

Co-authored-by: Ryan Eberhard <ryan.eberhard@oracle.com>

Author: doxiao

documentation/staging/content/userguide/managing-domains/domain-events.md

@@ -27,9 +27,22 @@ The operator generates these event types in a domain namespace, which indicate t
  *  `DomainDeleted`: An existing domain has been deleted.
  *  `DomainProcessingStarting`: The operator has started to process a new domain or to update an existing domain. This event may be a result of a `DomainCreate`, `DomainChanged`, or `DomainDeleted` event, or a result of a retry after a failed attempt.
  *  `DomainProcessingFailed`: The operator has encountered a problem while it was processing the domain resource. The failure either could be a configuration error or a Kubernetes API error.
- *  `DomainProcessingRetrying`: The operator is going to retry the processing of a domain after it encountered an failure.
+ *  `DomainProcessingRetrying`: The operator is going to retry the processing of a domain after it encountered a failure.
  *  `DomainProcessingCompleted`:  The operator successfully completed the processing of a domain resource.
  *  `DomainProcessingAborted`:  The operator stopped processing a domain when the operator encountered a fatal error or a failure that persisted after the specified maximum number of retries.
+ *  `DomainRollStarting`:  The operator has detected domain resource or Model in Image model
+    updates that require it to perform a rolling restart of the domain.
+    If the domain roll is due to a change to domain resource fields
+    `image`, `imagePullPolicy`, `livenessProbe`, `readinessProbe`, `restartVersion`,
+    `domainHome`, `includeServerOutInPodLog`, or `logHome`, then
+    the event message reports the field name plus its old and new values.
+    If the domain roll is due to other domain resource changes that cause servers to be restarted
+    (see [full list of fields that cause servers to be restarted]({{< relref "/userguide/managing-domains/domain-lifecycle/startup#fields-that-cause-servers-to-be-restarted" >}})),
+    then the event message simply reports that the domain resource has changed.
+    If the domain roll is due to a Model in Image model update,
+    then the event message reports there has been a change in the WebLogic domain configuration without the details.
+ *  `DomainRollCompleted`:  The operator has successfully completed a rolling restart of a domain.
+ *  `PodCycleStarting`:  The operator has started to replace a server pod after it detects that the current pod does not conform to the current domain resource or WebLogic domain configuration.
  *  `DomainValidationError`:  A validation error or warning is found in a domain resource. Please refer to the event message for details.
  *  `NamespaceWatchingStarted`: The operator has started watching for domains in a namespace.
  *  `NamespaceWatchingStopped`: The operator has stopped watching for domains in a namespace. Note that the creation of this event in a domain namespace is the operator's best effort only; the event will not be generated if the required Kubernetes privilege is removed when a namespace is no longer managed by the operator.
@@ -285,3 +298,98 @@ Source:
 Type:    Normal
 Events:  <none>
 ```
+
+Example of the sequence of operator generated events in a domain rolling restart after the domain resource's `image` and `logHomeEnabled` changed, which is the output of the command `kubectl get events -n sample-domain1-ns --selector=weblogic.domainUID=sample-domain1,weblogic.createdByOperator=true --sort-by=lastTimestamp'.
+
+```
+LAST SEEN   TYPE     REASON                     OBJECT           MESSAGE
+2m58s       Normal   DomainChanged              domain/sample-domain1   Domain resource sample-domain1 was changed
+2m58s       Normal   DomainProcessingStarting   domain/sample-domain1   Creating or updating Kubernetes presence for WebLogic Domain with UID sample-domain1
+2m58s       Normal   DomainRollStarting         domain/sample-domain1   Rolling restart WebLogic server pods in domain sample-domain1 because: 'image' changed from 'oracle/weblogic' to 'oracle/weblogic:14.1.1.0',
+  'logHome' changed from 'null' to '/shared/logs/sample-domain1'
+2m58s       Normal   PodCycleStarting           domain/sample-domain1   Replacing pod sample-domain1-adminserver because: In container 'weblogic-server':
+  'image' changed from 'oracle/weblogic' to 'oracle/weblogic:14.1.1.0',
+  env 'LOG_HOME' changed from 'null' to '/shared/logs/sample-domain1'
+2m7s        Normal   PodCycleStarting           domain/sample-domain1   Replacing pod sample-domain1-managed-server1 because: In container 'weblogic-server':
+  'image' changed from 'oracle/weblogic' to 'oracle/weblogic:14.1.1.0',
+  env 'LOG_HOME' changed from 'null' to '/shared/logs/sample-domain1'
+71s         Normal   PodCycleStarting           domain/sample-domain1   Replacing pod sample-domain1-managed-server2 because: In container 'weblogic-server':
+  'image' changed from 'oracle/weblogic' to 'oracle/weblogic:14.1.1.0',
+  env 'LOG_HOME' changed from 'null' to '/shared/logs/sample-domain1'
+19s         Normal   DomainRollCompleted         domain/sample-domain1   Rolling restart of domain sample-domain1 completed
+19s         Normal   DomainProcessingCompleted   domain/sample-domain1   Successfully completed processing domain resource sample-domain1
+
+```
+
+Example of a `DomainRollStarting` event:
+
+```
+Name:             sample-domain1.DomainRollStarting.7d33e9b787e9c318
+Namespace:        sample-domain1-ns
+Labels:           weblogic.createdByOperator=true
+                  weblogic.domainUID=sample-domain1
+Annotations:      <none>
+API Version:      v1
+Count:            1
+Event Time:       <nil>
+First Timestamp:  2021-05-18T02:00:24Z
+Involved Object:
+  API Version:   weblogic.oracle/v8
+  Kind:          Domain
+  Name:          sample-domain1
+  Namespace:     sample-domain1-ns
+  UID:           5df7dcda-d606-4509-9a06-32f25e16e166
+Kind:            Event
+Last Timestamp:  2021-05-18T02:00:24Z
+Message:         Rolling restart WebLogic server pods in domain sample-domain1 because: 'image' changed from 'oracle/weblogic' to 'oracle/weblogic:14.1.1.0',
+  'logHome' changed from 'null' to '/shared/logs/sample-domain1'
+Metadata:
+  Creation Timestamp:  2021-05-18T02:00:24Z
+  Resource Version:   12842363
+  Self Link:          /api/v1/namespaces/sample-domain1-ns/events/sample-domain1.DomainRollStarting.7d33e9b787e9c318
+  UID:                6ec92655-9d06-43b1-8b26-c01ebccadecf
+Reason:               DomainRollStarting
+Reporting Component:  weblogic.operator
+Reporting Instance:   weblogic-operator-fc4ccc8b5-rh4v6
+Source:
+Type:    Normal
+Events:  <none>
+
+```
+
+Example of a `PodCycleStarting` event:
+
+```
+Name:             sample-domain1.PodCycleStarting.7d34bc3232231f49
+Namespace:        sample-domain1-ns
+Labels:           weblogic.createdByOperator=true
+                  weblogic.domainUID=sample-domain1
+Annotations:      <none>
+API Version:      v1
+Count:            1
+Event Time:       <nil>
+First Timestamp:  2021-05-18T02:01:18Z
+Involved Object:
+  API Version:   weblogic.oracle/v8
+  Kind:          Domain
+  Name:          sample-domain1
+  Namespace:     sample-domain1-ns
+  UID:           5df7dcda-d606-4509-9a06-32f25e16e166
+Kind:            Event
+Last Timestamp:  2021-05-18T02:01:18Z
+Message:         Replacing pod sample-domain1-managed-server1 because: In container 'weblogic-server':
+  'image' changed from 'oracle/weblogic' to 'oracle/weblogic:14.1.1.0',
+  env 'LOG_HOME' changed from 'null' to '/shared/logs/sample-domain1'
+Metadata:
+  Creation Timestamp:  2021-05-18T02:01:18Z
+  Resource Version:   12842530
+  Self Link:          /api/v1/namespaces/sample-domain1-ns/events/sample-domain1.PodCycleStarting.7d34bc3232231f49
+  UID:                4c6a203e-9b93-4b46-b9e3-1a448b52c7ca
+Reason:               PodCycleStarting
+Reporting Component:  weblogic.operator
+Reporting Instance:   weblogic-operator-fc4ccc8b5-rh4v6
+Source:
+Type:    Normal
+Events:  <none>
+
+```

operator/src/main/java/oracle/kubernetes/operator/EventConstants.java

@@ -14,7 +14,10 @@ public interface EventConstants {
   String DOMAIN_PROCESSING_FAILED_EVENT = "DomainProcessingFailed";
   String DOMAIN_PROCESSING_RETRYING_EVENT = "DomainProcessingRetrying";
   String DOMAIN_PROCESSING_ABORTED_EVENT = "DomainProcessingAborted";
+  String DOMAIN_ROLL_COMPLETED_EVENT = "DomainRollCompleted";
+  String DOMAIN_ROLL_STARTING_EVENT = "DomainRollStarting";
   String DOMAIN_VALIDATION_ERROR_EVENT = "DomainValidationError";
+  String POD_CYCLE_STARTING_EVENT = "PodCycleStarting";
   String EVENT_NORMAL = "Normal";
   String EVENT_WARNING = "Warning";
   String WEBLOGIC_OPERATOR_COMPONENT = "weblogic.operator";
@@ -34,6 +37,7 @@ public interface EventConstants {
       = "Aborting the processing of domain resource %s permanently due to: %s";
   String DOMAIN_VALIDATION_ERROR_PATTERN
       = "Validation error in domain resource %s: %s";
+  String POD_CYCLE_STARTING_PATTERN = "Replacing pod %s because: %s";
   String NAMESPACE_WATCHING_STARTED_EVENT = "NamespaceWatchingStarted";
   String NAMESPACE_WATCHING_STARTED_PATTERN = "Started watching namespace %s";
   String NAMESPACE_WATCHING_STOPPED_EVENT = "NamespaceWatchingStopped";
@@ -47,4 +51,9 @@ public interface EventConstants {
   String STOP_MANAGING_NAMESPACE_PATTERN = "Stop managing namespace %s";
   String START_MANAGING_NAMESPACE_FAILED_EVENT = "StartManagingNamespaceFailed";
   String START_MANAGING_NAMESPACE_FAILED_PATTERN = "Start managing namespace %s failed due to an authorization error";
+  String DOMAIN_ROLL_STARTING_PATTERN = "Rolling restart WebLogic server pods in domain %s because: %s";
+  String DOMAIN_ROLL_COMPLETED_PATTERN = "Rolling restart of domain %s completed";
+  String ROLL_REASON_DOMAIN_RESOURCE_CHANGED = "domain resource changed";
+  String ROLL_REASON_WEBLOGIC_CONFIGURATION_CHANGED
+      = "WebLogic domain configuration changed due to a Model in Image model update";
 }

operator/src/main/java/oracle/kubernetes/operator/ProcessingConstants.java

@@ -35,6 +35,7 @@ public interface ProcessingConstants {
   String MII_DYNAMIC_UPDATE_WDTROLLBACKFILE = "miiDynamicUpdateRollbackFile";
   String MII_DYNAMIC_UPDATE_SUCCESS = "0";
   String MII_DYNAMIC_UPDATE_RESTART_REQUIRED = "103";
+  String DOMAIN_ROLL_START_EVENT_GENERATED = "domainRollStartEventGenerated";
 
   String DOMAIN_VALIDATION_ERRORS = "domainValidationErrors";
   String INTROSPECTOR_JOB_FAILURE_LOGGED = "introspectorJobFailureLogged";

operator/src/main/java/oracle/kubernetes/operator/helpers/CollectiveCompatibility.java

@@ -41,9 +41,30 @@ public String getIncompatibility() {
         reasons.add(getIndent() + check.getIncompatibility());
       }
     }
-    return reasons.isEmpty() ? null : getHeader() + String.join("\n", reasons);
+    return reasons.isEmpty() ? null : getHeader() + String.join(",\n", reasons);
   }
 
+  @Override
+  public String getScopedIncompatibility(CompatibilityScope scope) {
+    final List<String> reasons = new ArrayList<>();
+    for (CompatibilityCheck check : checks) {
+      if (!check.isCompatible() && check.getScopedIncompatibility(scope) != null) {
+        reasons.add(getIndent() + check.getScopedIncompatibility(scope));
+      }
+    }
+    return reasons.isEmpty()
+        ? null
+        : scope == CompatibilityScope.DOMAIN
+            ? String.join(",\n", reasons)
+            : getHeader() + String.join(",\n", reasons);
+  }
+
+  @Override
+  public CompatibilityScope getScope() {
+    return CompatibilityScope.MINIMUM;
+  }
+
+
   <T> void addSets(String description, List<T> expected, List<T> actual) {
     add(CheckFactory.create(description, expected, actual));
   }

operator/src/main/java/oracle/kubernetes/operator/helpers/CompatibilityCheck.java

@@ -4,10 +4,74 @@
 package oracle.kubernetes.operator.helpers;
 
 interface CompatibilityCheck {
+  enum CompatibilityScope {
+    DOMAIN {
+      @Override
+      public boolean contains(CompatibilityScope scope) {
+        switch (scope) {
+          case DOMAIN:
+          case MINIMUM:
+            return true;
+          case POD:
+          case UNKNOWN:
+          default:
+            return false;
+        }
+      }
+    },
+    POD {
+      @Override
+      public boolean contains(CompatibilityScope scope) {
+        switch (scope) {
+          case DOMAIN:
+          case POD:
+          case UNKNOWN:
+          case MINIMUM:
+            return true;
+          default:
+            return false;
+
+        }
+      }
+    },
+    UNKNOWN {
+      @Override
+      public boolean contains(CompatibilityScope scope) {
+        switch (scope) {
+          case DOMAIN:
+          case POD:
+          default:
+            return false;
+          case UNKNOWN:
+          case MINIMUM:
+            return true;
+        }
+      }
+    },
+    MINIMUM {
+      @Override
+      public boolean contains(CompatibilityScope scope) {
+        switch (scope) {
+          case DOMAIN:
+          case POD:
+          case UNKNOWN:
+          default:
+            return false;
+        }
+      }
+    };
+
+    public abstract boolean contains(CompatibilityScope scope);
+  }
+
   boolean isCompatible();
 
   String getIncompatibility();
 
+  String getScopedIncompatibility(CompatibilityScope scope);
+
+  CompatibilityScope getScope();
+
   default CompatibilityCheck ignoring(String... keys) {
     return this;
   }