Add tests for overrideDistributionStrategy (#1768)

* Adding config distribution strategy tests

* Adding distribution stratgey test

* Adding configmap in patch string

* fix configmap name

* wip

* wip

* wip

* patch the domain after config change

* adding test with distributionstrategy as DYNAMIC

* increment the introspect version

* Added more test scenarios

* Add configuration property reader servlet

* Deploy the application right after creating domain

* Formatting

* change the attribute value

* wip

* add a restart method

* Fix formatting

* Add the patching

* Fix pod check

* Get the pod creation timestamp after restart

* move the podready after startdomain

* wip

* wip

* Adding data source related tests

* wip

* wip

* wip

* wip

* wip

* wip

* override datasource with new url and user/password

* remove open option

* add secrets in the configuration

* correct the path for secrets

* Print jdbc data source attributes

* add checks for jdbc ds params

* move verify overrides to separate method

* Refactor and cleanup code

* fix t3channelport for domain

* lookup jdbc resource mbean from runtimeservicembean

* Fix the attribute values

* fix url

* Refactor the code

* use static imports

* Fix shutdown domain

* Remove overrides and secrets when restoring domain

* Add a flag for debug printing

* Move the iintrospector check and pod state checks to separate methods

* javadoc cleanup

* Use static imports

* Fix typo

* Move the copy operations in its own try/catch block to prevent build
failure

* Remove the override verifications since patching fails with invalid values
for overridestrategy

* don't assert the patching in afterEach

* Introduced a dynamic wait until config is updated.
Introduced introspector run check during cold restart.

* Correct param index

* Addressed Bhavani's review comments

* Addressing Marina's review comments

* liveness probe script 'copy new sit cfg' fix

* minor IT test fix - Always --> IfNotPresent

* revert minor IT test fix - IfNotPresent goes back to Always, fix missing import

* Adding a new cluster as part of override

* add a test method for resource restart

* Restart datasource

* Split the patching into multiple parts

* Corrected the patch string, replicas should be a number not a string

* Correct debug params

* Correct typo

* Refactor for new cluster

* Create second data source

* Fix listServers

* Refactor the code to create new cluster through onliine WLST and bring up
cluster using introspection

* add namespace for NGINX

* Fix typo

* correct typo

* correct syntax error

* Fix string concat

* Uninstall NGINX

* Move the confgi update check after new cluster startup verification

* Add ingress controller

* Fix nginx

* fix

* Add all targets for application clusterview

* rename vars

* Separate adding the new cluster and config overrides into separate
usecases

* Add a check for new servers

* fix imports

* Revert changes to ingress creation

* javadoc fix

* move the for out of try catch

* put domainRuntime lookup inside try catch since it is only available in
admin server

* deploy app to admin server as well

* Provide a empty string to delete overridesconfigmap

* Remove everything under configuration

* move delete secrets to on_restart test

* cleanup comments

* change the secret name

* Disable the failing tests until bug
https://jira.oraclecorp.com/jira/browse/OWLS-82977
is fixed

* Checking for the default token in default service account to fix the test
failures in kind cluster runs

* Using a utlity method to increment the introspectVersion

* Added a new test scenario where configmap is recreated with new override
content and applying the override through introspection

* Test fixes made based on the bug fix
https://jira.oraclecorp.com/jira/browse/OWLS-82977

* Fix secret name

Co-authored-by: Tom Barnes <tom.barnes@oracle.com>

Author: sankarpn

new-integration-tests/src/test/java/oracle/weblogic/domain/Configuration.java

@@ -34,6 +34,13 @@ public class Configuration {
           + " it overrides the Operator's config map data.introspectorJobActiveDeadlineSeconds value.")
   private Long introspectorJobActiveDeadlineSeconds;
 
+  @ApiModelProperty(
+      "Determines how updated configuration overrides are distributed to already running WebLogic servers "
+      + "following introspection when the domainHomeSourceType is PersistentVolume or Image.  Configuration overrides "
+      + "are generated during introspection from secrets, the overrideConfigMap field, and WebLogic domain topology. "
+      + "Legal values are DYNAMIC (the default) and ON_RESTART. See also introspectVersion.")
+  private String overrideDistributionStrategy;
+
   @ApiModelProperty("Istio service mesh integration")
   private Istio istio;
 
@@ -80,6 +87,19 @@ public List<String> secrets() {
     return secrets;
   }
 
+  public Configuration overrideDistributionStrategy(String overrideDistributionStrategy) {
+    this.overrideDistributionStrategy = overrideDistributionStrategy;
+    return this;
+  }
+
+  public String getOverrideDistributionStrategy() {
+    return overrideDistributionStrategy;
+  }
+
+  public void setOverrideDistributionStrategy(String overrideDistributionStrategy) {
+    this.overrideDistributionStrategy = overrideDistributionStrategy;
+  }
+
   /**
    * Adds item to secrets list.
    * @param secretsItem Secret

new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItConfigDistributionStrategy.java

@@ -6,6 +6,7 @@
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.net.http.HttpResponse;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -98,6 +99,7 @@
 import static oracle.weblogic.kubernetes.actions.TestActions.createPersistentVolumeClaim;
 import static oracle.weblogic.kubernetes.actions.TestActions.getDomainCustomResource;
 import static oracle.weblogic.kubernetes.actions.TestActions.getJob;
+import static oracle.weblogic.kubernetes.actions.TestActions.getNextIntrospectVersion;
 import static oracle.weblogic.kubernetes.actions.TestActions.getPodLog;
 import static oracle.weblogic.kubernetes.actions.TestActions.getServiceNodePort;
 import static oracle.weblogic.kubernetes.actions.TestActions.getServicePort;
@@ -182,15 +184,6 @@ public static void initAll(@Namespaces(3) List<String> namespaces) {
     assertNotNull(namespaces.get(2), "Namespace is null");
     nginxNamespace = namespaces.get(2);
 
-
-    // build the clusterview application
-    Path distDir = BuildApplication.buildApplication(Paths.get(APP_DIR, "clusterview"), null, null,
-        "dist", introDomainNamespace);
-    assertTrue(Paths.get(distDir.toString(),
-        "clusterview.war").toFile().exists(),
-        "Application archive is not available");
-    clusterViewAppPath = Paths.get(distDir.toString(), "clusterview.war");
-
     // install operator and verify its running in ready state
     installAndVerifyOperator(opNamespace, introDomainNamespace);
 
@@ -212,6 +205,14 @@ public static void initAll(@Namespaces(3) List<String> namespaces) {
       createOCRRepoSecret(introDomainNamespace);
     }
 
+    // build the clusterview application
+    Path distDir = BuildApplication.buildApplication(Paths.get(APP_DIR, "clusterview"), null, null,
+        "dist", introDomainNamespace);
+    assertTrue(Paths.get(distDir.toString(),
+        "clusterview.war").toFile().exists(),
+        "Application archive is not available");
+    clusterViewAppPath = Paths.get(distDir.toString(), "clusterview.war");
+
   }
 
 
@@ -393,10 +394,11 @@ public void testDomainIntrospectVersionNotRolling() {
     executeWLSTScript(configScript, wlstPropertiesFile.toPath(), introDomainNamespace);
 
     // patch the domain to increase the replicas of the cluster and add introspectVersion field
+    String introspectVersion = assertDoesNotThrow(() -> getNextIntrospectVersion(domainUid, introDomainNamespace));
     String patchStr =
         "["
             + "{\"op\": \"replace\", \"path\": \"/spec/clusters/0/replicas\", \"value\": 3},"
-            + "{\"op\": \"add\", \"path\": \"/spec/introspectVersion\", \"value\": \"2\"}"
+            + "{\"op\": \"add\", \"path\": \"/spec/introspectVersion\", \"value\": \"" + introspectVersion + "\"}"
             + "]";
 
     logger.info("Updating replicas in cluster {0} using patch string: {1}", clusterName, patchStr);
@@ -486,7 +488,7 @@ public void conditionEvaluated(EvaluatedCondition condition) {
     logger.info("Deploying clusterview app {0} to cluster {1}",
         clusterViewAppPath, clusterName);
     deployUsingWlst(K8S_NODEPORT_HOST, Integer.toString(t3channelNodePort),
-        ADMIN_USERNAME_DEFAULT, ADMIN_PASSWORD_DEFAULT, clusterName, clusterViewAppPath,
+        ADMIN_USERNAME_DEFAULT, ADMIN_PASSWORD_DEFAULT, adminServerName + "," + clusterName, clusterViewAppPath,
         introDomainNamespace);
 
     //access application in managed servers through NGINX load balancer
@@ -613,6 +615,104 @@ public void testDomainIntrospectVersionRolling() {
 
   }
 
+  /**
+   * Test brings up a new cluster and verifies it can successfully start by doing the following.
+   * a. Creates new WebLogic static cluster using WLST.
+   * b. Patch the Domain Resource with cluster
+   * c. Update the introspectVersion version
+   * d. Verifies the servers in the new WebLogic cluster comes up without affecting any of the running servers on
+   * pre-existing WebLogic cluster.
+   */
+  @Order(3)
+  @Test
+  @DisplayName("Test new cluster creation on demand using WLST and introspection")
+  public void testCreateNewCluster() {
+
+    final String domainUid = "mydomain";
+    final String clusterName = "cl2";
+
+    final String adminServerName = "admin-server";
+    final String adminServerPodName = domainUid + "-" + adminServerName;
+
+    final String managedServerNameBase = "cl2-ms-";
+    String managedServerPodNamePrefix = domainUid + "-" + managedServerNameBase;
+
+    final int replicaCount = 2;
+
+    logger.info("Getting node port for default channel");
+    int adminServerT3Port = getServiceNodePort(introDomainNamespace, adminServerPodName + "-external", "t3channel");
+
+    // create a temporary WebLogic WLST property file
+    File wlstPropertiesFile = assertDoesNotThrow(() -> File.createTempFile("wlst", "properties"),
+        "Creating WLST properties file failed");
+    Properties p = new Properties();
+    p.setProperty("admin_host", K8S_NODEPORT_HOST);
+    p.setProperty("admin_port", Integer.toString(adminServerT3Port));
+    p.setProperty("admin_username", ADMIN_USERNAME_DEFAULT);
+    p.setProperty("admin_password", ADMIN_PASSWORD_DEFAULT);
+    p.setProperty("test_name", "create_cluster");
+    p.setProperty("cluster_name", clusterName);
+    p.setProperty("server_prefix", managedServerNameBase);
+    p.setProperty("server_count", "3");
+    assertDoesNotThrow(() -> p.store(new FileOutputStream(wlstPropertiesFile), "wlst properties file"),
+        "Failed to write the WLST properties to file");
+
+    // changet the admin server port to a different value to force pod restart
+    Path configScript = Paths.get(RESOURCE_DIR, "python-scripts", "introspect_version_script.py");
+    executeWLSTScript(configScript, wlstPropertiesFile.toPath(), introDomainNamespace);
+
+    String introspectVersion = assertDoesNotThrow(() -> getNextIntrospectVersion(domainUid, introDomainNamespace));
+
+    logger.info("patch the domain resource with new cluster and introspectVersion");
+    String patchStr
+        = "["
+        + "{\"op\": \"add\",\"path\": \"/spec/clusters/-\", \"value\": "
+        + "    {\"clusterName\" : \"" + clusterName + "\", \"replicas\": 2, \"serverStartState\": \"RUNNING\"}"
+        + "},"
+        + "{\"op\": \"replace\", \"path\": \"/spec/introspectVersion\", \"value\": \"" + introspectVersion + "\"}"
+        + "]";
+    logger.info("Updating domain configuration using patch string: {0}\n", patchStr);
+    V1Patch patch = new V1Patch(patchStr);
+    assertTrue(patchDomainCustomResource(domainUid, introDomainNamespace, patch, V1Patch.PATCH_FORMAT_JSON_PATCH),
+        "Failed to patch domain");
+
+    //verify the introspector pod is created and runs
+    String introspectPodName = domainUid + "-" + "introspect-domain-job";
+
+    checkPodExists(introspectPodName, domainUid, introDomainNamespace);
+    checkPodDoesNotExist(introspectPodName, domainUid, introDomainNamespace);
+
+    // verify new cluster managed server services created
+    for (int i = 1; i <= replicaCount; i++) {
+      logger.info("Checking managed server service {0} is created in namespace {1}",
+          managedServerPodNamePrefix + i, introDomainNamespace);
+      checkServiceExists(managedServerPodNamePrefix + i, introDomainNamespace);
+    }
+
+    // verify new cluster managed server pods are ready
+    for (int i = 1; i <= replicaCount; i++) {
+      logger.info("Waiting for managed server pod {0} to be ready in namespace {1}",
+          managedServerPodNamePrefix + i, introDomainNamespace);
+      checkPodReady(managedServerPodNamePrefix + i, domainUid, introDomainNamespace);
+    }
+
+    String baseUri = "http://" + K8S_NODEPORT_HOST + ":" + adminServerT3Port + "/clusterview/";
+
+    String serverListUri = "ClusterViewServlet?listServers=true";
+    HttpResponse<String> response = assertDoesNotThrow(() -> OracleHttpClient.get(baseUri + serverListUri, true));
+
+    assertEquals(200, response.statusCode(), "Status code not equals to 200");
+
+    // verify managed server pods are ready
+    for (int i = 1; i <= replicaCount; i++) {
+      logger.info("Checking {0} health", managedServerNameBase + i);
+      assertTrue(response.body().contains(managedServerNameBase + i + ":HEALTH_OK"),
+          "Didn't get " + managedServerNameBase + i + ":HEALTH_OK");
+    }
+
+  }
+
+
   /**
    * Create a WebLogic domain on a persistent volume by doing the following.
    * Create a configmap containing WLST script and property file.

new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItIntrospectVersion.java

@@ -67,8 +67,8 @@
 import static oracle.weblogic.kubernetes.actions.ActionConstants.WLS_BASE_IMAGE_NAME;
 import static oracle.weblogic.kubernetes.actions.ActionConstants.WLS_BASE_IMAGE_TAG;
 import static oracle.weblogic.kubernetes.actions.TestActions.getServiceNodePort;
-import static oracle.weblogic.kubernetes.actions.TestActions.restartDomain;
 import static oracle.weblogic.kubernetes.actions.TestActions.shutdownDomain;
+import static oracle.weblogic.kubernetes.actions.TestActions.startDomain;
 import static oracle.weblogic.kubernetes.assertions.TestAssertions.adminNodePortAccessible;
 import static oracle.weblogic.kubernetes.assertions.TestAssertions.podStateNotChanged;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodDoesNotExist;
@@ -528,19 +528,19 @@ private void restartDomain1AndVerifyNoImpactOnDomain2() {
     verifyDomain2NotChanged();
 
     // restart domain1
-    logger.info("Restarting domain1");
-    assertTrue(restartDomain(domain1Uid, domain1Namespace),
-        String.format("restart domain %s in namespace %s failed", domain1Uid, domain1Namespace));
+    logger.info("Starting domain1");
+    assertTrue(startDomain(domain1Uid, domain1Namespace),
+        String.format("start domain %s in namespace %s failed", domain1Uid, domain1Namespace));
 
     // verify domain1 is restarted
     // check domain1 admin server pod is ready, also check admin service exists in the domain1 namespace
-    logger.info("Checking admin server pod in domain1 was restarted");
+    logger.info("Checking admin server pod in domain1 was started");
     checkPodReadyAndServiceExists(domain1AdminServerPodName, domain1Uid, domain1Namespace);
     checkPodRestarted(domain1Uid, domain1Namespace, domain1AdminServerPodName,
         domainAdminPodOriginalTimestamps.get(0));
 
     // check managed server pods in domain1
-    logger.info("Checking managed server pods in domain1 were restarted");
+    logger.info("Checking managed server pods in domain1 were started");
     for (int i = 1; i <= replicasAfterScale; i++) {
       String domain1ManagedServerPodName = domain1Uid + "-" + MANAGED_SERVER_NAME_BASE + i;
       checkPodReadyAndServiceExists(domain1ManagedServerPodName, domain1Uid, domain1Namespace);
@@ -549,7 +549,7 @@ private void restartDomain1AndVerifyNoImpactOnDomain2() {
     }
 
     // verify domain2 was not changed after domain1 was restarted
-    logger.info("Verifying that domain2 was not changed after domain1 was restarted");
+    logger.info("Verifying that domain2 was not changed after domain1 was started");
     verifyDomain2NotChanged();
   }
 

new-integration-tests/src/test/java/oracle/weblogic/kubernetes/ItOperatorTwoDomains.java

@@ -138,4 +138,7 @@ public interface TestConstants {
   // istio constants
   public static final String ISTIO_VERSION = "1.5.4";
 
+  //MySQL database constants
+  public static final String MYSQL_VERSION = "5.6";
+
 }

new-integration-tests/src/test/java/oracle/weblogic/kubernetes/TestConstants.java

@@ -177,8 +177,8 @@ public static boolean shutdownDomain(String domainUid, String namespace) {
    * @param namespace name of namespace
    * @return true on success, false otherwise
    */
-  public static boolean restartDomain(String domainUid, String namespace) {
-    return Domain.restart(domainUid, namespace);
+  public static boolean startDomain(String domainUid, String namespace) {
+    return Domain.start(domainUid, namespace);
   }
 
   /**
@@ -222,6 +222,28 @@ public static boolean patchDomainResourceWithNewIntrospectVersion(
     return Domain.patchDomainResourceWithNewIntrospectVersion(domainUid, namespace);
   }
 
+  /**
+   * Get next introspectVersion for a given domain.
+   *
+   * @param domainUid domain id
+   * @param namespace namespace in which the domain resource exists
+   * @return String containing next introspectVersion
+   * @throws ApiException when getting domain resource fails
+   */
+  public static String getNextIntrospectVersion(String domainUid, String namespace) throws ApiException {
+    LoggingFacade logger = getLogger();
+    oracle.weblogic.domain.Domain domain = Domain.getDomainCustomResource(domainUid, namespace);
+    String introspectVersion = domain.getSpec().getIntrospectVersion();
+    if (null != introspectVersion) {
+      logger.info("current introspectVersion: {0}", introspectVersion);
+      introspectVersion = Integer.toString(Integer.valueOf(introspectVersion) + 1);
+      logger.info("modified introspectVersion: {0}", introspectVersion);
+    } else {
+      introspectVersion = Integer.toString(1);
+    }
+    return introspectVersion;
+  }
+
   /**
    * Scale the cluster of the domain in the specified namespace by patching the domain resource.
    *

new-integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/TestActions.java

@@ -86,7 +86,7 @@ public static boolean shutdown(String domainUid, String namespace) {
     // change the /spec/serverStartPolicy to NEVER to shut down all servers in the domain
     // create patch string to shut down the domain
     StringBuffer patchStr = new StringBuffer("[{")
-        .append("\"op\": \"add\", ")
+        .append("\"op\": \"replace\", ")
         .append("\"path\": \"/spec/serverStartPolicy\", ")
         .append("\"value\": \"NEVER\"")
         .append("}]");
@@ -100,13 +100,13 @@ public static boolean shutdown(String domainUid, String namespace) {
   }
 
   /**
-   * Restart a domain in the specified namespace.
+   * Start domain in the specified namespace.
    *
    * @param domainUid the domain to restart
    * @param namespace the namespace in which the domain exists
    * @return true if patching domain resource succeeded, false otherwise
    */
-  public static boolean restart(String domainUid, String namespace) {
+  public static boolean start(String domainUid, String namespace) {
     LoggingFacade logger = getLogger();
     // change the /spec/serverStartPolicy to IF_NEEDED to start all servers in the domain
     // create patch string to start the domain