Fix for tests in failures in OLCNE environment (#2524)

* Fix for tests in failures in OLCNE environment

Author: sankarpn

integration-tests/pom.xml

@@ -304,5 +304,46 @@
                 </includes-failsafe>
             </properties>
         </profile>
+        <profile>
+            <id>wko-olcne-cert</id>
+            <properties>
+                <skipITs>false</skipITs>
+                <includes-failsafe>
+                   **/ItCoherenceTests,
+                   **/ItConfigDistributionStrategy,
+                   **/ItCrossDomainTransaction,
+                   **/ItElasticLogging,
+                   **/ItElasticLoggingFluentd,
+                   **/ItExternalRmiTunneling,
+                   **/ItInitContainers,
+                   **/ItIntrospectVersion,
+                   **/ItIstioCrossDomainTransaction,
+                   **/ItIstioDomainInImage,
+                   **/ItIstioDomainInPV,
+                   **/ItIstioMiiDomain,
+                   **/ItIstioTwoDomainsInImage,
+                   **/ItKubernetesEvents,
+                   **/ItManageNameSpace,
+                   **/ItMiiDomain,
+                   **/ItMiiDomainModelInPV,
+                   **/ItMiiDynamicUpdate,
+                   **/ItMiiMultiModel,
+                   **/ItMiiSample,
+                   **/ItMiiUpdateDomainConfig,
+                   **/ItMonitoringExporter,
+                   **/ItOperatorRestart,
+                   **/ItParameterizedDomain,
+                   **/ItPodTemplates,
+                   **/ItPodsRestart,
+                   **/ItPodsShutdownOption,
+                   **/ItProductionSecureMode,
+                   **/ItWlsSamples,
+                   **/ItServerStartPolicy,
+                   **/ItSessionMigration,
+                   **/ItSystemResOverrides,
+                   **/ItUsabilityOperatorHelmChart
+                </includes-failsafe>
+            </properties>
+        </profile>        
     </profiles>
 </project>

integration-tests/src/test/java/oracle/weblogic/kubernetes/ItElasticLogging.java

@@ -425,7 +425,7 @@ private static void createDomainCrAndVerify(String adminSecretName,
                 .model(new Model()
                     .domainType("WLS")
                     .runtimeEncryptionSecret(encryptionSecretName))
-                .introspectorJobActiveDeadlineSeconds(300L)));
+                .introspectorJobActiveDeadlineSeconds(600L)));
     setPodAntiAffinity(domain);
     // create domain using model in image
     logger.info("Create model in image domain {0} in namespace {1} using docker image {2}",

integration-tests/src/test/java/oracle/weblogic/kubernetes/ItManageNameSpace.java

@@ -55,6 +55,7 @@
 import static oracle.weblogic.kubernetes.actions.TestActions.getServiceNodePort;
 import static oracle.weblogic.kubernetes.actions.TestActions.scaleClusterWithRestApi;
 import static oracle.weblogic.kubernetes.actions.TestActions.uninstallOperator;
+import static oracle.weblogic.kubernetes.utils.CleanupUtil.deleteNamespacedArtifacts;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodReadyAndServiceExists;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.scaleAndVerifyCluster;
 import static oracle.weblogic.kubernetes.utils.DomainUtils.createDomainAndVerify;
@@ -100,6 +101,8 @@ class ItManageNameSpace {
   private static LoggingFacade logger = null;
   private static String miiImage = MII_BASIC_IMAGE_NAME + ":" + MII_BASIC_IMAGE_TAG;
 
+  List<String> namespacesToClean = new ArrayList<>();
+
   /**
    * Get namespaces for operator, domain.
    *
@@ -144,23 +147,26 @@ public void tearDownAll() {
       logger.info("Delete domain1test custom resource in namespace {0}", "test-" + domainNamespaces[0]);
       deleteDomainCustomResource(domainsUid[0] + "test", "test-" + domainNamespaces[0]);
       logger.info("Deleted Domain Custom Resource " + domainsUid[0] + "test from test-" + domainNamespaces[0]);
-  
+
       logger.info("Delete domain2test custom resource in namespace {0}", "test-" + domainNamespaces[1]);
       deleteDomainCustomResource(domainsUid[1] + "test", "test-" + domainNamespaces[1]);
       logger.info("Deleted Domain Custom Resource " + domainsUid[1] + "test from test-" + domainNamespaces[1]);
-  
+
       logger.info("Delete weblogic custom resource in namespace {0}", "weblogic" + domainNamespaces[1]);
       deleteDomainCustomResource("weblogic", "weblogic" + domainNamespaces[1]);
       logger.info("Deleted Domain Custom Resource weblogic from weblogic" + domainNamespaces[1]);
     } finally {
       deleteSecrets("default");
       deleteSecrets("atest-" + domainNamespaces[0]);
-     
       deleteNamespace("atest-" + domainNamespaces[0]);
       //delete operator
       for (HelmParams helmParam : opHelmParams) {
         uninstallOperator(helmParam);
       }
+      for (var namespace : namespacesToClean) {
+        deleteNamespacedArtifacts(namespace);
+        deleteNamespace(namespace);
+      }
     }
   }
 
@@ -200,6 +206,10 @@ void testNameSpaceManageByRegularExpression() {
     assertDoesNotThrow(() -> Kubernetes.createNamespace(manageByExp2NS));
     assertDoesNotThrow(() -> Kubernetes.createNamespace(manageByExp3NS));
 
+    namespacesToClean.add(manageByExp1NS);
+    namespacesToClean.add(manageByExp2NS);
+    namespacesToClean.add(manageByExp3NS);
+
     opHelmParams[1] = installAndVerifyOperatorCanManageDomainBySelector(managedByExpDomains,unmanagedByExpDomains,
         "RegExp","^test",
         opNamespaces[1], null);
@@ -214,6 +224,7 @@ void testNameSpaceManageByRegularExpression() {
     //upgrade operator to manage domains with Labeled namespaces
     int externalRestHttpsPort = getServiceNodePort(opNamespaces[1], "external-weblogic-operator-svc");
     assertDoesNotThrow(() -> createNamespace(manageByLabelNS));
+    namespacesToClean.add(manageByLabelNS);
     Map<String, String> labels = new HashMap<>();
     labels.put("mytest", "weblogic2");
     setLabelToNamespace(manageByLabelNS, labels);
@@ -290,6 +301,7 @@ void testNameSpaceManagedByLabelSelector() {
 
     //upgrade operator1 to replace managing domains using RegExp namespaces
     assertDoesNotThrow(() -> createNamespace(manageByExpDomainNS));
+    namespacesToClean.add(manageByExpDomainNS);
     int externalRestHttpsPort = getServiceNodePort(opNamespaces[0], "external-weblogic-operator-svc");
     //set helm params to use domainNamespaceSelectionStrategy=RegExp for namespaces names started with weblogic
     OperatorParams opParams = new OperatorParams()
@@ -332,6 +344,7 @@ void testNameSpaceWithOperatorRbacFalse() {
     String manageByLabelDomainNS = domainNamespaces[0] + "test4";
     String manageByLabelDomainUid = domainsUid[0] + "test4";
     assertDoesNotThrow(() -> createNamespace(manageByLabelDomainNS));
+    namespacesToClean.add(manageByLabelDomainNS);
     opHelmParams[2] = installAndVerifyOperator(OPERATOR_RELEASE_NAME,
         opNamespaces[3], "LabelSelector",
         "mytest4", false).getHelmParams();
@@ -521,7 +534,8 @@ private Domain createDomainResource(String domainNamespace, String domainUid) {
             .configuration(new Configuration()
                 .model(new Model()
                     .domainType(WLS_DOMAIN_TYPE)
-                    .runtimeEncryptionSecret(encryptionSecretName))));
+                    .runtimeEncryptionSecret(encryptionSecretName))
+                .introspectorJobActiveDeadlineSeconds(600L)));
     setPodAntiAffinity(domain);
     return domain;
   }

integration-tests/src/test/java/oracle/weblogic/kubernetes/ItParameterizedDomain.java

@@ -547,6 +547,9 @@ void testLivenessProbe() {
           true, "/bin/sh", "-c", destLocation + " " + serverName),
           String.format("Failed to execute script %s in pod %s namespace %s", destLocation,
               serverName, domainNamespace));
+      assertTrue(execResult.exitValue() == 0,
+          String.format("Failed to execute kill server inside pod, stderr %s stdout %s", destLocation,
+              execResult.stderr(), execResult.stdout()));
       logger.info("Command executed to kill server inside pod, exit value {0}, stdout {1}, stderr {2}",
           execResult.exitValue(), execResult.stdout(), execResult.stderr());
 
@@ -912,6 +915,7 @@ private static Domain createMiiDomainWithMultiClusters(String domainUid, String
                         .nodePort(0))))
             .clusters(clusterList)
             .configuration(new Configuration()
+                .introspectorJobActiveDeadlineSeconds(300L)
                 .model(new Model()
                     .domainType(WLS_DOMAIN_TYPE)
                     .runtimeEncryptionSecret(encryptionSecretName))));
@@ -1079,7 +1083,9 @@ private static Domain createDomainOnPvUsingWdt(String domainNamespace) {
             .addClustersItem(new Cluster() //cluster
                 .clusterName(clusterName)
                 .replicas(replicaCount)
-                .serverStartState("RUNNING")));
+                .serverStartState("RUNNING"))
+            .configuration(new Configuration()
+                .introspectorJobActiveDeadlineSeconds(300L)));
     setPodAntiAffinity(domain);
     // verify the domain custom resource is created
     createDomainAndVerify(domain, domainNamespace);

integration-tests/src/test/java/oracle/weblogic/kubernetes/ItPodsShutdownOption.java

@@ -303,6 +303,7 @@ private Domain buildDomainResource(Shutdown[] shutDownObject) {
                 .serverStartState("RUNNING")
                 )
             .configuration(new Configuration()
+                .introspectorJobActiveDeadlineSeconds(300L)
                 .model(new Model()
                     .configMap(cmName)
                     .domainType(WLS_DOMAIN_TYPE)

integration-tests/src/test/java/oracle/weblogic/kubernetes/ItStickySession.java

@@ -222,11 +222,9 @@ void testSameSessionStickinessUsingVoyager() {
     for (int i = 0; i < maxRetry; i++) {
       hostNames =
           installVoyagerIngressAndVerify(domainUid, domainNamespace, ingressName, clusterNameMsPortMap);
-
       if (hostNames != null && !hostNames.isEmpty()) {
         break;
       }
-
       try {
         // sometimes the ingress may not be ready even the condition check is ready, sleep a little bit
         Thread.sleep(1000);
@@ -300,14 +298,14 @@ void testSameSessionStickinessinOKD() {
     // create route for cluster service
     String ingressHost = createRouteForOKD(serviceName, domainNamespace);
 
-    // Since the app seems to take a bit longer to be available, 
+    // Since the app seems to take a bit longer to be available,
     // checking if the app is running by executing the curl command
-    String curlString =
-        buildCurlCommand(ingressHost, 0, SESSMIGR_APP_WAR_NAME + "/?getCounter", " -b ");
+    String curlString
+        = buildCurlCommand(ingressHost, 0, SESSMIGR_APP_WAR_NAME + "/?getCounter", " -b ");
     logger.info("Command to set HTTP request or get HTTP response {0} ", curlString);
     testUntil(
-        assertDoesNotThrow(() ->
-            () -> exec(curlString, true).stdout().contains("managed-server")),
+        assertDoesNotThrow(()
+            -> () -> exec(curlString, true).stdout().contains("managed-server")),
         logger,
         "Checking if app is available");
     // verify that two HTTP connections are sticky to the same server

integration-tests/src/test/java/oracle/weblogic/kubernetes/ItWlsSamples.java

@@ -29,9 +29,9 @@
 import static oracle.weblogic.kubernetes.TestConstants.ADMIN_PASSWORD_DEFAULT;
 import static oracle.weblogic.kubernetes.TestConstants.ADMIN_USERNAME_DEFAULT;
 import static oracle.weblogic.kubernetes.TestConstants.BASE_IMAGES_REPO_SECRET;
+import static oracle.weblogic.kubernetes.TestConstants.DOMAIN_IMAGES_REPO;
 import static oracle.weblogic.kubernetes.TestConstants.DOMAIN_VERSION;
 import static oracle.weblogic.kubernetes.TestConstants.K8S_NODEPORT_HOST;
-import static oracle.weblogic.kubernetes.TestConstants.KIND_REPO;
 import static oracle.weblogic.kubernetes.TestConstants.PV_ROOT;
 import static oracle.weblogic.kubernetes.TestConstants.WEBLOGIC_IMAGE_TO_USE_IN_SPEC;
 import static oracle.weblogic.kubernetes.actions.ActionConstants.ITTESTS_DIR;
@@ -148,9 +148,7 @@ public static void initAll(@Namespaces(5) List<String> namespaces) {
   void testSampleDomainInImage(String model) {
     String domainName = model.split(":")[1];
     String script = model.split(":")[0];
-    String imageName = (KIND_REPO != null
-            ? KIND_REPO + diiImageNameBase + "_" + script + ":" + diiImageTag
-            : diiImageNameBase + "_" + script + ":" + diiImageTag);
+    String imageName = DOMAIN_IMAGES_REPO + diiImageNameBase + "_" + script + ":" + diiImageTag;
 
     //copy the samples directory to a temporary location
     setupSample();
@@ -674,11 +672,11 @@ private void deleteDomainResourceAndVerify(String domainName, Path sampleBase) {
   }
 
   private void setupLoadBalancer(Path sampleBase, String ingressType, String additionalOptions) {
-    // run setupLoadBalancer.sh to install/uninstall ingress controller 
+    // run setupLoadBalancer.sh to install/uninstall ingress controller
     CommandParams params = new CommandParams().defaults();
     params.command("sh "
            + Paths.get(sampleBase.toString(), "setupLoadBalancer.sh").toString()
-           + " -t " + ingressType 
+           + " -t " + ingressType
            + additionalOptions);
     logger.info("Run setupLoadBalancer.sh to manage {0} ingress controller", ingressType);
     boolean result = Command.withParams(params).execute();

integration-tests/src/test/java/oracle/weblogic/kubernetes/TestConstants.java

@@ -147,7 +147,7 @@ public interface TestConstants {
       System.getProperty("java.io.tmpdir") + "/it-testsresults");
 
   public static final String LOGS_DIR = RESULTS_BASE + "/diagnostics";
-  public static final String PV_ROOT = System.getenv().getOrDefault("PV_ROOT", 
+  public static final String PV_ROOT = System.getenv().getOrDefault("PV_ROOT",
       RESULTS_BASE + "/pvroot");
   public static final String RESULTS_ROOT = RESULTS_BASE + "/workdir";
 
@@ -271,7 +271,7 @@ public interface TestConstants {
   public static final String DEFAULT_EXTERNAL_REST_IDENTITY_SECRET_NAME = "weblogic-operator-external-rest-identity";
 
   // Default ISTIO version is 1.7.3
-  public static final String ISTIO_VERSION = 
+  public static final String ISTIO_VERSION =
         Optional.ofNullable(System.getenv("ISTIO_VERSION")).orElse("1.7.3");
 
   //MySQL database constants

integration-tests/src/test/java/oracle/weblogic/kubernetes/actions/impl/LoggingExporter.java

@@ -13,6 +13,7 @@
 import java.util.concurrent.Callable;
 
 import io.kubernetes.client.custom.IntOrString;
+import io.kubernetes.client.openapi.models.V1Capabilities;
 import io.kubernetes.client.openapi.models.V1Container;
 import io.kubernetes.client.openapi.models.V1ContainerPort;
 import io.kubernetes.client.openapi.models.V1Deployment;
@@ -368,6 +369,9 @@ private static V1Deployment createElasticsearchDeploymentCr(LoggingExporterParam
                     .containers(Arrays.asList(new V1Container()
                         .name(elasticsearchName)
                         .image(elasticsearchImage)
+                        .securityContext(new V1SecurityContext()
+                            .capabilities(new V1Capabilities()
+                                .add(Arrays.asList("SYS_CHROOT"))))
                         .addPortsItem(new V1ContainerPort()
                             .containerPort(new Integer(elasticsearchHttpPort)))
                         .addPortsItem(new V1ContainerPort()

integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonMiiTestUtils.java

@@ -99,7 +99,6 @@
 import static oracle.weblogic.kubernetes.utils.PatchDomainUtils.patchDomainWithNewSecretAndVerify;
 import static oracle.weblogic.kubernetes.utils.PersistentVolumeUtils.createfixPVCOwnerContainer;
 import static oracle.weblogic.kubernetes.utils.PodUtils.checkPodDoesNotExist;
-import static oracle.weblogic.kubernetes.utils.PodUtils.checkPodExists;
 import static oracle.weblogic.kubernetes.utils.PodUtils.checkPodReady;
 import static oracle.weblogic.kubernetes.utils.PodUtils.getExternalServicePodName;
 import static oracle.weblogic.kubernetes.utils.PodUtils.getPodCreationTime;
@@ -292,7 +291,7 @@ public static Domain createDomainResource(
                 .model(new oracle.weblogic.domain.Model()
                     .domainType("WLS")
                     .runtimeEncryptionSecret(encryptionSecretName))
-                .introspectorJobActiveDeadlineSeconds(300L)));
+                .introspectorJobActiveDeadlineSeconds(600L)));
     for (String clusterName : clusterNames) {
       domain.spec()
           .addClustersItem(new oracle.weblogic.domain.Cluster()
@@ -839,7 +838,7 @@ public static boolean checkApplicationRuntime(
         expectedStatusCode);
   }
 
-  private static ExecResult readRuntimeResource(String adminSvcExtHost, String domainNamespace, 
+  private static ExecResult readRuntimeResource(String adminSvcExtHost, String domainNamespace,
       String adminServerPodName, String resourcePath, String callerName) {
     LoggingFacade logger = getLogger();
 
@@ -1153,21 +1152,18 @@ public static void verifyIntrospectorRuns(String domainUid, String domainNamespa
     LoggingFacade logger = getLogger();
     logger.info("Verifying introspector pod is created, runs and deleted");
     String introspectJobName = getIntrospectJobName(domainUid);
-    checkPodExists(introspectJobName, domainUid, domainNamespace);
+    checkPodReady(introspectJobName, domainUid, domainNamespace);
 
     String labelSelector = String.format("weblogic.domainUID in (%s)", domainUid);
     V1Pod introspectorPod = assertDoesNotThrow(() -> getPod(domainNamespace, labelSelector, introspectJobName),
         "Could not get introspector pod");
     assertTrue(introspectorPod != null && introspectorPod.getMetadata() != null,
         "introspector pod or metadata is null");
-    try {
-      String introspectorLog = getPodLog(introspectorPod.getMetadata().getName(), domainNamespace);
-      logger.info("Introspector pod log START");
-      logger.info(introspectorLog);
-      logger.info("Introspector pod log END");
-    } catch (Exception ex) {
-      logger.info("Failed to get introspector pod log", ex);
-    }
+    String introspectorLog = assertDoesNotThrow(() -> getPodLog(introspectorPod.getMetadata().getName(),
+        domainNamespace), "Could not get introspector pod log");
+    logger.info("Introspector pod log START");
+    logger.info(introspectorLog);
+    logger.info("Introspector pod log END");
     checkPodDoesNotExist(introspectJobName, domainUid, domainNamespace);
   }