Skip to content

Commit 42abb47

Browse files
marinakogmarinakog
authored
Added test to monitor WLS by Istio provided Prometheus (#2369)
* Added test to monitor WLS by Istio provided Prometheus
1 parent 17af907 commit 42abb47

File tree

6 files changed

+1115
-131
lines changed

6 files changed

+1115
-131
lines changed

‎integration-tests/src/test/java/oracle/weblogic/kubernetes/ItIstioMonitoringExporter.java

Lines changed: 291 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,291 @@
1+
// Copyright (c) 2021, Oracle and/or its affiliates.
2+
// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
3+
4+
package oracle.weblogic.kubernetes;
5+
6+
import java.nio.file.Path;
7+
import java.nio.file.Paths;
8+
import java.util.ArrayList;
9+
import java.util.Collections;
10+
import java.util.HashMap;
11+
import java.util.List;
12+
import java.util.Map;
13+
14+
import oracle.weblogic.domain.Domain;
15+
import oracle.weblogic.kubernetes.annotations.IntegrationTest;
16+
import oracle.weblogic.kubernetes.annotations.Namespaces;
17+
import oracle.weblogic.kubernetes.logging.LoggingFacade;
18+
import oracle.weblogic.kubernetes.utils.ExecResult;
19+
import org.awaitility.core.ConditionFactory;
20+
import org.junit.jupiter.api.BeforeAll;
21+
import org.junit.jupiter.api.DisplayName;
22+
import org.junit.jupiter.api.Test;
23+
24+
import static java.util.concurrent.TimeUnit.MINUTES;
25+
import static java.util.concurrent.TimeUnit.SECONDS;
26+
import static oracle.weblogic.kubernetes.TestConstants.ADMIN_PASSWORD_DEFAULT;
27+
import static oracle.weblogic.kubernetes.TestConstants.ADMIN_USERNAME_DEFAULT;
28+
import static oracle.weblogic.kubernetes.TestConstants.K8S_NODEPORT_HOST;
29+
import static oracle.weblogic.kubernetes.TestConstants.OCIR_SECRET_NAME;
30+
import static oracle.weblogic.kubernetes.TestConstants.RESULTS_ROOT;
31+
import static oracle.weblogic.kubernetes.TestConstants.WEBLOGIC_SLIM;
32+
import static oracle.weblogic.kubernetes.actions.ActionConstants.ITTESTS_DIR;
33+
import static oracle.weblogic.kubernetes.actions.ActionConstants.MODEL_DIR;
34+
import static oracle.weblogic.kubernetes.actions.ActionConstants.RESOURCE_DIR;
35+
import static oracle.weblogic.kubernetes.actions.TestActions.addLabelsToNamespace;
36+
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkAppUsingHostHeader;
37+
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodReady;
38+
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkServiceExists;
39+
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createConfigMapAndVerify;
40+
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createDomainAndVerify;
41+
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createMiiImageAndVerify;
42+
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createOcirRepoSecret;
43+
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.createSecretWithUsernamePassword;
44+
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.dockerLoginAndPushImageToRegistry;
45+
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.generateFileFromTemplate;
46+
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.installAndVerifyOperator;
47+
import static oracle.weblogic.kubernetes.utils.DeployUtil.deployToClusterUsingRest;
48+
import static oracle.weblogic.kubernetes.utils.IstioUtils.createIstioDomainResource;
49+
import static oracle.weblogic.kubernetes.utils.IstioUtils.deployHttpIstioGatewayAndVirtualservice;
50+
import static oracle.weblogic.kubernetes.utils.IstioUtils.deployIstioDestinationRule;
51+
import static oracle.weblogic.kubernetes.utils.IstioUtils.deployIstioPrometheus;
52+
import static oracle.weblogic.kubernetes.utils.IstioUtils.getIstioHttpIngressPort;
53+
import static oracle.weblogic.kubernetes.utils.IstioUtils.getIstioHttpIngressPort;
54+
import static oracle.weblogic.kubernetes.utils.MonitoringUtils.checkMetricsViaPrometheus;
55+
import static oracle.weblogic.kubernetes.utils.MonitoringUtils.downloadMonitoringExporterApp;
56+
import static oracle.weblogic.kubernetes.utils.TestUtils.getNextFreePort;
57+
import static oracle.weblogic.kubernetes.utils.ThreadSafeLogger.getLogger;
58+
import static org.awaitility.Awaitility.with;
59+
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
60+
import static org.junit.jupiter.api.Assertions.assertEquals;
61+
import static org.junit.jupiter.api.Assertions.assertNotNull;
62+
import static org.junit.jupiter.api.Assertions.assertTrue;
63+
64+
@DisplayName("Test the monitoring WebLogic Domain via istio provided Prometheus")
65+
@IntegrationTest
66+
class ItIstioMonitoringExporter {
67+
68+
private static String opNamespace = null;
69+
private static String domainNamespace = null;
70+
71+
private String domainUid = "istio-mii";
72+
private String configMapName = "dynamicupdate-istio-configmap";
73+
private final String clusterName = "cluster-1"; // do not modify
74+
private final String adminServerPodName = domainUid + "-admin-server";
75+
private final String managedServerPrefix = domainUid + "-cluster-1-managed-server";
76+
private final String workManagerName = "newWM";
77+
private final int replicaCount = 2;
78+
79+
80+
// create standard, reusable retry/backoff policy
81+
private static ConditionFactory withStandardRetryPolicy = null;
82+
private static LoggingFacade logger = null;
83+
84+
/**
85+
* Install Operator.
86+
* @param namespaces list of namespaces created by the IntegrationTestWatcher
87+
*/
88+
@BeforeAll
89+
public static void initAll(@Namespaces(2) List<String> namespaces) {
90+
logger = getLogger();
91+
// create standard, reusable retry/backoff policy
92+
withStandardRetryPolicy = with().pollDelay(2, SECONDS)
93+
.and().with().pollInterval(10, SECONDS)
94+
.atMost(6, MINUTES).await();
95+
96+
// get a new unique opNamespace
97+
logger.info("Assign unique namespace for Operator");
98+
assertNotNull(namespaces.get(0), "Namespace list is null");
99+
opNamespace = namespaces.get(0);
100+
101+
logger.info("Assign unique namespace for Domain");
102+
assertNotNull(namespaces.get(1), "Namespace list is null");
103+
domainNamespace = namespaces.get(1);
104+
105+
// Label the domain/operator namespace with istio-injection=enabled
106+
Map<String, String> labelMap = new HashMap();
107+
labelMap.put("istio-injection", "enabled");
108+
assertDoesNotThrow(() -> addLabelsToNamespace(domainNamespace,labelMap));
109+
assertDoesNotThrow(() -> addLabelsToNamespace(opNamespace,labelMap));
110+
111+
// install and verify operator
112+
installAndVerifyOperator(opNamespace, domainNamespace);
113+
}
114+
115+
/**
116+
* Create a domain using model-in-image model.
117+
* Add istio configuration with default readinessPort.
118+
* Do not add any AdminService under AdminServer configuration.
119+
* Deploy istio gateways and virtual service.
120+
* Verify server pods are in ready state and services are created.
121+
* Verify login to WebLogic console is successful thru istio ingress port.
122+
* Deploy a web application thru istio http ingress port using REST api.
123+
* Access web application thru istio http ingress port using curl.
124+
* Deploy Istio provided Promethues
125+
* Verify Weblogic metrics can be processed via istio based prometheus
126+
*/
127+
@Test
128+
@DisplayName("Create istio provided prometheus and verify "
129+
+ "it can monitor Weblogic domain via weblogic exporter webapp")
130+
public void testIstioPrometheus() {
131+
assertDoesNotThrow(() -> setupIstioModelInImageDomain(), "setup for istio based domain failed");
132+
int prometheusPort = getNextFreePort(30510, 30600);
133+
assertTrue(deployIstioPrometheus(domainNamespace, domainUid,
134+
String.valueOf(prometheusPort)), "failed to install istio prometheus");
135+
136+
//verify metrics via prometheus
137+
String testappPrometheusSearchKey =
138+
"wls_servlet_invocation_total_count%7Bapp%3D%22testwebapp%22%7D%5B15s%5D";
139+
assertDoesNotThrow(() -> checkMetricsViaPrometheus(testappPrometheusSearchKey,
140+
"testwebapp", prometheusPort));
141+
}
142+
143+
/**
144+
* Create mii image with monitoring exporter webapp.
145+
*/
146+
private static String createAndVerifyMiiImageWithMonitoringExporter(String monexpAppDir, String modelFilePath) {
147+
// create image with model files
148+
logger.info("Create image with model file with monitoring exporter app and verify");
149+
150+
List<String> appList = new ArrayList();
151+
appList.add(monexpAppDir);
152+
appList.add("sessmigr-app");
153+
154+
// build the model file list
155+
final List<String> modelList = Collections.singletonList(modelFilePath);
156+
String myImage =
157+
createMiiImageAndVerify("monexp", modelList, appList);
158+
159+
// docker login and push image to docker registry if necessary
160+
dockerLoginAndPushImageToRegistry(myImage);
161+
162+
return myImage;
163+
}
164+
165+
private void setupIstioModelInImageDomain() {
166+
167+
// Create the repo secret to pull the image
168+
// this secret is used only for non-kind cluster
169+
createOcirRepoSecret(domainNamespace);
170+
171+
// create secret for admin credentials
172+
logger.info("Create secret for admin credentials");
173+
String adminSecretName = "weblogic-credentials";
174+
assertDoesNotThrow(() -> createSecretWithUsernamePassword(
175+
adminSecretName,
176+
domainNamespace,
177+
"weblogic",
178+
"welcome1"),
179+
String.format("createSecret failed for %s", adminSecretName));
180+
181+
// create encryption secret
182+
logger.info("Create encryption secret");
183+
String encryptionSecretName = "encryptionsecret";
184+
assertDoesNotThrow(() -> createSecretWithUsernamePassword(
185+
encryptionSecretName,
186+
domainNamespace,
187+
"weblogicenc",
188+
"weblogicenc"),
189+
String.format("createSecret failed for %s", encryptionSecretName));
190+
191+
// create WDT config map without any files
192+
createConfigMapAndVerify(configMapName, domainUid, domainNamespace, Collections.EMPTY_LIST);
193+
assertDoesNotThrow(() -> downloadMonitoringExporterApp(RESOURCE_DIR
194+
+ "/exporter/exporter-config.yaml", RESULTS_ROOT), "Failed to download monitoring exporter application");
195+
String miiImage = createAndVerifyMiiImageWithMonitoringExporter(RESULTS_ROOT + "/wls-exporter.war",
196+
MODEL_DIR + "/model.monexp.yaml");
197+
// create the domain object
198+
Domain domain = createIstioDomainResource(domainUid,
199+
domainNamespace,
200+
adminSecretName,
201+
OCIR_SECRET_NAME,
202+
encryptionSecretName,
203+
replicaCount,
204+
miiImage,
205+
configMapName,
206+
clusterName);
207+
208+
// create model in image domain
209+
createDomainAndVerify(domain, domainNamespace);
210+
211+
logger.info("Check admin service {0} is created in namespace {1}",
212+
adminServerPodName, domainNamespace);
213+
checkServiceExists(adminServerPodName, domainNamespace);
214+
215+
// check admin server pod is ready
216+
logger.info("Wait for admin server pod {0} to be ready in namespace {1}",
217+
adminServerPodName, domainNamespace);
218+
checkPodReady(adminServerPodName, domainUid, domainNamespace);
219+
220+
// check managed server services created
221+
for (int i = 1; i <= replicaCount; i++) {
222+
logger.info("Check managed service {0} is created in namespace {1}",
223+
managedServerPrefix + i, domainNamespace);
224+
checkServiceExists(managedServerPrefix + i, domainNamespace);
225+
}
226+
227+
// check managed server pods are ready
228+
for (int i = 1; i <= replicaCount; i++) {
229+
logger.info("Wait for managed pod {0} to be ready in namespace {1}",
230+
managedServerPrefix + i, domainNamespace);
231+
checkPodReady(managedServerPrefix + i, domainUid, domainNamespace);
232+
}
233+
234+
String clusterService = domainUid + "-cluster-" + clusterName + "." + domainNamespace + ".svc.cluster.local";
235+
236+
Map<String, String> templateMap = new HashMap();
237+
templateMap.put("NAMESPACE", domainNamespace);
238+
templateMap.put("DUID", domainUid);
239+
templateMap.put("ADMIN_SERVICE",adminServerPodName);
240+
templateMap.put("CLUSTER_SERVICE", clusterService);
241+
242+
Path srcHttpFile = Paths.get(RESOURCE_DIR, "istio", "istio-http-template.yaml");
243+
Path targetHttpFile = assertDoesNotThrow(
244+
() -> generateFileFromTemplate(srcHttpFile.toString(), "istio-http.yaml", templateMap));
245+
logger.info("Generated Http VS/Gateway file path is {0}", targetHttpFile);
246+
247+
boolean deployRes = assertDoesNotThrow(
248+
() -> deployHttpIstioGatewayAndVirtualservice(targetHttpFile));
249+
assertTrue(deployRes, "Failed to deploy Http Istio Gateway/VirtualService");
250+
251+
Path srcDrFile = Paths.get(RESOURCE_DIR, "istio", "istio-dr-template.yaml");
252+
Path targetDrFile = assertDoesNotThrow(
253+
() -> generateFileFromTemplate(srcDrFile.toString(), "istio-dr.yaml", templateMap));
254+
logger.info("Generated DestinationRule file path is {0}", targetDrFile);
255+
256+
deployRes = assertDoesNotThrow(
257+
() -> deployIstioDestinationRule(targetDrFile));
258+
assertTrue(deployRes, "Failed to deploy Istio DestinationRule");
259+
260+
int istioIngressPort = getIstioHttpIngressPort();
261+
logger.info("Istio Ingress Port is {0}", istioIngressPort);
262+
263+
// We can not verify Rest Management console thru Adminstration NodePort
264+
// in istio, as we can not enable Adminstration NodePort
265+
if (!WEBLOGIC_SLIM) {
266+
String consoleUrl = "http://" + K8S_NODEPORT_HOST + ":" + istioIngressPort + "/console/login/LoginForm.jsp";
267+
boolean checkConsole =
268+
checkAppUsingHostHeader(consoleUrl, domainNamespace + ".org");
269+
assertTrue(checkConsole, "Failed to access WebLogic console");
270+
logger.info("WebLogic console is accessible");
271+
} else {
272+
logger.info("Skipping WebLogic console in WebLogic slim image");
273+
}
274+
275+
Path archivePath = Paths.get(ITTESTS_DIR, "../operator/integration-tests/apps/testwebapp.war");
276+
ExecResult result = null;
277+
result = deployToClusterUsingRest(K8S_NODEPORT_HOST,
278+
String.valueOf(istioIngressPort),
279+
ADMIN_USERNAME_DEFAULT, ADMIN_PASSWORD_DEFAULT,
280+
clusterName, archivePath, domainNamespace + ".org", "testwebapp");
281+
assertNotNull(result, "Application deployment failed");
282+
logger.info("Application deployment returned {0}", result.toString());
283+
assertEquals("202", result.stdout(), "Deployment didn't return HTTP status code 202");
284+
285+
String url = "http://" + K8S_NODEPORT_HOST + ":" + istioIngressPort + "/testwebapp/index.jsp";
286+
logger.info("Application Access URL {0}", url);
287+
boolean checkApp = checkAppUsingHostHeader(url, domainNamespace + ".org");
288+
assertTrue(checkApp, "Failed to access WebLogic application");
289+
}
290+
291+
}

0 commit comments

Comments
 (0)