Reduced execution time, added support to execute against different versions of Grafana, Prom, WDT, monitoring exporter

integration-tests/src/test/java/oracle/kubernetes/operator/BaseTest.java

@@ -59,6 +59,10 @@ public class BaseTest {
   public static boolean JENKINS;
   public static boolean SHARED_CLUSTER;
   public static String WDT_VERSION;
+  //currently certified chart versions of Prometheus and Grafana
+  public static String PROMETHEUS_CHART_VERSION;
+  public static String GRAFANA_CHART_VERSION;
+  public static String MONITORING_EXPORTER_VERSION;
   public static boolean INGRESSPERDOMAIN = true;
   protected static String appLocationInPod = "/u01/oracle/apps";
   private static String resultRoot = "";
@@ -141,7 +145,19 @@ public static void initialize(String appPropsFile) throws Exception {
     WDT_VERSION =
         System.getenv("WDT_VERSION") != null
             ? System.getenv("WDT_VERSION")
-            : appProps.getProperty("WDT_VERSION");            
+            : appProps.getProperty("WDT_VERSION");
+    PROMETHEUS_CHART_VERSION =
+            System.getenv("PROMETHEUS_CHART_VERSION") != null
+                    ? System.getenv("PROMETHEUS_CHART_VERSION")
+                    : appProps.getProperty("PROMETHEUS_CHART_VERSION");
+    GRAFANA_CHART_VERSION =
+            System.getenv("GRAFANA_CHART_VERSION") != null
+                    ? System.getenv("GRAFANA_CHART_VERSION")
+                    : appProps.getProperty("GRAFANA_CHART_VERSION");
+    MONITORING_EXPORTER_VERSION =
+            System.getenv("MONITORING_EXPORTER_VERSION") != null
+                    ? System.getenv("MONITORING_EXPORTER_VERSION")
+                    : appProps.getProperty("MONITORING_EXPORTER_VERSION");
 
     maxIterationsPod =
         new Integer(appProps.getProperty("maxIterationsPod", "" + maxIterationsPod)).intValue();

integration-tests/src/test/resources/OperatorIT.properties

@@ -13,3 +13,6 @@ weblogicImageName = container-registry.oracle.com/middleware/weblogic
 OCR_SERVER=container-registry.oracle.com
 DOMAIN_API_VERSION = weblogic.oracle/v5
 WDT_VERSION=1.2.0
+PROMETHEUS_CHART_VERSION=9.1.1
+GRAFANA_CHART_VERSION=3.8.14
+MONITORING_EXPORTER_VERSION=1.1.0

integration-tests/src/test/resources/OperatorIT.properties.v12.4

@@ -0,0 +1,17 @@
+# Copyright 2018, 2019, Oracle Corporation and/or its affiliates.  All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl.
+
+baseDir=/scratch
+#wls admin user
+username=weblogic
+password=welcome1
+maxIterationsPod=25
+waitTimePod=10
+weblogicImageTag = 12.2.1.4
+weblogicImageDevTag = 12.2.1.4-developer
+weblogicImageName = wlsldi-v2.docker.oraclecorp.com/weblogic
+OCR_SERVER=wlsldi-v2.docker.oraclecorp.com
+DOMAIN_API_VERSION = weblogic.oracle/v5
+PROMETHEUS_CHART_VERSION=9.1.1
+GRAFANA_CHART_VERSION=3.8.14
+MONITORING_EXPORTER_VERSION=1.1.0

integration-tests/src/test/resources/exporter/buildMonitoringExporter.sh

@@ -3,6 +3,8 @@
 # Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upload
 monitoringExporterDir=$1
 resourceExporterDir=$2
+monitoringExporterBranch=${3:-master}
+monitoringExporterVersion=${4:-1.1.0}
 monitoringExporterSrcDir=${monitoringExporterDir}/src
 monitoringExporterWar=${monitoringExporterDir}/apps/monitoringexporter/wls-exporter.war
 
@@ -13,16 +15,19 @@ fi
 mkdir $monitoringExporterDir
 echo "Installing monitoring exporter files to ${monitoringExporterDir}..."
 cd ${monitoringExporterDir}
-git clone  https://github.com/oracle/weblogic-monitoring-exporter.git $monitoringExporterSrcDir
+git clone  -b ${monitoringExporterBranch} https://github.com/oracle/weblogic-monitoring-exporter.git $monitoringExporterSrcDir
 
 echo "Building monitoring exporter files to ${monitoringExporterDir}..."
-cd ${monitoringExporterSrcDir}
-mvn clean install --log-file output.txt
-cd ${monitoringExporterSrcDir}/webapp
-mvn package -Dconfiguration=${resourceExporterDir}/rest_webapp.yml
+cd ${monitoringExporterDir}
+echo "Download webapp from ://github.com/oracle/weblogic-monitoring-exporter/releases/download/v${monitoringExporterVersion}/get${monitoringExporterVersion}.sh..."
+wget https://github.com/oracle/weblogic-monitoring-exporter/releases/download/v${monitoringExporterVersion}/get${monitoringExporterVersion}.sh
+bash get${monitoringExporterVersion}.sh ${resourceExporterDir}/rest_webapp.yml
+#mvn clean install --log-file output.txt
+#cd ${monitoringExporterSrcDir}/webapp
+#mvn package -Dconfiguration=${resourceExporterDir}/rest_webapp.yml
 cd ${monitoringExporterSrcDir}/config_coordinator
 docker build -t config_coordinator .
 mkdir ${monitoringExporterDir}/apps
 mkdir ${monitoringExporterDir}/apps/monitoringexporter
-cp ${monitoringExporterSrcDir}/webapp/target/wls-exporter.war ${monitoringExporterWar}
-echo "Run the script [buildMonitoringExporter.sh] ..."
+cp ${monitoringExporterDir}/wls-exporter.war ${monitoringExporterWar}
+echo "Run the script [buildMonitoringExporter.sh] ..."

integration-tests/src/test/resources/exporter/coordinator_test5.yaml

@@ -0,0 +1,41 @@
+# Copyright 2019, Oracle Corporation and/or its affiliates.  All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl.
+
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: coordinator
+  namespace: test5
+  labels:
+    app: coordinator
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: coordinator
+    spec:
+      containers:
+        - name: coordinator
+          image: config_coordinator
+          imagePullPolicy: Never
+          ports:
+            - containerPort: 8999
+      restartPolicy: Always
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: coordinator
+  name: coordinator
+  namespace: test5
+spec:
+  type: NodePort
+  ports:
+    - port: 8999
+      targetPort: 8999
+  selector:
+    app: coordinator

integration-tests/src/test/resources/exporter/createPromGrafanaMySqlCoordWebhook.sh

@@ -0,0 +1,54 @@
+#!/bin/bash -x
+# Copyright 2019, Oracle Corporation and/or its affiliates.  All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upload
+monitoringExporterDir=$1
+domainNS=test5
+resourceExporterDir=$2
+promVersionArgs=$3
+grafanaVersionArgs=$4
+monitoringExporterEndToEndDir=${monitoringExporterDir}/src/samples/kubernetes/end2end
+
+
+#create database
+kubectl apply -f ${monitoringExporterEndToEndDir}/mysql/persistence.yaml 
+kubectl apply -f ${monitoringExporterEndToEndDir}/mysql/mysql.yaml
+
+sleep 15
+
+POD_NAME=$(kubectl get pod -l app=mysql -o jsonpath="{.items[0].metadata.name}")
+kubectl exec -it $POD_NAME -- mysql -p123456 -e "CREATE DATABASE domain1;"
+kubectl exec -it $POD_NAME -- mysql -p123456 -e "CREATE USER 'wluser1' IDENTIFIED BY 'wlpwd123';"
+kubectl exec -it $POD_NAME -- mysql -p123456 -e "GRANT ALL ON domain1.* TO 'wluser1';"
+kubectl exec -it $POD_NAME -- mysql -u wluser1 -pwlpwd123 -D domain1 -e "show tables;"
+
+kubectl create ns monitoring
+
+kubectl apply -f ${monitoringExporterEndToEndDir}/prometheus/persistence.yaml
+kubectl apply -f ${monitoringExporterEndToEndDir}/prometheus/alert-persistence.yaml
+kubectl get pv -n monitoring
+kubectl get pvc -n monitoring
+
+helm repo update
+
+helm install --wait --name prometheus --namespace monitoring --values  ${resourceExporterDir}/promvalues.yaml stable/prometheus  --version ${promVersionArgs}
+
+
+POD_NAME=$(kubectl get pod -l app=prometheus -n monitoring -o jsonpath="{.items[0].metadata.name}")
+kubectl describe $POD_NAME -n monitoring
+
+
+kubectl --namespace monitoring create secret generic grafana-secret --from-literal=username=admin --from-literal=password=12345678
+
+
+kubectl apply -f ${monitoringExporterEndToEndDir}/grafana/persistence.yaml
+helm install --wait --name grafana --namespace monitoring --values  ${monitoringExporterEndToEndDir}/grafana/values.yaml stable/grafana --version ${grafanaVersionArgs}
+
+cd ${monitoringExporterEndToEndDir}
+docker build ./webhook -t webhook-log:1.0;
+kubectl create ns webhook
+kubectl apply -f ${monitoringExporterEndToEndDir}/webhook/server.yaml
+
+#create coordinator
+kubectl apply -f ${resourceExporterDir}/coordinator_${domainNS}.yaml
+
+echo "Run the script [createPromGrafanaMySqlCoordWebhook.sh] ..."

integration-tests/src/test/resources/exporter/deletePromGrafanaMySqlCoordWebhook.sh

@@ -0,0 +1,30 @@
+#!/bin/bash -x
+# Copyright 2019, Oracle Corporation and/or its affiliates.  All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upload
+monitoringExporterDir=$1
+domainNS=test5
+resourceExporterDir=$2
+monitoringExporterEndToEndDir=${monitoringExporterDir}/src/samples/kubernetes/end2end
+#delete webhook 
+kubectl delete -f ${monitoringExporterEndToEndDir}/webhook/server.yaml --ignore-not-found
+kubectl delete ns webhook
+
+#delete grafana
+helm delete --purge grafana
+kubectl -n monitoring delete secret grafana-secret --ignore-not-found
+kubectl delete -f ${monitoringExporterEndToEndDir}/grafana/persistence.yaml --ignore-not-found
+
+#delete prometheus
+helm delete --purge prometheus
+kubectl delete -f ${monitoringExporterEndToEndDir}/prometheus/persistence.yaml --ignore-not-found
+kubectl delete -f ${monitoringExporterEndToEndDir}/prometheus/alert-persistence.yaml --ignore-not-found
+
+
+#delete coordinator
+kubectl delete -f ${resourceExporterDir}/coordinator_${domainNS}.yaml
+
+#delete database
+kubectl delete -f ${monitoringExporterEndToEndDir}/mysql/mysql.yaml --ignore-not-found
+kubectl delete -f ${monitoringExporterEndToEndDir}/mysql/persistence.yaml --ignore-not-found
+
+echo "Run the script [deletePromGrafanaMySqlCoordWebhook.sh] ..."

integration-tests/src/test/resources/exporter/promvalues.yaml

@@ -0,0 +1,92 @@
+# Copyright 2019, Oracle Corporation and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl.
+
+serviceAccounts:
+  pushgateway:
+    create: false
+
+pushgateway:
+  enabled: false
+
+alertmanager:
+  enabled: true
+  persistentVolume:
+    existingClaim: pvc-alertmanager
+  service:
+    type: NodePort
+    nodePort: 32000
+  securityContext:
+    runAsNonRoot: false
+    runAsUser: 0
+
+server:
+  persistentVolume:
+    existingClaim: pvc-prometheus
+  service:
+    type: NodePort
+    nodePort: 30000
+  securityContext:
+    runAsNonRoot: false
+    runAsUser: 0
+  global:
+    evaluation_interval: 1m
+    scrape_interval: 10s
+    scrape_timeout: 5s
+
+alertmanagerFiles:
+  alertmanager.yml:
+    global:
+      resolve_timeout: 5m
+      #http_config:
+       # proxy_url: 'http://www-proxy-hqdc.us.oracle.com:80'
+
+    route:
+      group_by: ['alertname']
+      group_wait: 10s
+      group_interval: 10s
+      repeat_interval: 1h
+      receiver: 'logging-webhook'
+    receivers:
+      - name: 'logging-webhook'
+        webhook_configs:
+        - url: 'http://webhook.webhook.svc.cluster.local:8080/log'
+
+serverFiles:
+  alerts:
+    groups:
+      - name: node_rules
+        rules:
+          - alert: ClusterWarning
+            for: 15s
+            expr: sum by(weblogic_domainUID, weblogic_clusterName) (up{weblogic_domainUID=~'.+'}) == 1
+            labels:
+              severity: page
+            annotations:
+              description: 'Some WLS cluster has only one running server for more than 1 minutes.'
+              summary: 'Some wls cluster is in warning state.'
+
+extraScrapeConfigs: |
+    - job_name: 'wls-domain1'
+      kubernetes_sd_configs:
+      - role: pod
+      relabel_configs:
+      - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_pod_label_weblogic_domainUID, __meta_kubernetes_pod_label_weblogic_clusterName]
+        action: keep
+        regex: default;domain1;cluster-1
+      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
+        action: replace
+        target_label: __metrics_path__
+        regex: (.+)
+      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
+        action: replace
+        regex: ([^:]+)(?::\d+)?;(\d+)
+        replacement: $1:$2
+        target_label: __address__
+      - action: labelmap
+        regex: __meta_kubernetes_pod_label_(.+)
+      - source_labels: [__meta_kubernetes_pod_name]
+        action: replace
+        target_label: pod_name
+      basic_auth:
+        username: weblogic
+        password: welcome1

integration-tests/src/test/resources/exporter/redeployPromGrafanaLatestChart.sh

@@ -0,0 +1,14 @@
+#!/bin/bash -x
+# Copyright 2019, Oracle Corporation and/or its affiliates.  All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upload
+monitoringExporterDir=$1
+resourceExporterDir=$2
+monitoringExporterEndToEndDir=${monitoringExporterDir}/src/samples/kubernetes/end2end
+
+helm delete prometheus --purge
+helm delete grafana --purge
+
+helm install --wait --name prometheus --namespace monitoring --values  ${resourceExporterDir}/promvalues.yaml stable/prometheus
+
+helm install --wait --name grafana --namespace monitoring --values  ${monitoringExporterEndToEndDir}/grafana/values.yaml stable/grafana
+echo "Run the script [redeployPromGrafanaLatestChart.sh] ..."

integration-tests/src/test/resources/operatorexp.yaml

@@ -0,0 +1,10 @@
+# Copyright 2018, 2019, Oracle Corporation and/or its affiliates.  All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl.
+
+#any property can be provided here from kubernetes/charts/weblogic-operator/values.yaml
+releaseName: op1
+serviceAccount: weblogic-operator
+namespace: weblogic-operator1
+domainNamespaces: [ "default", "test1" , "test5" ]
+externalRestEnabled: true
+javaLoggingLevel: FINE

integration-tests/src/test/resources/webhook/scaleDnAction.sh

@@ -5,4 +5,4 @@ echo scale down  action
 MASTER=https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT
 echo Kubernetes master is $MASTER
 
-sh /var/scripts/scalingAction.sh --action=scaleDown --domain_uid=test5 --cluster_name=cluster-1 --kubernetes_master=$MASTER --wls_domain_namespace=test5 --operator_namespace=weblogic-operator5  2>&1 | tee -a scaleUpAction.log
+sh /var/scripts/scalingAction.sh --action=scaleDown --domain_uid=test5 --cluster_name=cluster-1 --kubernetes_master=$MASTER --wls_domain_namespace=test5 --operator_namespace=weblogic-operator1  2>&1 | tee -a scaleDnAction.log

integration-tests/src/test/resources/webhook/scaleUpAction.sh

@@ -5,5 +5,5 @@ echo scale up action > scaleUpAction.log
 MASTER=https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT
 echo Kubernetes master is $MASTER >> scaleUpAction.log
 
-sh /var/scripts/scalingAction.sh --action=scaleUp --domain_uid=test5 --cluster_name=cluster-1 --kubernetes_master=$MASTER --wls_domain_namespace=test5 --operator_namespace=weblogic-operator5  2>&1 | tee -a scaleUpAction.log
+sh /var/scripts/scalingAction.sh --action=scaleUp --domain_uid=test5 --cluster_name=cluster-1 --kubernetes_master=$MASTER --wls_domain_namespace=test5 --operator_namespace=weblogic-operator1  2>&1 | tee -a scaleUpAction.log