tests: enable external-prometheus-deep with cleanup logic (#5779)

This PR enables the temporarily disabled external-prometheus-deep
integration test. This also fixes the clean up issue by essentially
moving the external-prometheus resources into `external-prometheus`
which has the relevant annotation required to be deleted by
`bin/test-cleanup`

This PR also updates the test resource label to be `test.linkerd.io/is-test-data-plane` from
`linkerd.io/is-test-data-plane` to prevent `linkerd inject` from removing it.

Signed-off-by: Tarun Pothulapati <tarunpothulapati@outlook.com>

Author: Pothulapati

bin/_test-helpers.sh

@@ -6,7 +6,7 @@ set +e
 
 ##### Test setup helpers #####
 
-export default_test_names=(deep external-issuer helm-deep helm-upgrade uninstall upgrade-edge upgrade-stable)
+export default_test_names=(deep external-issuer external-prometheus-deep helm-deep helm-upgrade uninstall upgrade-edge upgrade-stable)
 export all_test_names=(cluster-domain cni-calico-deep multicluster "${default_test_names[*]}")
 
 tests_usage() {
@@ -374,7 +374,7 @@ install_version() {
 
     #Now we need to install the app that will be used to verify that upgrade does not break anything
     kubectl --context="$context" create namespace "$test_app_namespace" > /dev/null 2>&1
-    kubectl --context="$context" label namespaces "$test_app_namespace" 'linkerd.io/is-test-data-plane'='true' > /dev/null 2>&1
+    kubectl --context="$context" label namespaces "$test_app_namespace" 'test.linkerd.io/is-test-data-plane'='true' > /dev/null 2>&1
     (
         set -x
         "$linkerd_path" inject "$test_directory/testdata/upgrade_test.yaml" | kubectl --context="$context" apply --namespace="$test_app_namespace" -f - 2>&1

bin/test-cleanup

@@ -19,8 +19,11 @@ echo "cleaning up multicluster resources, if present [${k8s_context}]"
 echo "cleaning up jaeger extension resources, if present [${k8s_context}]"
 "$linkerd_path" jaeger uninstall 2> /dev/null | kubectl --context="$k8s_context" delete -f -
 
-echo "cleaning up the all namespaces labelled with linkerd.io/is-test-data-plane"
-kubectl --context="$k8s_context" delete ns -l linkerd.io/is-test-data-plane
+echo "cleaning up the all namespaces labelled with test.linkerd.io/is-test-data-plane"
+kubectl --context="$k8s_context" delete ns -l test.linkerd.io/is-test-data-plane
+
+echo "cleaning up cluster-scoped resources labelled with test.linkerd.io/is-test-data-plane"
+kubectl --context="$k8s_context" delete clusterRole,clusterRoleBindings -l test.linkerd.io/is-test-data-plane
 
 echo "cleaning up linkerd resources [${k8s_context}]"
 "$linkerd_path" uninstall 2> /dev/null | kubectl --context="$k8s_context" delete -f -

test/integration/edges/testdata/external_prometheus/direct_edges.golden

@@ -1,19 +1,19 @@
 \[
   \{
     "src": "prometheus",
-    "src_namespace": "default",
+    "src_namespace": "external\-prometheus",
     "dst": "slow-cooker",
     "dst_namespace": "{{.Ns}}",
-    "client_id": "prometheus.default",
+    "client_id": "prometheus.external\-prometheus",
     "server_id": "default.{{.Ns}}",
     "no_tls_reason": ""
   \},
   \{
     "src": "prometheus",
-    "src_namespace": "default",
+    "src_namespace": "external\-prometheus",
     "dst": "terminus",
     "dst_namespace": "{{.Ns}}",
-    "client_id": "prometheus.default",
+    "client_id": "prometheus.external\-prometheus",
     "server_id": "default.{{.Ns}}",
     "no_tls_reason": ""
   \},

test/integration/endpoints/endpoints_test.go

@@ -38,7 +38,7 @@ func TestGoodEndpoints(t *testing.T) {
 	if !TestHelper.ExternalPrometheus() {
 		cmd = append(cmd, fmt.Sprintf("linkerd-prometheus.%s.svc.cluster.local:9090", vizNs))
 	} else {
-		cmd = append(cmd, "prometheus.default.svc.cluster.local:9090")
+		cmd = append(cmd, "prometheus.external-prometheus.svc.cluster.local:9090")
 		testDataPath += "/external_prometheus"
 	}
 

test/integration/endpoints/testdata/external_prometheus/linkerd_endpoints.golden

@@ -1,10 +1,10 @@
 \[
   \{
-    "namespace": "default",
+    "namespace": "external\-prometheus",
     "ip": "\d+\.\d+\.\d+\.\d+",
     "port": 9090,
     "pod": "prometheus\-[a-f0-9]+\-[a-z0-9]+",
-    "service": "prometheus.default"
+    "service": "prometheus.external\-prometheus"
   \},
   \{
     "namespace": "{{.Ns}}",

test/integration/install_test.go

@@ -429,7 +429,7 @@ func TestInstallOrUpgradeCli(t *testing.T) {
 		}
 
 		// Update args to use external proemtheus
-		vizArgs = append(vizArgs, "--set", "prometheusUrl=http://prometheus.default.svc.cluster.local:9090", "--set", "prometheus.enabled=false")
+		vizArgs = append(vizArgs, "--set", "prometheusUrl=http://prometheus.external-prometheus.svc.cluster.local:9090", "--set", "prometheus.enabled=false")
 	}
 
 	// Install Linkerd Viz Extension

test/integration/stat/stat_test.go

@@ -49,7 +49,7 @@ func TestCliStatForLinkerdNamespace(t *testing.T) {
 
 	// Retrieve Prometheus pod details
 	if TestHelper.ExternalPrometheus() {
-		prometheusNamespace = "default"
+		prometheusNamespace = "external-prometheus"
 		prometheusDeployment = "prometheus"
 	} else {
 		prometheusNamespace = TestHelper.GetVizNamespace()

test/integration/testdata/external_prometheus.yaml

@@ -1,7 +1,16 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: external-prometheus
+  labels:
+    test.linkerd.io/is-test-data-plane: "true"
+---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: prometheus
+  labels:
+    test.linkerd.io/is-test-data-plane: "true"
 rules:
 - apiGroups: [""]
   resources: ["nodes", "nodes/proxy", "pods"]
@@ -11,26 +20,28 @@ kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: prometheus
+  labels:
+    test.linkerd.io/is-test-data-plane: "true"
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
   name: prometheus
 subjects:
 - kind: ServiceAccount
   name: prometheus
-  namespace: default
+  namespace: external-prometheus
 ---
 kind: ServiceAccount
 apiVersion: v1
 metadata:
   name: prometheus
-  namespace: default
+  namespace: external-prometheus
 ---
 kind: ConfigMap
 apiVersion: v1
 metadata:
   name: prometheus-config
-  namespace: default
+  namespace: external-prometheus
 data:
   prometheus.yml: |-
     global:
@@ -168,7 +179,7 @@ kind: Service
 apiVersion: v1
 metadata:
   name: prometheus
-  namespace: default
+  namespace: external-prometheus
 spec:
   type: ClusterIP
   selector:
@@ -182,7 +193,7 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: prometheus
-  namespace: default
+  namespace: external-prometheus
 spec:
   replicas: 1
   selector:

testutil/kubernetes_helper.go

@@ -113,9 +113,9 @@ func (h *KubernetesHelper) CreateControlPlaneNamespaceIfNotExists(ctx context.Co
 }
 
 // CreateDataPlaneNamespaceIfNotExists creates a dataplane namespace if it does not already exist,
-// with a linkerd.io/is-test-data-plane label for easier cleanup afterwards
+// with a test.linkerd.io/is-test-data-plane label for easier cleanup afterwards
 func (h *KubernetesHelper) CreateDataPlaneNamespaceIfNotExists(ctx context.Context, namespace string, annotations map[string]string) error {
-	return h.createNamespaceIfNotExists(ctx, namespace, annotations, map[string]string{"linkerd.io/is-test-data-plane": "true"})
+	return h.createNamespaceIfNotExists(ctx, namespace, annotations, map[string]string{"test.linkerd.io/is-test-data-plane": "true"})
 }
 
 // KubectlApply applies a given configuration string in a namespace. If the