UPSTREAM: <carry>: e2e: Adjust OpenShift tests for easier management

* Adjust CPU test for openshift metrics window

The cpu scaler test previously never worked on OpenShift, and it turns
out that it's because our prometheus-adapter metrics window is set to 5
minutes (unlike the expected metrics-server default of 30s from
upstream), and that means on OpenShift it takes 5 minutes for the HPA to
start getting metrics before it can scale anything.

So the test would run, but we'd spend so long waiting for the metrics
to populate that we didn't react to the load being generated by the test
and the test would fail.

This adds a helper function that either waits for 10 minutes or until
the HPA metrics are populated, whichever happens first, before starting
the test, which makes the test function properly on OpenShift.

* Pull test container dockerfile out of CI and into keda repo

Previously we were building the test container in CI from a
dockerfile_literal, which was kind of hacky and more difficult to manage
than it being here in the keda repo.

This pulls that dockerfile out of CI and into a Dockerfile.tests which
we now just reference from CI.

* Add Makefile targets to makefile for OpenShift tests

We kind of stuffed those tests into CI quick so we had something, and
when we did we didn't heavily consider ergonomics. Now that we find
ourselves having to enable additional tests for fixes and new features,
it will be much easier in the long run if we can manage the test targets
here in the repo so we don't have to put in a separate PR to the release
repo to see if our changes work.

This adds some e2e-test-openshift* makefile targets that we can point
and whatever we need to, and once CI is updated, it can just call those
targets, whatever they happen to entail.

* Reenable CPU scaler test

Now that we figured out how the CPU test was broken, we can add it back
in to the testing since it's supported.

This adds the cpu test into the e2e-test-openshift Makefile target, so
when CI calls it, it will run with the rest of the scaler tests

Dockerfile.tests

@@ -0,0 +1,14 @@
+ # This dockerfile is intended to build something approximating the upstream test 
+ # image, but for use in OpenShift CI. Upstream keda bumps the keda-tools image version 
+ # when they feel like it, so we might just need to pay attention when we do releases. 
+ FROM ghcr.io/kedacore/keda-tools:1.20.5
+ COPY . /src
+ RUN chmod 777 -R /src
+ WORKDIR /src
+ RUN mkdir -p /go && chmod 777 -R /go
+ RUN mkdir -p /bin && chmod 777 -R /bin
+ ENV GOCACHE=/src/.cache
+ ENV USE_SUDO=false
+ ENV PATH=/src/helm:/src/:$PATH
+ ENV HELM_INSTALL_DIR=/src
+ RUN curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 && chmod 700 get_helm.sh && ./get_helm.sh

Makefile

@@ -119,6 +119,28 @@ e2e-test-clean-crds: ## Delete all scaled objects and jobs across all namespaces
 e2e-test-clean: get-cluster-context ## Delete all namespaces labeled with type=e2e
 	kubectl delete ns -l type=e2e
 
+# The OpenShift tests are split into 3 targets because when we test the CMA operator, 
+# we want to do the setup and cleanup with the operator, but still run the test suite
+# from the test image, so we need that granularity. 
+.PHONY: e2e-test-openshift-setup
+e2e-test-openshift-setup: ## Setup the tests for OpenShift
+	@echo "--- Performing Setup ---"
+	cd tests; go test -v -timeout 15m -tags e2e ./utils/setup_test.go 
+
+.PHONY: e2e-test-openshift
+e2e-test-openshift: ## Run tests for OpenShift
+	@echo "--- Running Internal Tests ---"
+	cd tests; go test -p 1 -v -timeout 60m -tags e2e $(shell cd tests; go list -tags e2e ./internals/... | grep -v internals/global_custom_ca)
+	@echo "--- Running Scaler Tests ---"
+	cd tests; go test -p 1 -v -timeout 60m -tags e2e ./scalers/cpu/... ./scalers/kafka/...  ./scalers/memory/... ./scalers/prometheus/...
+	@echo "--- Running Sequential Tests ---"
+	cd tests; go test -p 1 -v -timeout 60m -tags e2e ./sequential/...
+
+.PHONY: e2e-test-openshift-clean
+e2e-test-openshift-clean: ## Setup the tests for OpenShift
+	@echo "--- Cleaning Up ---"
+	cd tests; go test -v -timeout 60m -tags e2e ./utils/cleanup_test.go
+
 .PHONY: smoke-test
 smoke-test: ## Run e2e tests against Kubernetes cluster configured in ~/.kube/config.
 	./tests/run-smoke-tests.sh

tests/helper/helper.go

@@ -10,6 +10,7 @@ import (
 	"crypto/rsa"
 	"crypto/x509"
 	"crypto/x509/pkix"
+	"encoding/json"
 	"encoding/pem"
 	"fmt"
 	"io"
@@ -458,6 +459,27 @@ func WaitForAllPodRunningInNamespace(t *testing.T, kc *kubernetes.Clientset, nam
 	return false
 }
 
+// Waits until the Horizontal Pod Autoscaler for the scaledObject reports that it has metrics available
+// to calculate or until the number of iterations are done.
+func WaitForHPAMetricsToPopulate(t *testing.T, kc *kubernetes.Clientset, name, namespace string,
+	iterations, intervalSeconds int) bool {
+	totalWaitDuration := time.Duration(iterations) * time.Duration(intervalSeconds) * time.Second
+	startedWaiting := time.Now()
+	for i := 0; i < iterations; i++ {
+		t.Logf("Waiting up to %s for HPA to populate metrics - %s so far", totalWaitDuration, time.Now().Sub(startedWaiting).Round(time.Second))
+
+		hpa, _ := kc.AutoscalingV2().HorizontalPodAutoscalers(namespace).Get(context.Background(), "keda-hpa-cpu-test-so", metav1.GetOptions{})
+		if hpa.Status.CurrentMetrics != nil {
+			j, _ := json.MarshalIndent(hpa.Status.CurrentMetrics, "  ", "    ")
+			t.Logf("HPA has metrics after %s: %s", time.Now().Sub(startedWaiting), j)
+			return true
+		}
+
+		time.Sleep(time.Duration(intervalSeconds) * time.Second)
+	}
+	return false
+}
+
 // Waits until deployment ready replica count hits target or number of iterations are done.
 func WaitForDeploymentReplicaReadyCount(t *testing.T, kc *kubernetes.Clientset, name, namespace string,
 	target, iterations, intervalSeconds int) bool {

tests/scalers/cpu/cpu_test.go

@@ -9,6 +9,8 @@ import (
 
 	"github.com/joho/godotenv"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
 	"k8s.io/client-go/kubernetes"
 
 	. "github.com/kedacore/keda/v2/tests/helper"
@@ -204,6 +206,15 @@ func scaleOut(t *testing.T, kc *kubernetes.Clientset, data templateData) {
 	assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, 1, 60, 1),
 		"Replica count should start out as 1")
 
+	// The default metrics-server window is 30s, and that's what keda is used to, but on OpenShift we use
+	// prometheus-adapter, and we have it tuned to a window of 5m, so on OpenShift it takes 5 minutes before
+	// the HPA can even start scaling, and we fail the test unless we wait for the metrics. We'd read the
+	// window straight from the adapter-config configmap in openshift-monitoring, but then the test would be
+	// tied to a specific metrics server implementation, so we just wait up to 10 minutes for the metrics before
+	// we proceed with the test.
+	require.True(t, WaitForHPAMetricsToPopulate(t, kc, deploymentName, testNamespace, 120, 5),
+		"HPA should populate metrics within 10 minutes")
+
 	t.Log("--- testing scale out ---")
 	t.Log("--- applying job ---")