From fb9d70bc2e2af5fbfb868e3e1a6413ae8f7aac08 Mon Sep 17 00:00:00 2001 From: Ivan Santos <301291+pragmaticivan@users.noreply.github.com> Date: Thu, 4 Aug 2022 12:24:10 -0500 Subject: [PATCH] test: migrate new relic tests to go Signed-off-by: Ivan Santos <301291+pragmaticivan@users.noreply.github.com> --- tests/.env | 4 + tests/scalers/new-relic.test.ts | 285 ------------------ tests/scalers_go/new_relic/new_relic_test.go | 297 +++++++++++++++++++ 3 files changed, 301 insertions(+), 285 deletions(-) delete mode 100644 tests/scalers/new-relic.test.ts create mode 100644 tests/scalers_go/new_relic/new_relic_test.go diff --git a/tests/.env b/tests/.env index 56a41c243d3..23c6f3385e3 100644 --- a/tests/.env +++ b/tests/.env @@ -31,3 +31,7 @@ OPENSTACK_PASSWORD= OPENSTACK_PROJECT_ID= OPENSTACK_USER_ID= PREDICTKUBE_API_KEY= +NEWRELIC_REGION= +NEWRELIC_ACCOUNT_ID= +NEWRELIC_LICENSE= +NEWRELIC_API_KEY= diff --git a/tests/scalers/new-relic.test.ts b/tests/scalers/new-relic.test.ts deleted file mode 100644 index 9008a34352a..00000000000 --- a/tests/scalers/new-relic.test.ts +++ /dev/null @@ -1,285 +0,0 @@ -/* -To use this test you will need: -* NewRelic License Key -* NewRelic API Key - -You can get a free account on https://www.newrelic.com/ - -once you have your license and api key you need to setup the following -environment variables - -NEWRELIC_API_KEY -NEWRELIC_LICENSE -NEWRELIC_ACCOUNT_ID - -the API key starts with 'NRAK' and the license ends in 'NRAL' - - */ -import * as fs from 'fs' -import * as sh from 'shelljs' -import * as tmp from 'tmp' -import test from 'ava' -import { createNamespace } from './helpers' - -const newRelicApiKey = process.env['NEWRELIC_API_KEY'] -const newRelicAccountId = process.env['NEWRELIC_ACCOUNT_ID'] -const testNamespace = 'new-relic-test' -const newRelicNamespace = 'new-relic' -const newRelicRepoUrl = 'https://helm-charts.newrelic.com' -const newRelicHelmRepoName = 'new-relic' -const newRelicHelmPackageName = 'nri-bundle' -const newRelicLicenseKey = process.env['NEWRELIC_LICENSE'] -const kuberneteClusterName = 'keda-new-relic' -let newRelicRegion = process.env['NEWRELIC_REGION'] - -test.before(t => { - if (!newRelicApiKey) { - t.fail('NEWRELIC_API_KEY environment variable is required for newrelic tests tests') - } - if (!newRelicLicenseKey) { - t.fail('NEWRELIC_LICENSE environment variable is required for newrelic tests tests') - } - if (!newRelicAccountId) { - t.fail('NEWRELIC_ACCOUNT_ID environment variable is required for newrelic tests tests') - } - if (!newRelicRegion) { - newRelicRegion = 'EU' - } - createNamespace(newRelicNamespace) - sh.exec(`helm repo add ${newRelicHelmRepoName} ${newRelicRepoUrl}`) - sh.exec(`helm repo update`) - let helmInstallStatus = sh.exec(`helm upgrade \ - --install --set global.cluster=${kuberneteClusterName} \ - --set prometheus.enabled=true \ - --set ksm.enabled=true \ - --set global.lowDataMode=true \ - --set global.licenseKey=${newRelicLicenseKey} \ - --timeout 600s \ - --set logging.enabled=false \ - --set ksm.enabled=true \ - --set logging.enabled=true \ - --namespace ${newRelicNamespace} \ - nri-keda ${newRelicHelmRepoName}/${newRelicHelmPackageName}`).code - sh.echo(`${helmInstallStatus}`) - t.is(0, - helmInstallStatus, - 'creating a New Relic Bundle Install should work.' - ) - - sh.config.silent = true - const tmpFile = tmp.fileSync() - fs.writeFileSync(tmpFile.name, deployYaml - .replace('{{NEWRELIC_API_KEY}}', Buffer.from(newRelicApiKey).toString('base64')) - .replace('{{NEWRELIC_ACCOUNT_ID}}', newRelicAccountId) - .replace('{{NEWRELIC_REGION}}', newRelicRegion) - ) - createNamespace(testNamespace) - sh.exec(`cp ${tmpFile.name} /tmp/paso.yaml`) - t.is( - 0, - sh.exec(`kubectl apply -f ${tmpFile.name} --namespace ${testNamespace}`).code, - 'creating a deployment should work.' - ) - for (let i = 0; i < 10; i++) { - const readyReplicaCount = sh.exec(`kubectl get deployment.apps/test-app \ - --namespace ${testNamespace} -o jsonpath="{.status.readyReplicas}"`).stdout - if (readyReplicaCount != '1') { - sh.exec('sleep 2s') - } - } -}) - -test.serial('Keda Deployment should have 0 replicas on start', t => { - const replicaCount = sh.exec( - `kubectl get deployment.apps/keda-test-app --namespace ${testNamespace} -o jsonpath="{.spec.replicas}"` - ).stdout - t.is(replicaCount, '0', 'replica count should start out as 0') -}) - -test.serial('Deployment should have 1 replicas on start', t => { - const replicaCount = sh.exec( - `kubectl get deployment.apps/test-app --namespace ${testNamespace} -o jsonpath="{.spec.replicas}"` - ).stdout - t.is(replicaCount, '1', 'replica count should start out as 0') -}) - -test.serial(`Deployment should scale to 3 (the max) with HTTP Requests exceeding in the rate then back to 0`, t => { - // generate a large number of HTTP requests (using Apache Bench) that will take some time - // so prometheus has some time to scrape it - const loadGeneratorFile = tmp.fileSync() - fs.writeFileSync(loadGeneratorFile.name, generateRequestsYaml.replace('{{NAMESPACE}}', testNamespace)) - t.is( - 0, - sh.exec(`kubectl apply -f ${loadGeneratorFile.name} --namespace ${testNamespace}`).code, - 'creating job should work.' - ) - - t.is( - '1', - sh.exec( - `kubectl get deployment.apps/test-app --namespace ${testNamespace} -o jsonpath="{.status.readyReplicas}"` - ).stdout, - 'There should be 1 replica for the test-app deployment' - ) - - // keda based deployment should start scaling up with http requests issued - let replicaCount = '0' - for (let i = 0; i < 60 && replicaCount !== '3'; i++) { - t.log(`Waited ${5 * i} seconds for new-relic-based deployments to scale up`) - const jobLogs = sh.exec(`kubectl logs -l job-name=generate-requests -n ${testNamespace}`).stdout - t.log(`Logs from the generate requests: ${jobLogs}`) - - replicaCount = sh.exec( - `kubectl get deployment.apps/keda-test-app --namespace ${testNamespace} -o jsonpath="{.spec.replicas}"` - ).stdout - if (replicaCount !== '3') { - sh.exec('sleep 10s') - } - } - - t.is('3', replicaCount, 'Replica count should be maxed at 3') - - t.is( - 0, - sh.exec(`kubectl delete -f ${loadGeneratorFile.name} --namespace ${testNamespace}`).code, - 'deleting job should work.' - ) - - for (let i = 0; i < 60 && replicaCount !== '0'; i++) { - replicaCount = sh.exec( - `kubectl get deployment.apps/keda-test-app --namespace ${testNamespace} -o jsonpath="{.spec.replicas}"` - ).stdout - if (replicaCount !== '0') { - sh.exec('sleep 10s') - } - } - - t.is('0', replicaCount, 'Replica count should be 0 after 6 minutes') - sh.exec('sleep 10s') -}) - - -test.after.always.cb('clean up newrelic resources', t => { - sh.exec(`helm delete --namespace ${newRelicNamespace} nri-keda`) - sh.exec(`helm repo rm ${newRelicHelmRepoName}`) - sh.exec(`kubectl delete namespace ${newRelicNamespace} --force`) - sh.exec(`kubectl delete namespace ${testNamespace} --force`) - t.end() -}) - -const generateRequestsYaml = `apiVersion: batch/v1 -kind: Job -metadata: - name: generate-requests -spec: - template: - spec: - containers: - - image: jordi/ab - name: test - command: ["/bin/sh"] - args: ["-c", "for i in $(seq 1 60);do echo $i;ab -c 5 -n 10000 -v 2 http://test-app/;sleep 1;done"] - restartPolicy: Never - activeDeadlineSeconds: 600 - backoffLimit: 2` - -const deployYaml = `apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: test-app - name: test-app -spec: - replicas: 1 - selector: - matchLabels: - app: test-app - template: - metadata: - labels: - app: test-app - type: keda-testing - spec: - containers: - - name: prom-test-app - image: tbickford/simple-web-app-prometheus:a13ade9 - imagePullPolicy: IfNotPresent ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: keda-test-app - name: keda-test-app -spec: - replicas: 0 - selector: - matchLabels: - app: keda-test-app - template: - metadata: - labels: - app: keda-test-app - type: keda-testing - spec: - containers: - - name: prom-test-app - image: tbickford/simple-web-app-prometheus:a13ade9 - imagePullPolicy: IfNotPresent ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: test-app - annotations: - prometheus.io/scrape: "true" - name: test-app -spec: - ports: - - name: http - port: 80 - protocol: TCP - targetPort: 8080 - selector: - type: keda-testing ---- -apiVersion: keda.sh/v1alpha1 -kind: TriggerAuthentication -metadata: - name: newrelic-trigger -spec: - secretTargetRef: - - parameter: queryKey - name: newrelic-secret - key: newRelicApiKey ---- -apiVersion: v1 -kind: Secret -metadata: - name: newrelic-secret -type: Opaque -data: - newRelicApiKey: {{NEWRELIC_API_KEY}} ---- -apiVersion: keda.sh/v1alpha1 -kind: ScaledObject -metadata: - name: new-relic-scaledobject -spec: - scaleTargetRef: - name: keda-test-app - minReplicaCount: 0 - maxReplicaCount: 3 - pollingInterval: 5 - cooldownPeriod: 10 - triggers: - - type: new-relic - metadata: - account: '{{NEWRELIC_ACCOUNT_ID}}' - region: '{{NEWRELIC_REGION}}' - threshold: '10' - nrql: SELECT average(\`http_requests_total\`) FROM Metric where serviceName='test-app' and namespaceName='new-relic-test' since 60 seconds ago - authenticationRef: - name: newrelic-trigger -` diff --git a/tests/scalers_go/new_relic/new_relic_test.go b/tests/scalers_go/new_relic/new_relic_test.go new file mode 100644 index 00000000000..691fc4f9aeb --- /dev/null +++ b/tests/scalers_go/new_relic/new_relic_test.go @@ -0,0 +1,297 @@ +//go:build e2e +// +build e2e + +package new_relic_test + +import ( + "encoding/base64" + "fmt" + "os" + "testing" + + "github.com/joho/godotenv" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/client-go/kubernetes" + + . "github.com/kedacore/keda/v2/tests/helper" +) + +// Load environment variables from .env file +var _ = godotenv.Load("../../.env") + +const ( + testName = "new-relic-test" +) + +var ( + testNamespace = fmt.Sprintf("%s-ns", testName) + deploymentName = fmt.Sprintf("%s-deployment", testName) + monitoredDeploymentName = fmt.Sprintf("%s-monitored-deployment", testName) + serviceName = fmt.Sprintf("%s-service-%v", testName, GetRandomNumber()) + scaledObjectName = fmt.Sprintf("%s-so", testName) + secretName = fmt.Sprintf("%s-secret", testName) + triggerAuthName = fmt.Sprintf("%s-ta", testName) + newRelicAPIKey = os.Getenv("NEWRELIC_API_KEY") + newRelicLicenseKey = os.Getenv("NEWRELIC_LICENSE") + newRelicRegion = os.Getenv("NEWRELIC_REGION") + newRelicAccountID = os.Getenv("NEWRELIC_ACCOUNT_ID") + kuberneteClusterName = "keda-new-relic-cluster" + newRelicHelmRepoURL = "https://helm-charts.newrelic.com" + deploymentReplicas = 1 + minReplicaCount = 0 + maxReplicaCount = 2 +) + +type templateData struct { + TestNamespace string + DeploymentName string + MonitoredDeploymentName string + ServiceName string + ScaledObjectName string + TriggerAuthName string + SecretName string + NewRelicAPIKey string + DeploymentReplicas string + NewRelicRegion string + NewRelicAccountID string + KuberneteClusterName string + MinReplicaCount string + MaxReplicaCount string +} +type templateValues map[string]string + +const ( + secretTemplate = `apiVersion: v1 +kind: Secret +metadata: + name: {{.SecretName}} + namespace: {{.TestNamespace}} +data: + newRelicApiKey: {{.NewRelicAPIKey}} +` + + triggerAuthenticationTemplate = `apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: {{.TriggerAuthName}} + namespace: {{.TestNamespace}} +spec: + secretTargetRef: + - parameter: queryKey # Required. + name: {{.SecretName}} # Required. + key: newRelicApiKey # Required. +` + + monitoredDeploymentTemplate = `apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{.MonitoredDeploymentName}} + namespace: {{.TestNamespace}} + labels: + app: {{.MonitoredDeploymentName}} +spec: + replicas: {{.DeploymentReplicas}} + selector: + matchLabels: + app: {{.MonitoredDeploymentName}} + template: + metadata: + labels: + app: {{.MonitoredDeploymentName}} + spec: + containers: + - name: prom-test-app + image: tbickford/simple-web-app-prometheus:a13ade9 + imagePullPolicy: IfNotPresent +` + + deploymentTemplate = `apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{.DeploymentName}} + namespace: {{.TestNamespace}} + labels: + app: {{.DeploymentName}} +spec: + replicas: 0 + selector: + matchLabels: + app: {{.DeploymentName}} + template: + metadata: + labels: + app: {{.DeploymentName}} + spec: + containers: + - name: prom-test-app + image: tbickford/simple-web-app-prometheus:a13ade9 + imagePullPolicy: IfNotPresent +` + + serviceTemplate = `apiVersion: v1 +kind: Service +metadata: + labels: + name: {{.ServiceName}} + namespace: {{.TestNamespace}} + annotations: + prometheus.io/scrape: "true" + name: {{.ServiceName}} +spec: + ports: + - name: http + port: 80 + protocol: TCP + targetPort: 8080 + selector: + app: {{.MonitoredDeploymentName}} + ` + + scaledObjectTemplate = `apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {{.ScaledObjectName}} + namespace: {{.TestNamespace}} + labels: + app: {{.DeploymentName}} +spec: + scaleTargetRef: + name: {{.DeploymentName}} + minReplicaCount: {{.MinReplicaCount}} + maxReplicaCount: {{.MaxReplicaCount}} + pollingInterval: 1 + cooldownPeriod: 1 + advanced: + horizontalPodAutoscalerConfig: + behavior: + scaleDown: + stabilizationWindowSeconds: 10 + triggers: + - type: new-relic + metadata: + account: "{{.NewRelicAccountID}}" + region: {{.NewRelicRegion}} + threshold: "3" + nrql: SELECT rate(sum(http_requests_total), 1 SECONDS) FROM Metric where serviceName='{{.ServiceName}}' and namespaceName='{{.TestNamespace}}' since 120 seconds ago + authenticationRef: + name: {{.TriggerAuthName}} +` + + lightLoadTemplate = `apiVersion: v1 +kind: Pod +metadata: + name: fake-light-traffic + namespace: {{.TestNamespace}} +spec: + containers: + - image: busybox + name: test + command: ["/bin/sh"] + args: ["-c", "while true; do wget -O /dev/null -o /dev/null http://{{.ServiceName}}/; sleep 0.9; done"]` + + heavyLoadTemplate = `apiVersion: v1 +kind: Pod +metadata: + name: fake-heavy-traffic + namespace: {{.TestNamespace}} +spec: + containers: + - image: busybox + name: test + command: ["/bin/sh"] + args: ["-c", "while true; do wget -O /dev/null -o /dev/null http://{{.ServiceName}}/; sleep 0.1; done"]` +) + +func TestNewRelicScaler(t *testing.T) { + // setup + t.Log("--- setting up ---") + require.NotEmpty(t, newRelicAPIKey, "NEWRELIC_API_KEY env variable is required for new relic tests") + require.NotEmpty(t, newRelicLicenseKey, "NEWRELIC_LICENSE env variable is required for new relic tests") + require.NotEmpty(t, newRelicAccountID, "NEWRELIC_ACCOUNT_ID env variable is required for new relic tests") + + if len(newRelicRegion) == 0 { + newRelicRegion = "EU" + } + + // Create kubernetes resources + kc := GetKubernetesClient(t) + data, templates := getTemplateData() + CreateKubernetesResources(t, kc, testNamespace, data, templates) + + installNewRelic(t) + + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicaCount, 60, 1), + "replica count should be %s after a minute", minReplicaCount) + + // test scaling + testActivation(t, kc, data) + testScaleUp(t, kc, data) + testScaleDown(t, kc, data) + + // cleanup + DeleteKubernetesResources(t, kc, testNamespace, data, templates) +} + +func testActivation(t *testing.T, kc *kubernetes.Clientset, data templateData) { + t.Log("--- testing activation ---") + KubectlApplyWithTemplate(t, data, "lightLoadTemplate", lightLoadTemplate) + + AssertReplicaCountNotChangeDuringTimePeriod(t, kc, deploymentName, testNamespace, minReplicaCount, 60) +} + +func testScaleUp(t *testing.T, kc *kubernetes.Clientset, data templateData) { + t.Log("--- testing scale up ---") + KubectlApplyWithTemplate(t, data, "heavyLoadTemplate", heavyLoadTemplate) + + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, maxReplicaCount, 60, 3), + "replica count should be %d after 3 minutes", maxReplicaCount) +} + +func testScaleDown(t *testing.T, kc *kubernetes.Clientset, data templateData) { + t.Log("--- testing scale down ---") + KubectlDeleteWithTemplate(t, data, "lightLoadTemplate", lightLoadTemplate) + KubectlDeleteWithTemplate(t, data, "heavyLoadTemplate", heavyLoadTemplate) + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicaCount, 60, 3), + "replica count should be %d after 3 minutes", minReplicaCount) +} + +func installNewRelic(t *testing.T) { + _, err := ExecuteCommand(fmt.Sprintf("helm repo add new-relic %s", newRelicHelmRepoURL)) + assert.NoErrorf(t, err, "cannot execute command - %s", err) + _, err = ExecuteCommand("helm repo update") + assert.NoErrorf(t, err, "cannot execute command - %s", err) + + cmd := fmt.Sprintf(`helm upgrade --install --set global.cluster=%s --set prometheus.enabled=true --set ksm.enabled=true --set global.lowDataMode=true --set global.licenseKey=%s --timeout 600s --set logging.enabled=false --set ksm.enabled=true --set logging.enabled=true --namespace %s ri-keda new-relic/nri-bundle`, + kuberneteClusterName, + newRelicLicenseKey, + testNamespace) + + _, err = ExecuteCommand(cmd) + assert.NoErrorf(t, err, "cannot execute command - %s", err) +} + +func getTemplateData() (templateData, templateValues) { + return templateData{ + TestNamespace: testNamespace, + DeploymentName: deploymentName, + MonitoredDeploymentName: monitoredDeploymentName, + ServiceName: serviceName, + TriggerAuthName: triggerAuthName, + ScaledObjectName: scaledObjectName, + SecretName: secretName, + NewRelicRegion: newRelicRegion, + KuberneteClusterName: kuberneteClusterName, + MinReplicaCount: fmt.Sprintf("%v", minReplicaCount), + MaxReplicaCount: fmt.Sprintf("%v", maxReplicaCount), + DeploymentReplicas: fmt.Sprintf("%v", deploymentReplicas), + NewRelicAccountID: newRelicAccountID, + NewRelicAPIKey: base64.StdEncoding.EncodeToString([]byte(newRelicAPIKey)), + }, templateValues{ + "secretTemplate": secretTemplate, + "triggerAuthenticationTemplate": triggerAuthenticationTemplate, + "serviceTemplate": serviceTemplate, + "monitoredDeploymentTemplate": monitoredDeploymentTemplate, + "deploymentTemplate": deploymentTemplate, + "scaledObjectTemplate": scaledObjectTemplate} +}