From 9f21dc15a615fea846d1f1bbb4d2c6358e41f993 Mon Sep 17 00:00:00 2001 From: jambrosiano Date: Wed, 24 Nov 2021 20:38:58 -0800 Subject: [PATCH] CSPL:1417 Added different app source per CR (#573) (#577) (#602) --- test/c3/appframework/appframework_test.go | 881 ++++++++++++---------- test/m4/appframework/appframework_test.go | 562 +++++++------- test/s1/appframework/appframework_test.go | 338 +++++---- test/testenv/appframework_utils.go | 10 +- test/testenv/s3utils.go | 12 +- test/testenv/util.go | 4 +- test/testenv/verificationutils.go | 45 +- 7 files changed, 1017 insertions(+), 835 deletions(-) diff --git a/test/c3/appframework/appframework_test.go b/test/c3/appframework/appframework_test.go index 36b2d9f21..43ecd5ef6 100644 --- a/test/c3/appframework/appframework_test.go +++ b/test/c3/appframework/appframework_test.go @@ -56,38 +56,45 @@ var _ = Describe("c3appfw test", func() { } }) - Context("Single Site Indexer Cluster with SHC (C3) with App Framework", func() { - It("smoke, c3, appframework: can deploy a C3 SVA with App Framework enabled, install apps and upgrade them", func() { + Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { + It("smoke, c3, appframework: can deploy a C3 SVA with App Framework enabled, install apps then upgrade them", func() { /* Test Steps ################## SETUP #################### - * Upload older versions of apps (V1) to S3 - * Create 2 app sources for Monitoring Console and C3 SVA (CM and SHC Deployer) - * Prepare and deploy MC CRD with app framework and wait for pod to be ready - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ############### VERIFICATIONS ############### + * Upload V1 apps to S3 for Monitoring Console + * Create app source for Monitoring Console + * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready + * Upload V1 apps to S3 for Indexer Cluster and Search Head Cluster + * Create app sources for Cluster Manager and Deployer + * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready + ######### INITIAL VERIFICATIONS ############# + * Verify V1 apps are downloaded on Cluster Manager, Deployer and Monitoring Console * Verify bundle push is successful - * Verify apps are copied, installed on MC and also on SH and Indexers pods + * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods ############### UPGRADE APPS ################ - * Upload newer versions of apps (V2) on S3 - * Wait for MC and C3 pods to be ready - ############### VERIFICATIONS ############### + * Upload V2 apps on S3 + * Wait for Monitoring Console and C3 pods to be ready + ############ FINAL VERIFICATIONS ############ + * Verify V2 apps are downloaded on Cluster Manager, Deployer and Monitoring Console * Verify bundle push is successful - * Verify apps are copied, installed and upgraded on MC and also on SH and indexers pods + * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods */ - // Upload older versions of apps (V1) to S3 for MC + //################## SETUP #################### + // Upload V1 apps to S3 for Monitoring Console + appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1, 1) + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) s3TestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory for MC") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) + // Prepare Monitoring Console spec with its own app source appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(testenvInstance, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) - // MC AppFramework Spec mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: splcommon.Spec{ @@ -98,28 +105,30 @@ var _ = Describe("c3appfw test", func() { AppFrameworkConfig: appFrameworkSpecMC, } - // Deploy MC CRD + // Deploy Monitoring Console testenvInstance.Log.Info("Deploy Monitoring Console") mcName := deployment.GetName() mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(testenvInstance.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - // Verify MC is Ready and stays in ready state + // Verify Monitoring Console is ready and stays in ready state testenv.VerifyMonitoringConsoleReady(deployment, deployment.GetName(), mc, testenvInstance) - // Upload V1 apps to S3 for IDXC + // Upload V1 apps to S3 for Indexer Cluster + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to S3 for SHC + // Upload V1 apps to S3 for Search Head Cluster + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App framework Spec for C3 appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) @@ -128,177 +137,195 @@ var _ = Describe("c3appfw test", func() { appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(testenvInstance, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD - testenvInstance.Log.Info("Deploy Single Site Indexer Cluster with SHC") + testenvInstance.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") indexerReplicas := 3 shReplicas := 3 err = deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the CM goes to Ready phase + // Ensure Cluster Manager goes to Ready phase testenv.ClusterManagerReady(deployment, testenvInstance) - // Ensure indexers go to Ready phase + // Ensure Indexers go to Ready phase testenv.SingleSiteIndexersReady(deployment, testenvInstance) - // Ensure SHC go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(deployment, testenvInstance) // Verify RF SF is met testenv.VerifyRFSFMet(deployment, testenvInstance) - // Verify MC is ready and stays in ready state + // Verify Monitoring Console is ready and stays in ready state testenv.VerifyMonitoringConsoleReady(deployment, deployment.GetName(), mc, testenvInstance) - // Verify apps are downloaded by init-container on CM, Deployer - appVersion := "V1" + //######### INITIAL VERIFICATIONS ############# + // Verify V1 apps are downloaded on Cluster Manager and Deployer initContDownloadLocationIdxc := "/init-apps/" + appSourceNameIdxc initContDownloadLocationShc := "/init-apps/" + appSourceNameShc podNames := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName()), fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - testenvInstance.Log.Info("Verify V1 apps are downloaded by init container for apps for C3", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded on Cluster Manager", appVersion)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appFileList, initContDownloadLocationIdxc) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded on Deployer", appVersion)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())}, appFileList, initContDownloadLocationShc) - // Verify apps are downloaded by init-container on MC + // Verify V1 apps are downloaded on Monitoring Console initContDownloadLocationMCPod := "/init-apps/" + appSourceNameMC mcPodName := fmt.Sprintf(testenv.MonitoringConsolePod, mcName, 0) - testenvInstance.Log.Info("Verify V1 apps are downloaded by init container for MC", "POD", mcPodName, "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded on Monitoring Console pod %s", appVersion, mcPodName)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{mcPodName}, appFileList, initContDownloadLocationMCPod) // Verify bundle push status + testenvInstance.Log.Info(fmt.Sprintf("Verify bundle push status (%s apps)", appVersion)) testenv.VerifyClusterManagerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), indexerReplicas, "") testenv.VerifyDeployerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), shReplicas) // Saving current V1 bundle hash for future comparison clusterManagerBundleHash := testenv.GetClusterManagerBundleHash(deployment) - // Verify apps are copied to location + // Verify V1 apps are copied to location allPodNames := podNames allPodNames = append(allPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) allPodNames = append(allPodNames, testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1)...) - testenvInstance.Log.Info("Verify apps are copied to correct location based on Pod KIND for app", "version", appVersion) + // Verify V1 apps are copied on Indexers and Search Heads + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to correct location on Indexers and Search Heads", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV1, true, true) - testenvInstance.Log.Info("Verify apps are copied to correct location for MC", "version", appVersion) + // Verify V2 apps are copied on Monitoring Console + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to correct location on Monitoring Console", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), []string{mcPodName}, appListV1, true, true) - // Verify apps are not copied in /etc/apps/ on CM and on Deployer (therefore not installed on Deployer and on CM) - testenvInstance.Log.Info("Verify apps are NOT copied to /etc/apps on CM and Deployer for app", "version", appVersion, "App List", appFileList) + // Verify V1 apps are not copied in /etc/apps/ on Cluster Manager and on Deployer (therefore not installed on Deployer and on Cluster Manager) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are NOT copied to /etc/apps on Cluster Manager and Deployer (App List: %s) ", appVersion, appFileList)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), podNames, appListV1, false, false) - // Verify apps are installed on C3 - testenvInstance.Log.Info("Verify apps are installed on C3 Indexers and SHs", "version", appVersion) + // Verify V1 apps are installed on Indexers and Search Heads + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are installed on Search Heads and Indexers pods: %s", appVersion, allPodNames)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV1, true, "enabled", false, true) - // Verify apps are installed on MC - testenvInstance.Log.Info("Verify apps are installed on MC", "version", appVersion) + // Verify V1 apps are installed on Monitoring Console + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are installed on Monitoring Console", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), []string{mcPodName}, appListV1, true, "enabled", false, false) + //############### UPGRADE APPS ################ // Delete apps on S3 - testenvInstance.Log.Info("Delete apps on S3 for", "Version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) uploadedApps = nil - testenvInstance.Log.Info("Testing upgrade scenario") - // Upload newer version of apps (V2) to S3 for C3 + // Upload V2 apps to S3 for Indexer Cluster appVersion = "V2" appFileList = testenv.GetAppFileList(appListV2, 2) - testenvInstance.Log.Info("Uploading apps S3 for", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory for C3") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) + + // Upload V2 apps to S3 for Search Head Cluster + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory for C3") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload newer version of apps (V2) to S3 for MC + // Upload V2 apps to S3 for Monitoring Console + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory for MC") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Wait for the poll period for the apps to be downloaded time.Sleep(2 * time.Minute) - // Ensure that the CM goes to Ready phase + // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(deployment, testenvInstance) - // Ensure indexers go to Ready phase + // Ensure Indexers go to Ready phase testenv.SingleSiteIndexersReady(deployment, testenvInstance) - // Ensure SHC go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(deployment, testenvInstance) // Verify RF SF is met testenv.VerifyRFSFMet(deployment, testenvInstance) - // Verify MC is ready and stays in ready state + // Verify Monitoring Console is ready and stays in ready state testenv.VerifyMonitoringConsoleReady(deployment, deployment.GetName(), mc, testenvInstance) - // Verify V2 apps are downloaded by init-container - testenvInstance.Log.Info("Verify V2 apps are downloaded by init container for C3 for apps", "version", appVersion) + //############ FINAL VERIFICATIONS ############ + // Verify V2 apps are downloaded + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded on Cluster Manager", appVersion)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appFileList, initContDownloadLocationIdxc) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded on Deployer", appVersion)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())}, appFileList, initContDownloadLocationShc) - testenvInstance.Log.Info("Verify V2 apps are downloaded by init container for MC for apps", "POD", mcPodName, "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded on Monitoring Console pod %s", appVersion, mcPodName)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{mcPodName}, appFileList, initContDownloadLocationMCPod) // Verify bundle push status and compare bundle hash with previous V1 bundle hash + testenvInstance.Log.Info(fmt.Sprintf("Verify bundle push status (%s apps)", appVersion)) testenv.VerifyClusterManagerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), indexerReplicas, clusterManagerBundleHash) testenv.VerifyDeployerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), shReplicas) - // Verify apps are copied to location - testenvInstance.Log.Info("Verify V2 apps are copied to correct location for C3 pods", "version", appVersion) + // Verify V2 apps are copied to location + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to correct location on C3 pods", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV2, true, true) - testenvInstance.Log.Info("Verify V2 apps are copied to correct location for MC", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to correct location on Monitoring Console", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), []string{mcPodName}, appListV2, true, false) - // Verify apps are not copied in /etc/apps/ on CM and on Deployer (therefore not installed on Deployer and on CM) - testenvInstance.Log.Info("Verify V2 apps are NOT copied to /etc/apps on CM and Deployer", "version", appVersion) + // Verify V2 apps are not copied in /etc/apps/ on Cluster Manager and on Deployer (therefore not installed on Cluster Manager and Deployer) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are NOT copied to /etc/apps on Cluster Manager and Deployer", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), podNames, appListV2, false, false) - // Verify apps are updated on C3(cluster-wide) - testenvInstance.Log.Info("Verify V2 apps are updated on the C3 pods", "version", appVersion) + // Verify V2 apps are updated on Search Heads and Indexers + testenvInstance.Log.Info(fmt.Sprintf("Verify apps have been updated to %s on Search Heads and Indexers pods", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV2, true, "enabled", true, true) - // Verify apps are updated on MC - testenvInstance.Log.Info("Verify V2 apps are updated on MC", "version", appVersion) + // Verify V2 apps are updated on Monitoring Console + testenvInstance.Log.Info(fmt.Sprintf("Verify apps have been updated to %s on Monitoring Console", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), []string{mcPodName}, appListV2, true, "enabled", true, false) }) }) - Context("Single Site Indexer Cluster with SHC (C3) with App Framework", func() { - It("smoke, c3, appframework: can deploy a C3 SVA with App Framework enabled, install apps and downgrade them", func() { + Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { + It("smoke, c3, appframework: can deploy a C3 SVA with App Framework enabled, install apps then downgrade them", func() { /* Test Steps ################## SETUP #################### - * Upload newer versions of apps (V2) to S3 - * Create 2 App Sources for MC (Monitoring Console) and C3 SVA (CM and SHC Deployer) - * Prepare and deploy MC CRD with app framework and wait for pod to be ready - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ############### VERIFICATIONS ############### + * Upload V2 apps to S3 for Monitoring Console + * Create app source for Monitoring Console + * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready + * Upload V2 apps to S3 for Indexer Cluster and Search Head Cluster + * Create app source for Cluster Manager and Deployer + * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready + ########### INITIAL VERIFICATIONS ########### + * Verify V2 apps are downloaded on Cluster Manager, Deployer and Monitoring Console * Verify bundle push is successful - * Verify apps are copied, installed on MC and also on SH and Indexers pods + * Verify V2 apps are copied, installed on Monitoring Console and also on Search Heads and Indexers pods ############## DOWNGRADE APPS ############### - * Upload older versions of apps (V1) on S3 - * Wait for MC and C3 to be READY - ############### VERIFICATIONS ############### + * Upload V1 apps on S3 + * Wait for Monitoring Console and C3 pods to be ready + ########### FINAL VERIFICATIONS ############# + * Verify V1 apps are downloaded on Cluster Manager, Deployer and Monitoring Console * Verify bundle push is successful - * Verify apps are copied, installed and downgraded on MC and also on SH and Indexers pods + * Verify apps are copied and downgraded on Monitoring Console and on Search Heads and Indexers pods */ - // Upload V2 apps to S3 for MC + //################## SETUP #################### + // Upload V2 apps to S3 for Monitoring Console + appVersion := "V2" appFileList := testenv.GetAppFileList(appListV2, 2) + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) s3TestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory for MC") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for MC + // Create App framework Spec for Monitoring Console appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(testenvInstance, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) - // MC AppFramework Spec + // Monitoring Console AppFramework Spec mcSpec := enterpriseApi.MonitoringConsoleSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: splcommon.Spec{ @@ -309,25 +336,27 @@ var _ = Describe("c3appfw test", func() { AppFrameworkConfig: appFrameworkSpecMC, } - // Deploy MC CRD + // Deploy Monitoring Console testenvInstance.Log.Info("Deploy Monitoring Console") mcName := deployment.GetName() mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(testenvInstance.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - // Verify MC is Ready and stays in ready state + // Verify Monitoring Console is Ready and stays in ready state testenv.VerifyMonitoringConsoleReady(deployment, deployment.GetName(), mc, testenvInstance) - // Upload newer version of apps to S3 for IDXC + // Upload V2 apps to S3 for Indexer Cluster + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory for C3") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload newer version of apps to S3 for Shc + // Upload V2 apps to S3 for Search Head Cluster + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory for C3") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec for C3 @@ -339,179 +368,196 @@ var _ = Describe("c3appfw test", func() { appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(testenvInstance, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD - testenvInstance.Log.Info("Deploy Single Site Indexer Cluster with SHC") + testenvInstance.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") indexerReplicas := 3 shReplicas := 3 err = deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the CM goes to Ready phase + // Ensure Cluster Manager goes to Ready phase testenv.ClusterManagerReady(deployment, testenvInstance) - // Ensure indexers go to Ready phase + // Ensure Indexers go to Ready phase testenv.SingleSiteIndexersReady(deployment, testenvInstance) - // Ensure SHC go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(deployment, testenvInstance) // Verify RF SF is met testenv.VerifyRFSFMet(deployment, testenvInstance) - // Verify MC is ready and stays in ready state + // Verify Monitoring Console is ready and stays in ready state testenv.VerifyMonitoringConsoleReady(deployment, deployment.GetName(), mc, testenvInstance) - // Verify apps are downloaded by init-container on CM, Deployer - appVersion := "V2" + //########### INITIAL VERIFICATIONS ########### + // Verify V2 apps are downloaded on Cluster Manager and Deployer initContDownloadLocationIdxc := "/init-apps/" + appSourceNameIdxc initContDownloadLocationShc := "/init-apps/" + appSourceNameShc - testenvInstance.Log.Info("Verify apps are downloaded by init container for C3", "version", appVersion) podNames := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName()), fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded on Cluster Manager", appVersion)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appFileList, initContDownloadLocationIdxc) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded on Deployer", appVersion)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())}, appFileList, initContDownloadLocationShc) - // Verify apps are downloaded by init-container on MC + // Verify V2 apps are downloaded on Monitoring Console initContDownloadLocationMCPod := "/init-apps/" + appSourceNameMC mcPodName := fmt.Sprintf(testenv.MonitoringConsolePod, mcName, 0) - testenvInstance.Log.Info("Verify apps are downloaded by init container for MC", "POD", mcPodName, "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded on Monitoring Console", appVersion)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{mcPodName}, appFileList, initContDownloadLocationMCPod) // Verify bundle push status + testenvInstance.Log.Info(fmt.Sprintf("Verify bundle push status (%s apps)", appVersion)) testenv.VerifyClusterManagerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), indexerReplicas, "") testenv.VerifyDeployerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), shReplicas) // Saving current V2 bundle hash for future comparison clusterManagerBundleHash := testenv.GetClusterManagerBundleHash(deployment) - // Verify apps are copied to location + // Verify V2 apps are copied to location allPodNames := podNames allPodNames = append(allPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) allPodNames = append(allPodNames, testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1)...) - testenvInstance.Log.Info("Verify apps are copied to correct location based on Pod KIND for app", "version", appVersion) + // Verify V2 apps are copied on Indexers and Search Heads + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to correct location on Indexers and Search Heads", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV2, true, true) - testenvInstance.Log.Info("Verify apps are copied to correct location for MC", "version", appVersion) + // Verify V2 apps are copied on Monitoring Console + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to correct location on Monitoring Console", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), []string{mcPodName}, appListV2, true, false) - // Verify apps are not copied in /etc/apps/ on CM and on Deployer (therefore not installed on Deployer and on CM) - testenvInstance.Log.Info("Verify apps are NOT copied to /etc/apps on CM and Deployer for app", "version", appVersion, "App List", appFileList) + // Verify apps are not copied in /etc/apps/ on Cluster Manager and on Deployer (therefore not installed on Deployer and on Cluster Manager) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are NOT copied to /etc/apps on Cluster Manager and Deployer (App list: %s)", appVersion, appFileList)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), podNames, appListV2, false, false) - // Verify apps are installed on C3 - testenvInstance.Log.Info("Verify V2 apps are installed on C3 Indexers and SHs", "version", appVersion) + // Verify V2 apps are installed on Indexers and Search Heads + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are installed on Indexers and Search Heads", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV2, true, "enabled", true, true) - // Verify apps are installed on MC - testenvInstance.Log.Info("Verify V2 apps are installed on MC", "version", appVersion) + // Verify V2 apps are installed on Monitoring Console + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are installed on Monitoring Console", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), []string{mcPodName}, appListV2, true, "enabled", true, false) + //############## DOWNGRADE APPS ############### // Delete apps on S3 - testenvInstance.Log.Info("Delete apps on S3 for", "Version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) uploadedApps = nil - // Upload older version of apps (V1) to S3 for C3 + // Upload V1 apps to S3 for Indexer Cluster appVersion = "V1" appFileList = testenv.GetAppFileList(appListV1, 1) + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexers", appVersion)) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory for C3") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexers", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) + + // Upload V1 apps to S3 for Search Head Cluster + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory for C3") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload older version of apps (V1) to MC + // Upload V1 apps to S3 for Monitoring Console + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory for MC") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Wait for the poll period for the apps to be downloaded time.Sleep(2 * time.Minute) - // Ensure that the CM goes to Ready phase + // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(deployment, testenvInstance) - // Ensure indexers go to Ready phase + // Ensure Indexers go to Ready phase testenv.SingleSiteIndexersReady(deployment, testenvInstance) - // Ensure SHC go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(deployment, testenvInstance) // Verify RF SF is met testenv.VerifyRFSFMet(deployment, testenvInstance) - // Verify MC is ready and stays in ready state + // Verify Monitoring Console is ready and stays in ready state testenv.VerifyMonitoringConsoleReady(deployment, deployment.GetName(), mc, testenvInstance) - // Verify apps are downloaded by init-container - testenvInstance.Log.Info("Verify apps are downloaded by init container for C3 for apps", " version", appVersion) + //########### FINAL VERIFICATIONS ############# + // Verify V1 apps are downloaded + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded on Cluster Manager", appVersion)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appFileList, initContDownloadLocationIdxc) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded on Deployer", appVersion)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())}, appFileList, initContDownloadLocationShc) - testenvInstance.Log.Info("Verify apps are downloaded by init container for MC for apps", "POD", mcPodName, "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded on Monitoring Console", appVersion)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{mcPodName}, appFileList, initContDownloadLocationMCPod) // Verify bundle push status + testenvInstance.Log.Info(fmt.Sprintf("Verify bundle push status (%s apps)", appVersion)) testenv.VerifyClusterManagerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), indexerReplicas, clusterManagerBundleHash) testenv.VerifyDeployerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), shReplicas) - // Verify apps are copied to location - testenvInstance.Log.Info("Verify apps are copied to correct location for C3 pods", " version", appVersion) + // Verify V1 apps are copied to location + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to correct location on C3 pods", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV1, true, true) - testenvInstance.Log.Info("Verify V2 apps are copied to correct location for MC", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to correct location on Monitoring Console", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), []string{mcPodName}, appListV1, true, false) - // Verify apps are not copied in /etc/apps/ on CM and on Deployer (therefore not installed on Deployer and on CM) - testenvInstance.Log.Info("Verify apps are NOT copied to /etc/apps on CM and Deployer for app", " version", appVersion) + // Verify V1 apps are not copied in /etc/apps/ on Cluster Manager and Deployer (therefore not installed on Cluster Manager and Deployer) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are NOT copied to /etc/apps on Cluster Manager and Deployer", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), podNames, appListV1, false, false) - // Verify apps are downgraded on C3(cluster-wide) - testenvInstance.Log.Info("Verify apps are downgraded on the C3 pods", "version", appVersion) + // Verify V1 apps are downgraded on Search Heads and Indexers + testenvInstance.Log.Info(fmt.Sprintf("Verify apps have been downgraded to %s on Search Heads and Indexers pods", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV1, true, "enabled", false, true) - // Verify apps are downgraded on MC - testenvInstance.Log.Info("Verify apps are downgraded on MC", "version", appVersion) + // Verify V1 apps are downgraded on Monitoring Console + testenvInstance.Log.Info(fmt.Sprintf("Verify apps have been downgraded to %s on Monitoring Console pod", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), []string{mcPodName}, appListV1, true, "enabled", false, false) }) }) - Context("Single Site Indexer Cluster with SHC (C3) with App Framework", func() { - It("integration, c3, appframework: can deploy a C3 SVA with App Framework enabled, install apps,scale up IDXC and SHC, install app on new pods, scale down", func() { + Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { + It("integration, c3, appframework: can deploy a C3 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { /* Test Steps ################## SETUP #################### - * Upload apps on S3 - * Create App Source for C3 SVA (CM and SHC Deployer) + * Upload V1 apps on S3 for Indexer Cluster and Search Head Cluster + * Create app sources for Cluster Manager and Deployer * Prepare and deploy C3 CRD with app config and wait for pods to be ready - ################## VERIFICATIONS ############# + ########## INITIAL VERIFICATIONS ############ * Verify bundle push is successful - * Verify apps are copied, installed on SH,Indexers pods - ########## SCALING UP ########### + * Verify apps are copied, installed on Search Heads and Indexers + ############# SCALING UP ################### * Scale up indexers and Search Heads * Wait for C3 to be ready - ############### VERIFICATION ############## + ########## SCALING UP VERIFICATIONS ######### * Verify bundle push is sucessful - * Verify apps are copied and installed on new SH and Indexers pods + * Verify apps are copied and installed on all Search Heads and Indexers pods ############### SCALING DOWN ################ - * Scale down indexers and Search Heads + * Scale down Indexers and Search Heads * Wait for C3 to be ready - ############### VERIFICATION ############## + ######## SCALING DOWN VERIFICATIONS ######### * Verify bundle push is sucessful - * Verify apps are still copied and installed on all SH and Indexers pods + * Verify apps are still copied and installed on all Search Heads and Indexers pods */ - // Upload V1 apps to S3 for IDXC + //################## SETUP ################## + // Upload V1 apps to S3 for Indexer Cluster + appVersion := "V1" + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) appFileList := testenv.GetAppFileList(appListV1, 1) s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to S3 for SHC + // Upload V1 apps to S3 for Search Head Cluster + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec for C3 @@ -523,67 +569,70 @@ var _ = Describe("c3appfw test", func() { appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(testenvInstance, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) // Deploy C3 CRD - testenvInstance.Log.Info("Deploy Single Site Indexer Cluster") + testenvInstance.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") indexerReplicas := 3 shReplicas := 3 err = deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the CM goes to Ready phase + // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(deployment, testenvInstance) - // Ensure indexers go to Ready phase + // Ensure Indexers go to Ready phase testenv.SingleSiteIndexersReady(deployment, testenvInstance) - // Ensure search head cluster go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(deployment, testenvInstance) // Verify RF SF is met testenv.VerifyRFSFMet(deployment, testenvInstance) - // Verify apps are downloaded by init-container on CM, Deployer - appVersion := "V1" + //########## INITIAL VERIFICATIONS ############ + // Verify V1 apps are downloaded on Cluster Manager and Deployer initContDownloadLocationIdxc := "/init-apps/" + appSourceNameIdxc initContDownloadLocationShc := "/init-apps/" + appSourceNameShc managerPodNames := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName()), fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - testenvInstance.Log.Info("Verify apps are downloaded by init container for apps", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded on Cluster Manager", appVersion)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appFileList, initContDownloadLocationIdxc) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded on Deployer", appVersion)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())}, appFileList, initContDownloadLocationShc) // Verify bundle push status + testenvInstance.Log.Info("Verify bundle push status") testenv.VerifyClusterManagerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), indexerReplicas, "") testenv.VerifyDeployerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), shReplicas) // Verify apps are copied to correct location allPodNames := testenv.DumpGetPods(testenvInstance.GetName()) - testenvInstance.Log.Info("Verify apps are copied to correct location based on Pod KIND for app", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to correct location on all pods", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV1, true, true) - // Verify apps are not copied in /etc/apps/ on CM and on Deployer (therefore not installed on Deployer and on CM) - testenvInstance.Log.Info("Verify apps are NOT copied to /etc/apps on CM and Deployer for app", "version", appVersion, "App List", appFileList) + // Verify apps are not copied in /etc/apps/ on Cluster Manager and on Deployer (therefore not installed on Deployer and on Cluster Manager) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are NOT copied to /etc/apps on Cluster Manager and Deployer (App list: %s)", appVersion, appFileList)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), managerPodNames, appListV1, false, false) // Verify apps are installed on C3 - testenvInstance.Log.Info("Verify apps are installed on C3 Indexers and SHs", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are installed cluster-wide", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV1, true, "enabled", false, true) - // Get instance of current SHC CR with latest config + //############# SCALING UP ################### + // Get instance of current Search Head Cluster CR with latest config shcName := deployment.GetName() + "-shc" shc := &enterpriseApi.SearchHeadCluster{} err = deployment.GetInstance(shcName, shc) Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - // Scale up SHC + // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas scaledSHReplicas := defaultSHReplicas + 1 - testenvInstance.Log.Info("Scaling up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) + testenvInstance.Log.Info("Scale up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - // Update Replicas of SHC + // Update Replicas of Search Head Cluster shc.Spec.Replicas = int32(scaledSHReplicas) err = deployment.UpdateCR(shc) - Expect(err).To(Succeed(), "Failed to scale Search Head Cluster") + Expect(err).To(Succeed(), "Failed to scale up Search Head Cluster") - // Ensure SHC scales up and go to ScalingUp phase + // Ensure Search Head Cluster scales up and go to ScalingUp phase testenv.VerifySearchHeadClusterPhase(deployment, testenvInstance, splcommon.PhaseScalingUp) // Get instance of current Indexer CR with latest config @@ -593,63 +642,66 @@ var _ = Describe("c3appfw test", func() { Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 - testenvInstance.Log.Info("Scaling up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) + testenvInstance.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) // Update Replicas of Indexer Cluster idxc.Spec.Replicas = int32(scaledIndexerReplicas) err = deployment.UpdateCR(idxc) - Expect(err).To(Succeed(), "Failed to scale Indxer Cluster") + Expect(err).To(Succeed(), "Failed to scale up Indexer Cluster") - // Ensure Indexer cluster scales up and go to ScalingUp phase + // Ensure Indexer Cluster scales up and go to ScalingUp phase testenv.VerifyIndexerClusterPhase(deployment, testenvInstance, splcommon.PhaseScalingUp, idxcName) - // Ensure Indexer cluster go to Ready phase + // Ensure Indexer Cluster go to Ready phase testenv.SingleSiteIndexersReady(deployment, testenvInstance) // Verify New Indexer On Cluster Manager indexerName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), scaledIndexerReplicas-1) - testenvInstance.Log.Info("Checking for New Indexer On Cluster Manager", "Indexer Name", indexerName) + testenvInstance.Log.Info(fmt.Sprintf("Checking for New Indexer %s On Cluster Manager", indexerName)) Expect(testenv.CheckIndexerOnCM(deployment, indexerName)).To(Equal(true)) - // Ensure SHC go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(deployment, testenvInstance) // Verify RF SF is met testenv.VerifyRFSFMet(deployment, testenvInstance) + //########## SCALING UP VERIFICATIONS ######### // Verify bundle push status. Bundle hash not compared as scaleup does not involve new config + testenvInstance.Log.Info("Verify bundle push status after scaling up") testenv.VerifyClusterManagerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), int(scaledIndexerReplicas), "") testenv.VerifyDeployerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), int(scaledSHReplicas)) - // Verify apps are copied to location + // Verify V1 apps are copied to location allPodNames = testenv.DumpGetPods(testenvInstance.GetName()) - testenvInstance.Log.Info("Verify apps are copied to correct location based on Pod KIND for app", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to correct location after scaling up Indexers and Search Heads", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV1, true, true) - // Verify apps are not copied in /etc/apps/ on CM and on Deployer (therefore not installed on Deployer and on CM) - testenvInstance.Log.Info("Verify apps are NOT copied to /etc/apps on CM and Deployer for app", "version", appVersion) + // Verify V1 apps are not copied in /etc/apps/ on Cluster Manager and on Deployer (therefore not installed on Deployer and on Cluster Manager) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are NOT copied to /etc/apps on Cluster Manager and Deployer after scaling up of Indexers and Search Heads", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), managerPodNames, appListV1, false, false) - // Verify apps are installed on C3 - testenvInstance.Log.Info("Verify apps are installed on C3 Indexers and SHs after scaling up", "version", appVersion) + // Verify V1 apps are installed on C3 + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are installed on Indexers and Search Heads after scaling up", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV1, true, "enabled", false, true) - // Get instance of current SHC CR with latest config + //############### SCALING DOWN ################ + // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} err = deployment.GetInstance(shcName, shc) Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - // Scale down SHC + // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas scaledSHReplicas = defaultSHReplicas - 1 - testenvInstance.Log.Info("Scaling down Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) + testenvInstance.Log.Info("Scale down Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - // Update Replicas of SHC + // Update Replicas of Search Head Cluster shc.Spec.Replicas = int32(scaledSHReplicas) err = deployment.UpdateCR(shc) Expect(err).To(Succeed(), "Failed to scale down Search Head Cluster") - // Ensure SHC scales down and go to ScalingDown phase + // Ensure Search Head Cluster scales down and go to ScalingDown phase testenv.VerifySearchHeadClusterPhase(deployment, testenvInstance, splcommon.PhaseScalingDown) // Get instance of current Indexer CR with latest config @@ -664,66 +716,73 @@ var _ = Describe("c3appfw test", func() { err = deployment.UpdateCR(idxc) Expect(err).To(Succeed(), "Failed to Scale down Indexer Cluster") - // Ensure Indexer cluster scales down and go to ScalingDown phase + // Ensure Indexer Cluster scales down and go to ScalingDown phase testenv.VerifyIndexerClusterPhase(deployment, testenvInstance, splcommon.PhaseScalingDown, idxcName) - // Ensure Indexer cluster go to Ready phase + // Ensure Indexer Cluster go to Ready phase testenv.SingleSiteIndexersReady(deployment, testenvInstance) - // Ensure SHC go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(deployment, testenvInstance) // Verify RF SF is met testenv.VerifyRFSFMet(deployment, testenvInstance) + //######## SCALING DOWN VERIFICATIONS ######### // Verify bundle push status + testenvInstance.Log.Info("Verify bundle push status after scaling down") testenv.VerifyClusterManagerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), int(scaledIndexerReplicas), "") testenv.VerifyDeployerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), int(scaledSHReplicas)) // Verify apps are copied to correct location allPodNames = testenv.DumpGetPods(testenvInstance.GetName()) - testenvInstance.Log.Info("Verify apps are copied to correct location based on Pod KIND after scaling down of indexers and SH", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to correct location after scaling down of Indexers and Search Heads", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV1, true, true) - // Verify apps are not copied in /etc/apps/ on CM and on Deployer (therefore not installed on Deployer and on CM) - testenvInstance.Log.Info("Verify apps are NOT copied to /etc/apps on CM and Deployer after scaling down of indexers and SH", "version", appVersion) + // Verify apps are not copied in /etc/apps/ on Cluster Manager and on Deployer (therefore not installed on Deployer and on Cluster Manager) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are NOT copied to /etc/apps on Cluster Manager and Deployer after scaling down of Indexers and Search Heads", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), managerPodNames, appListV1, false, false) - // Verify apps are installed on C3 - testenvInstance.Log.Info("Verify apps are installed on C3 Indexers and SHs after scaling down", "version", appVersion) + // Verify apps are installed cluster-wide after scaling down + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are installed on the pods after scaling down of Indexers and Search Heads", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV1, true, "enabled", false, true) }) }) Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("smoke, c3, appframework: can deploy a C3 SVA and have apps installed locally on CM and SHC Deployer", func() { + It("smoke, c3, appframework: can deploy a C3 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps ################## SETUP #################### - * Create App Source with local scope for C3 SVA (CM and SHC Deployer) + * Upload V1 apps to S3 + * Create app source with local scope for C3 SVA (Cluster Manager and Deployer) * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ################## VERIFICATION ############# - * Verify apps are installed locally on CM and Deployer + ############# INITIAL VERIFICATION ########## + * Verify apps are installed locally on Cluster Manager and Deployer ############### UPGRADE APPS ################ * Upgrade apps in app sources * Wait for pods to be ready - ############### VERIFICATIONS ############### + ########### UPGRADE VERIFICATIONS ########### * Verify bundle push is successful - * Verify apps are copied, installed and upgraded on CM and Deployer + * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer */ - // Upload V1 apps to S3 for IDXC + //################## SETUP #################### + // Upload V1 apps to S3 for Indexer Cluster + appVersion := "V1" s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) appFileList := testenv.GetAppFileList(appListV1, 1) + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Indexer Cluster", appVersion)) uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to S3 for SHC + // Upload V1 apps to S3 for Search Head Cluster s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Search Head Cluster", appVersion)) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec @@ -736,96 +795,100 @@ var _ = Describe("c3appfw test", func() { // Deploy C3 CRD indexerReplicas := 3 - testenvInstance.Log.Info("Deploy Single Site Indexer Cluster") + testenvInstance.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") err = deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the CM goes to Ready phase + // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(deployment, testenvInstance) // Ensure Indexers go to Ready phase testenv.SingleSiteIndexersReady(deployment, testenvInstance) - // Ensure SHC go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(deployment, testenvInstance) // Verify RF SF is met testenv.VerifyRFSFMet(deployment, testenvInstance) - // Verify apps are downloaded by init-container + //############## INITIAL VERIFICATION ########## + // Verify V1 apps are downloaded initContDownloadLocationIdxc := "/init-apps/" + appSourceNameIdxc initContDownloadLocationShc := "/init-apps/" + appSourceNameShc podNames := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName()), fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - appVersion := "V1" - testenvInstance.Log.Info("Verify apps are downloaded by init container", "version", appVersion, "App List", appFileList) + + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded on Cluster Manager (App list: %s)", appVersion, appFileList)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appFileList, initContDownloadLocationIdxc) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded on Deployer (App list: %s)", appVersion, appFileList)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())}, appFileList, initContDownloadLocationShc) - // Verify apps are copied at the correct location on CM and on Deployer (/etc/apps/) - testenvInstance.Log.Info("Verify apps are copied to correct location on CM and on Deployer (/etc/apps/)", "version", appVersion) + // Verify V1 apps are copied at the correct location on Cluster Manager and on Deployer (/etc/apps/) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to correct location on Cluster Manager and on Deployer (/etc/apps/)", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), podNames, appListV1, true, false) - // Verify apps are installed locally on CM and on SHC Deployer - testenvInstance.Log.Info("Verify apps are installed locally on CM and Deployer by running Splunk CLI commands for app", "version", appVersion) + // Verify V1 apps are installed locally on Cluster Manager and on Deployer + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are installed locally on Cluster Manager and Deployer", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), podNames, appListV1, false, "enabled", false, false) - // Verify apps are not copied in the apps folder on CM and /etc/shcluster/ on Deployer (therefore not installed on peers and on SH) - testenvInstance.Log.Info("Verify apps are NOT copied to "+splcommon.ManagerAppsLoc+" on CM and "+splcommon.SHCluster+" on Deployer for app", "version", appVersion, "App List", appFileList) + // Verify V1 apps are not copied in the apps folder on Cluster Manager and /etc/shcluster/ on Deployer (therefore not installed on Indexers and on Search Heads) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are NOT copied to "+splcommon.ManagerAppsLoc+" on Cluster Manager and "+splcommon.SHCluster+" on Deployer (App list: %s)", appVersion, appFileList)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), podNames, appListV1, false, true) - // Delete apps on S3 - testenvInstance.Log.Info("Delete apps on S3 for", "Version", appVersion) + //############### UPGRADE APPS ################ + // Delete V1 apps on S3 + testenvInstance.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) uploadedApps = nil - testenvInstance.Log.Info("Testing upgrade scenario") - // Upload newer version of apps (V2) to S3 + // Upload V2 apps to S3 appVersion = "V2" - testenvInstance.Log.Info("Uploading apps S3 for", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3", appVersion)) appFileList = testenv.GetAppFileList(appListV2, 2) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Wait for the poll period for the apps to be downloaded time.Sleep(2 * time.Minute) - // Ensure that the CM goes to Ready phase + // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(deployment, testenvInstance) // Ensure Indexers go to Ready phase testenv.SingleSiteIndexersReady(deployment, testenvInstance) - // Ensure SHC go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(deployment, testenvInstance) // Verify RF SF is met testenv.VerifyRFSFMet(deployment, testenvInstance) - // Verify apps are downloaded by init-container - testenvInstance.Log.Info("Verify apps are downloaded by init container for apps", "version", appVersion) + //########### UPGRADE VERIFICATIONS ########### + // Verify V2 apps are downloaded on Cluster Manager and Deployer + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded on Cluster Manager", appVersion)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appFileList, initContDownloadLocationIdxc) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded on Deployer", appVersion)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())}, appFileList, initContDownloadLocationShc) - // Verify apps are copied at the correct location on CM and on Deployer (/etc/apps/) - testenvInstance.Log.Info("Verify apps are copied to correct location on CM and on Deployer (/etc/apps/) for app", "version", appVersion) + // Verify V2 apps are copied at the correct location on Cluster Manager and on Deployer (/etc/apps/) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to correct location on Cluster Manager and on Deployer (/etc/apps/)", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), podNames, appListV2, true, false) - // Verify apps are installed locally on CM and on SHC Deployer - testenvInstance.Log.Info("Verify apps are installed Locally on CM and Deployer by running Splunk CLI commands for app", "version", appVersion) + // Verify V2 apps are installed locally on Cluster Manager and on Deployer + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are installed locally on Cluster Manager and Deployer", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), podNames, appListV2, true, "enabled", true, false) - // Verify apps are not copied in the apps folder on CM and /etc/shcluster/ on Deployer (therefore not installed on peers and on SH) - testenvInstance.Log.Info("Verify apps are NOT copied to "+splcommon.ManagerAppsLoc+" on CM and "+splcommon.SHCluster+" on Deployer for app", "version", appVersion, "App List", appFileList) + // Verify V2 apps are not copied in the apps folder on Cluster Manager and /etc/shcluster/ on Deployer (therefore not installed on Indexers and on Search Heads) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are NOT copied to "+splcommon.ManagerAppsLoc+" on Cluster Manager and "+splcommon.SHCluster+" on Deployer (App list: %s)", appVersion, appFileList)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), podNames, appListV2, false, true) }) }) Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3, appframework: can deploy a C3 SVA and have ES app installed on SHC", func() { + It("integration, c3, appframework: can deploy a C3 SVA and have ES app installed on Search Head Cluster", func() { /* Test Steps ################## SETUP #################### @@ -836,18 +899,20 @@ var _ = Describe("c3appfw test", func() { * Verify ES app is installed on Deployer and on Search Heads */ - // Create local directory for file download - s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) + //################## SETUP #################### - // Upload ES app to S3 + // Download ES app from S3 + testenvInstance.Log.Info("Download ES app from S3") esApp := []string{"SplunkEnterpriseSecuritySuite"} appFileList := testenv.GetAppFileList(esApp, 1) - - // Download ES app from S3 err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download ES app file") + Expect(err).To(Succeed(), "Unable to download ES app file from S3") + + // Create local directory for file download + s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) // Upload ES app to S3 + testenvInstance.Log.Info("Upload ES app to S3") uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload ES app to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -858,18 +923,19 @@ var _ = Describe("c3appfw test", func() { appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(testenvInstance, appSourceVolumeNameShc, enterpriseApi.ScopeClusterWithPreConfig, appSourceName, s3TestDirShc, 60) // Deploy C3 SVA - // Deploy the Cluster manager - testenvInstance.Log.Info("Deploy Cluster manager in single site configuration") + // Deploy the Cluster Manager + testenvInstance.Log.Info("Deploy Cluster Manager") _, err = deployment.DeployClusterMaster(deployment.GetName(), "", "", "") Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - // Deploy the Indexer cluster - testenvInstance.Log.Info("Deploy Indexer Cluster in single site configuration") + // Deploy the Indexer Cluster + testenvInstance.Log.Info("Deploy Single Site Indexer Cluster") indexerReplicas := 3 _, err = deployment.DeployIndexerCluster(deployment.GetName()+"-idxc", deployment.GetName(), indexerReplicas, deployment.GetName(), "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster") - // Deploy the SHC + // Deploy the Search Head Cluster + testenvInstance.Log.Info("Deploy Search Head Cluster") shSpec := enterpriseApi.SearchHeadClusterSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: splcommon.Spec{ @@ -889,35 +955,33 @@ var _ = Describe("c3appfw test", func() { AppFrameworkConfig: appFrameworkSpecShc, } _, err = deployment.DeploySearchHeadClusterWithGivenSpec(deployment.GetName()+"-shc", shSpec) - Expect(err).To(Succeed(), "Unable to deploy SHC with App framework") + Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster") - // Ensure that the CM goes to Ready phase + // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(deployment, testenvInstance) // Ensure Indexers go to Ready phase testenv.SingleSiteIndexersReady(deployment, testenvInstance) - // Ensure SHC go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(deployment, testenvInstance) // Verify RF SF is met testenv.VerifyRFSFMet(deployment, testenvInstance) - // Verify ES is downloaded by init-container - testenvInstance.Log.Info("Verfiy ES app is downloaded by init container on deployer pod") + //################## VERIFICATIONS ############# + // Verify ES is downloaded + testenvInstance.Log.Info("Verify ES app is downloaded on Deployer") initContDownloadLocation := "/init-apps/" + appSourceName deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), deployerPod, appFileList, initContDownloadLocation) - // Verify ES app is installed locally on SHC Deployer - testenvInstance.Log.Info("Verfiy ES app is installed locally on deployer pod") + // Verify ES app is installed locally on Deployer + testenvInstance.Log.Info("Verify ES app is installed locally on Deployer") testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), deployerPod, esApp, true, "disabled", false, false) - // Verify bundle push status - testenv.VerifyDeployerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), int(shSpec.Replicas)) - - // Verify apps are installed on SHs - testenvInstance.Log.Info("Verfiy ES app is installed on Search Heads") + // Verify ES is installed on Search Heads + testenvInstance.Log.Info("Verify ES app is installed on Search Heads") podNames := []string{} for i := 0; i < int(shSpec.Replicas); i++ { sh := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i) @@ -928,40 +992,44 @@ var _ = Describe("c3appfw test", func() { }) Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration, appframework: can deploy a C3 SVA with apps installed locally on CM and Deployer, cluster-wide on Peers and SHs, then upgrade them", func() { + It("c3, integration, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them", func() { /* Test Steps ################## SETUP #################### - * Create App framework spec with app sources having both local and cluster scopes for C3 SVA - * Prepare and deploy C3 CRD and wait for pods to be ready - ################## VERIFICATION ############# - * Verify apps with local scope are installed locally on Cluster manager and SHC deployer - * Verify apps with cluster scope are installed cluster-wide on indexers and Search Heads + * Create app sources having both local and cluster scopes for C3 SVA + * Prepare and deploy C3 CRD with app framework and wait for pods to be ready + ############ INITIAL VERIFICATIONS ########## + * Verify apps with local scope are installed locally on Cluster Manager and Deployer + * Verify apps with cluster scope are installed cluster-wide on Indexers and Search Heads ############### UPGRADE APPS ################ - * Upload newer versions of apps (V2) on S3 + * Upload V2 apps on S3 * Wait for pods to be ready - ############### VERIFICATION ################ + ########## UPGRADE VERIFICATION ############# * Verify bundle push is successful - * Verify apps with local scope are upgraded locally on Cluster manager and on SHC deployer - * Verify apps with cluster scope are upgraded cluster-wide on indexers and Search Heads + * Verify apps with local scope are upgraded locally on Cluster Manager and on Deployer + * Verify apps with cluster scope are upgraded cluster-wide on Indexers and Search Heads */ + //################## SETUP #################### // Split Applist into 2 lists for local and cluster install + appVersion := "V1" appListLocal := appListV1[len(appListV1)/2:] appListCluster := appListV1[:len(appListV1)/2] - // Upload appListLocal to S3 + // Upload appListLocal list of apps to S3 (to be used for local install) + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) s3TestDir := "c3appfw-" + testenv.RandomDNSName(4) appFileList := testenv.GetAppFileList(appListLocal, 1) uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload apps to a 2nd directory on S3 bucket as we need 2 buckets locations for this test.(appListLocal to be used for local install, appListCluster for cluster install) + // Upload appListCluster list of apps to S3 (to be used for cluster-wide install) + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for cluster-wide install (cluster scope)", appVersion)) s3TestDirCluster := "c3appfw-cluster-" + testenv.RandomDNSName(4) clusterappFileList := testenv.GetAppFileList(appListCluster, 1) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec @@ -987,52 +1055,53 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameCluster, s3TestDirCluster, appSourceClusterDefaultSpec)} appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - // Create Single site Cluster and SHC, with App Framework enabled on CM and SHC Deployer - testenvInstance.Log.Info("Create Single site Indexer Cluster with Local and Cluster Install for apps") + // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer + testenvInstance.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") indexerReplicas := 3 shReplicas := 3 err = deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the CM goes to Ready phase + // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(deployment, testenvInstance) // Ensure Indexers go to Ready phase testenv.SingleSiteIndexersReady(deployment, testenvInstance) - // Ensure SHC go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(deployment, testenvInstance) // Verify RF SF is met testenv.VerifyRFSFMet(deployment, testenvInstance) - // Verify apps with local scope are downloaded by init-container - appVersion := "V1" + //############ INITIAL VERIFICATIONS ########## + // Verify V1 apps with local scope are downloaded initContDownloadLocation := "/init-apps/" + appSourceNameLocal downloadPodNames := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName()), fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} appFileList = testenv.GetAppFileList(appListLocal, 1) - testenvInstance.Log.Info("Verify apps are downloaded by init container for apps", "version", appVersion, "App List", appFileList) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps with local scope are downloaded (App list: %s)", appVersion, appFileList)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), downloadPodNames, appFileList, initContDownloadLocation) - // Verify apps with cluster scope are downloaded by init-container + // Verify V1 apps with cluster scope are downloaded initContDownloadLocation = "/init-apps/" + appSourceNameCluster appFileList = testenv.GetAppFileList(appListCluster, 1) - testenvInstance.Log.Info("Verify apps are downloaded by init container for apps", "version", appVersion, "App List", appFileList) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps with cluster scope are downloaded (App list: %s)", appVersion, appFileList)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), downloadPodNames, appFileList, initContDownloadLocation) // Verify bundle push status + testenvInstance.Log.Info(fmt.Sprintf("Verify bundle push status (%s apps)", appVersion)) testenv.VerifyClusterManagerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), indexerReplicas, "") testenv.VerifyDeployerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), shReplicas) // Saving current V1 bundle hash for future comparison clusterManagerBundleHash := testenv.GetClusterManagerBundleHash(deployment) - // Verify apps with local scope are installed locally on CM and on SHC Deployer + // Verify apps with local scope are installed locally on Cluster Manager and on Deployer localPodNames := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName()), fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - testenvInstance.Log.Info("Verify apps are installed Locally on CM and Deployer by running Splunk CLI commands for app", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps with local scope are installed locally on Cluster Manager and Deployer", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), localPodNames, appListLocal, true, "enabled", false, false) - // Verify apps with cluster scope are installed on indexers + // Verify apps with cluster scope are installed on Indexers clusterPodNames := []string{} for i := 0; i < int(indexerReplicas); i++ { sh := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) @@ -1043,102 +1112,111 @@ var _ = Describe("c3appfw test", func() { sh := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i) clusterPodNames = append(clusterPodNames, string(sh)) } - testenvInstance.Log.Info("Verify apps are installed clusterwide on indexers and search-heads by running Splunk CLI commands for app", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps with cluster scope are installed on Indexers and Search Heads", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), clusterPodNames, appListCluster, true, "enabled", false, true) + //############### UPGRADE APPS ################ // Delete apps on S3 + testenvInstance.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) uploadedApps = nil - // Upload newer version of apps to S3 + // Upload appListLocal list of V2 apps to S3 (to be used for local install) appVersion = "V2" + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) appFileList = testenv.GetAppFileList(appListLocal, 2) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV2) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for local install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload apps to a 2nd directory on S3 bucket as we need 2 buckets locations for this test.(appListLocal apps to be used for local install, appListCluster apps for cluster install) + // Upload appListCluster list of V2 apps to S3 (to be used for cluster-wide install) clusterappFileList = testenv.GetAppFileList(appListCluster, 2) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for cluster-wide install", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Wait for the poll period for the apps to be downloaded time.Sleep(2 * time.Minute) - // Ensure that the CM goes to Ready phase + // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(deployment, testenvInstance) // Ensure Indexers go to Ready phase testenv.SingleSiteIndexersReady(deployment, testenvInstance) - // Ensure SHC go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(deployment, testenvInstance) // Verify RF SF is met testenv.VerifyRFSFMet(deployment, testenvInstance) - // Verify apps with local scope are downloaded by init-container + //########## UPGRADE VERIFICATION ############# + // Verify apps with local scope are downloaded initContDownloadLocation = "/init-apps/" + appSourceNameLocal appFileList = testenv.GetAppFileList(appListLocal, 2) - testenvInstance.Log.Info("Verify apps are downloaded by init container for apps", "version", appVersion, "App List", appFileList) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps with local scope are downloaded (App list: %s)", appVersion, appFileList)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), downloadPodNames, appFileList, initContDownloadLocation) - // Verify apps with cluster scope are downloaded by init-container + // Verify apps with cluster scope are downloaded initContDownloadLocation = "/init-apps/" + appSourceNameCluster appFileList = testenv.GetAppFileList(appListCluster, 2) - testenvInstance.Log.Info("Verify apps are downloaded by init container for apps", "version", appVersion, "App List", appFileList) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps with cluster scope are downloaded (App list: %s)", appVersion, appFileList)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), downloadPodNames, appFileList, initContDownloadLocation) // Verify bundle push status + testenvInstance.Log.Info(fmt.Sprintf("Verify bundle push status (%s apps)", appVersion)) testenv.VerifyClusterManagerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), indexerReplicas, clusterManagerBundleHash) testenv.VerifyDeployerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), shReplicas) - // Verify apps with local scope are upgraded locally on CM and on SHC Deployer - testenvInstance.Log.Info("Verify apps are upgraded Locally on CM and Deployer by running Splunk CLI commands for app", "version", appVersion) + // Verify apps with local scope are upgraded locally on Cluster Manager and on Deployer + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps with local scope are upgraded locally on Cluster Manager and Deployer", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), localPodNames, appListLocal, true, "enabled", true, false) - // Verify apps with cluster scope are upgraded on indexers - testenvInstance.Log.Info("Verify apps are upgraded clusterwide on indexers and search-heads by running Splunk CLI commands for app", "version", appVersion) + // Verify apps with cluster scope are upgraded on Indexers + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps with cluster scope are upgraded on Indexers and Search Heads", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), clusterPodNames, appListCluster, true, "enabled", true, true) }) }) Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration, appframework: can deploy a C3 SVA with apps installed locally on CM and Deployer, cluster-wide on Peers and SHs, then downgrade them", func() { + It("c3, integration, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then downgrade them", func() { /* Test Steps ################## SETUP #################### - * Create App framework spec with app sources having both local and cluster scopes for C3 SVA + * Create app sources having both local and cluster scopes for C3 SVA * Prepare and deploy C3 CRD and wait for pods to be ready - ################## VERIFICATION ############# - * Verify apps with local scope are installed locally on Cluster manager and SHC deployer - * Verify apps with cluster scope are installed cluster-wide on indexers and Search Heads - ############### DOWNGRADE APPS ############## - * Upload older versions of apps (V1) on S3 + ############# INITIAL VERIFICATION ########## + * Verify apps with local scope are installed locally on Cluster Manager and Deployer + * Verify apps with cluster scope are installed cluster-wide on Indexers and Search Heads + ############# DOWNGRADE APPS ################ + * Upload V1 apps on S3 * Wait for pods to be ready - ############### VERIFICATION ################ + ########## DOWNGRADE VERIFICATION ########### * Verify bundle push is successful - * Verify apps with local scope are downgraded locally on Cluster manager and on SHC deployer - * Verify apps with cluster scope are downgraded cluster-wide on indexers and Search Heads + * Verify apps with local scope are downgraded locally on Cluster Manager and on Deployer + * Verify apps with cluster scope are downgraded cluster-wide on Indexers and Search Heads */ + //################## SETUP #################### // Split Applist into 2 lists for local and cluster install + appVersion := "V2" appListLocal := appListV2[len(appListV2)/2:] appListCluster := appListV2[:len(appListV2)/2] // Upload appListLocal to S3 + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) s3TestDir := "c3appfw-" + testenv.RandomDNSName(4) appFileList := testenv.GetAppFileList(appListLocal, 2) uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV2) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s local apps to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload apps to a 2nd directory on S3 bucket as we need 2 buckets locations for this test.(appListLocal to be used for local install, appListCluster for cluster install) + // Upload appListCluster list of apps to S3 (to be used for cluster-wide install) + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for cluster-wide install (cluster scope)", appVersion)) s3TestDirCluster := "c3appfw-cluster-" + testenv.RandomDNSName(4) clusterappFileList := testenv.GetAppFileList(appListCluster, 2) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s cluster apps to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec @@ -1164,52 +1242,53 @@ var _ = Describe("c3appfw test", func() { appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameCluster, s3TestDirCluster, appSourceClusterDefaultSpec)} appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - // Create Single site Cluster and SHC, with App Framework enabled on CM and SHC Deployer - testenvInstance.Log.Info("Create Single site Indexer Cluster with Local and Cluster scope for apps") + // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer + testenvInstance.Log.Info("Create Single site Indexer Cluster with Local and Cluster scope") indexerReplicas := 3 shReplicas := 3 err = deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the CM goes to Ready phase + // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(deployment, testenvInstance) // Ensure Indexers go to Ready phase testenv.SingleSiteIndexersReady(deployment, testenvInstance) - // Ensure SHC go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(deployment, testenvInstance) // Verify RF SF is met testenv.VerifyRFSFMet(deployment, testenvInstance) - // Verify apps with local scope are downloaded by init-container - appVersion := "V2" + //############# INITIAL VERIFICATION ########## + // Verify V2 apps with local scope are downloaded initContDownloadLocation := "/init-apps/" + appSourceNameLocal downloadPodNames := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName()), fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} appFileList = testenv.GetAppFileList(appListLocal, 2) - testenvInstance.Log.Info("Verify apps are downloaded by init container", "version", appVersion, "App List", appFileList) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps with local scope are downloaded (App list: %s)", appVersion, appFileList)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), downloadPodNames, appFileList, initContDownloadLocation) - // Verify apps with cluster scope are downloaded by init-container + // Verify V2 apps with cluster scope are downloaded initContDownloadLocation = "/init-apps/" + appSourceNameCluster appFileList = testenv.GetAppFileList(appListCluster, 2) - testenvInstance.Log.Info("Verify apps are downloaded by init container for apps", "version", appVersion, "App List", appFileList) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps with cluster scope are downloaded (App list: %s)", appVersion, appFileList)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), downloadPodNames, appFileList, initContDownloadLocation) // Verify bundle push status + testenvInstance.Log.Info(fmt.Sprintf("Verify bundle push status (%s apps)", appVersion)) testenv.VerifyClusterManagerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), indexerReplicas, "") testenv.VerifyDeployerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), shReplicas) // Saving current V2 bundle hash for future comparison clusterManagerBundleHash := testenv.GetClusterManagerBundleHash(deployment) - // Verify apps with local scope are installed locally on CM and on SHC Deployer + // Verify apps with local scope are installed locally on Cluster Manager and on Deployer localPodNames := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName()), fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - testenvInstance.Log.Info("Verify apps are installed Locally on CM and Deployer by running Splunk CLI commands for app", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps with local scope are installed locally on Cluster Manager and Deployer", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), localPodNames, appListLocal, true, "enabled", true, false) - // Verify apps with cluster scope are installed on indexers + // Verify apps with cluster scope are installed on Indexers clusterPodNames := []string{} for i := 0; i < int(indexerReplicas); i++ { sh := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) @@ -1220,10 +1299,12 @@ var _ = Describe("c3appfw test", func() { sh := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), i) clusterPodNames = append(clusterPodNames, string(sh)) } - testenvInstance.Log.Info("Verify apps are installed clusterwide on indexers and search-heads by running Splunk CLI commands for app", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps with cluster scope are installed on Indexers and Search Heads", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), clusterPodNames, appListCluster, true, "enabled", true, true) + //############# DOWNGRADE APPS ################ // Delete apps on S3 + testenvInstance.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) uploadedApps = nil @@ -1231,56 +1312,59 @@ var _ = Describe("c3appfw test", func() { appListLocal = appListV1[len(appListV1)/2:] appListCluster = appListV1[:len(appListV1)/2] - // Upload older version of apps to S3 + // Upload appListLocal list of V1 apps to S3 (to be used for local install) appVersion = "V1" + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for local install (local scope)", appVersion)) appFileList = testenv.GetAppFileList(appListLocal, 1) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for local install to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload apps to a 2nd directory on S3 bucket as we need 2 buckets locations for this test.(appListLocal apps to be used for local install, appListCluster apps for cluster install) + // Upload appListCluster list of V1 apps to S3 (to be used for cluster-wide install) clusterappFileList = testenv.GetAppFileList(appListCluster, 1) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps for cluster-wide install to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Wait for the poll period for the apps to be downloaded time.Sleep(2 * time.Minute) - // Ensure that the CM goes to Ready phase + // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(deployment, testenvInstance) // Ensure Indexers go to Ready phase testenv.SingleSiteIndexersReady(deployment, testenvInstance) - // Ensure SHC go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(deployment, testenvInstance) // Verify RF SF is met testenv.VerifyRFSFMet(deployment, testenvInstance) - // Verify apps with local scope are downloaded by init-container + //########## DOWNGRADE VERIFICATION ########### + // Verify apps with local scope are downloaded initContDownloadLocation = "/init-apps/" + appSourceNameLocal appFileList = testenv.GetAppFileList(appListLocal, 1) - testenvInstance.Log.Info("Verify apps are downloaded by init container for apps", "version", appVersion, "App List", appFileList) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps with local scope are downloaded (App list: %s)", appVersion, appFileList)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), downloadPodNames, appFileList, initContDownloadLocation) - // Verify apps with cluster scope are downloaded by init-container + // Verify apps with cluster scope are downloaded initContDownloadLocation = "/init-apps/" + appSourceNameCluster appFileList = testenv.GetAppFileList(appListCluster, 1) - testenvInstance.Log.Info("Verify apps are downloaded by init container for apps", "version", appVersion, "App List", appFileList) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps with cluster scope are downloaded (App list: %s)", appVersion, appFileList)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), downloadPodNames, appFileList, initContDownloadLocation) // Verify bundle push status + testenvInstance.Log.Info(fmt.Sprintf("Verify bundle push status (%s apps)", appVersion)) testenv.VerifyClusterManagerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), indexerReplicas, clusterManagerBundleHash) testenv.VerifyDeployerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), shReplicas) - // Verify apps with local scope are downgraded locally on CM and on SHC Deployer - testenvInstance.Log.Info("Verify apps are downgraded locally on CM and Deployer", "version", appVersion) + // Verify apps with local scope are downgraded locally on Cluster Manager and on Deployer + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps with local scope are downgraded locally on Cluster Manager and Deployer", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), localPodNames, appListLocal, true, "enabled", false, false) - // Verify apps with cluster scope are downgraded on indexers - testenvInstance.Log.Info("Verify apps are downgraded cluster-wide on indexers and search-heads", "version", appVersion) + // Verify apps with cluster scope are downgraded on Indexers + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps with cluster scope are downgraded on Indexers and Search Heads", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), clusterPodNames, appListCluster, true, "enabled", false, true) }) }) @@ -1290,19 +1374,21 @@ var _ = Describe("c3appfw test", func() { /* Test Steps ################## SETUP #################### - * Create App Source for C3 SVA (CM and SHC Deployer) + * Create App Source for C3 SVA (Cluster Manager and Deployer) * Add more apps than usual on S3 for this test * Prepare and deploy C3 CRD with app framework and wait for pods to be ready ############### VERIFICATIONS ############### * Verify bundle push is successful - * Verify apps are copied, installed on SH and Indexers pods + * Verify apps are copied, installed on Search Heads and Indexers pods */ + //################## SETUP #################### // Creating a bigger list of apps to be installed than the default one appList := []string{"splunk_app_db_connect", "splunk_app_aws", "Splunk_TA_microsoft-cloudservices", "Splunk_ML_Toolkit", "Splunk_Security_Essentials"} appFileList := testenv.GetAppFileList(appList, 1) - // Download App from S3 + // Download apps from S3 + testenvInstance.Log.Info("Download bigger amount of apps from S3 for this test") err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) Expect(err).To(Succeed(), "Unable to download apps files") @@ -1310,16 +1396,16 @@ var _ = Describe("c3appfw test", func() { appList = append(appListV1, appList...) appFileList = testenv.GetAppFileList(appList, 1) - // Upload app to S3 for IDXC + // Upload app to S3 for Indexer Cluster s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory for Indexer Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload app to S3 for SHC + // Upload app to S3 for Search Head Cluster s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory for Search Head Cluster") uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec @@ -1330,26 +1416,26 @@ var _ = Describe("c3appfw test", func() { appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(testenvInstance, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(testenvInstance, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) - // Create Single site Cluster and SHC, with App Framework enabled on CM and SHC Deployer - testenvInstance.Log.Info("Create Single site Indexer Cluster and SHC with App framework") + // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer + testenvInstance.Log.Info("Create Single Site Indexer Cluster and Search Head Cluster") indexerReplicas := 3 shReplicas := 3 err = deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - // Ensure that the CM goes to Ready phase + // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(deployment, testenvInstance) // Ensure Indexers go to Ready phase testenv.SingleSiteIndexersReady(deployment, testenvInstance) - // Ensure SHC go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(deployment, testenvInstance) // Verify RF SF is met testenv.VerifyRFSFMet(deployment, testenvInstance) - // Verify apps are downloaded by init-container + // Verify apps are downloaded initContDownloadLocationIdxc := "/init-apps/" + appSourceNameIdxc initContDownloadLocationShc := "/init-apps/" + appSourceNameShc managerPodNames := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName()), fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} @@ -1357,43 +1443,45 @@ var _ = Describe("c3appfw test", func() { testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())}, appFileList, initContDownloadLocationShc) // Verify bundle push status + testenvInstance.Log.Info("Verify bundle push status") testenv.VerifyClusterManagerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), indexerReplicas, "") testenv.VerifyDeployerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), shReplicas) // Verify apps are copied to location allPodNames := testenv.DumpGetPods(testenvInstance.GetName()) - testenvInstance.Log.Info("Verify apps are copied to correct location based on Pod KIND") + testenvInstance.Log.Info("Verify apps are copied to correct location") testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV1, true, true) - // Verify apps are not copied in /etc/apps/ on CM and on Deployer (therefore not installed on Deployer and on CM) - testenvInstance.Log.Info("Verify apps are NOT copied to /etc/apps on CM and Deployer") + // Verify apps are not copied in /etc/apps/ on Cluster Manager and on Deployer (therefore not installed on Deployer and on Cluster Manager) + testenvInstance.Log.Info("Verify apps are NOT copied to /etc/apps on Cluster Manager and Deployer") testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), managerPodNames, appListV1, false, false) - // Get indexers and SH pod names + // Get Indexers and Search Heads pod names podNames := []string{} podNames = append(podNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) podNames = append(podNames, testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1)...) - // Verify apps are installed on indexers and SH - testenvInstance.Log.Info("Verify apps are installed on the Indexers and Search Heads") + // Verify apps are installed on Indexers and Search Heads + testenvInstance.Log.Info("Verify apps are installed on Indexers and Search Heads") testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), podNames, appListV1, true, "enabled", false, true) }) }) - Context("Single Site Indexer Cluster with SHC (C3) with App Framework", func() { + Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { It("integration, c3, appframework: can deploy a C3 SVA with App Framework enabled for manual update", func() { - // Upload V1 apps to S3 for IDXC + // Upload V1 apps to S3 for Indexer Cluster + appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1, 1) s3TestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload V1 apps to S3 for SHC + // Upload V1 apps to S3 for Search Head Cluster s3TestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec @@ -1404,79 +1492,78 @@ var _ = Describe("c3appfw test", func() { appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(testenvInstance, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 0) appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(testenvInstance, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 0) - // Create Single site Cluster and SHC, with App Framework enabled on CM and SHC Deployer + // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer indexerReplicas := 3 testenvInstance.Log.Info("Deploy Single Site Indexer Cluster") err = deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") - // Ensure that the cluster-master goes to Ready phase + // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(deployment, testenvInstance) - // Ensure indexers go to Ready phase + // Ensure Indexers go to Ready phase testenv.SingleSiteIndexersReady(deployment, testenvInstance) - // Ensure search head cluster go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(deployment, testenvInstance) // Verify RF SF is met testenv.VerifyRFSFMet(deployment, testenvInstance) - // Verify Apps are downloaded by init-container - appVersion := "V1" + // Verify apps are downloaded by init-container initContDownloadLocationIdxc := "/init-apps/" + appSourceNameIdxc initContDownloadLocationShc := "/init-apps/" + appSourceNameShc - testenvInstance.Log.Info("Verify Apps are downloaded by init container for apps", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded by init container for apps", appVersion)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appFileList, initContDownloadLocationIdxc) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())}, appFileList, initContDownloadLocationShc) - //Verify Apps are copied to location + // Verify apps are copied to location allPodNames := testenv.DumpGetPods(testenvInstance.GetName()) - testenvInstance.Log.Info("Verify Apps are copied to correct location based on Pod KIND for app", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to correct location based on Pod KIND", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV1, true, true) - // Verify apps are not copied in /etc/apps/ on CM and on Deployer (therefore not installed on Deployer and on CM) - testenvInstance.Log.Info("Verify Apps are NOT copied to /etc/apps on CM and Deployer for app", "verison", appVersion, "App List", appFileList) + // Verify apps are not copied in /etc/apps/ on Cluster Manager and on Deployer (therefore not installed on Deployer and on Cluster Manager) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are NOT copied to /etc/apps on Cluster Manager and Deployer for app, (App List: %s)", appVersion, appFileList)) masterPodNames := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName()), fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), masterPodNames, appListV1, false, false) - //Verify Apps are installed - testenvInstance.Log.Info("Verify Apps are installed on the pods by running Splunk CLI commands for app", "version", appVersion) + // Verify apps are installed + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are installed on the pods", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV1, true, "enabled", false, true) - //Delete apps on S3 for new Apps - testenvInstance.Log.Info("Delete Apps on S3 for", "Version", appVersion) + // Delete V1 apps on S3 + testenvInstance.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) uploadedApps = nil - // Upload newer version of apps (V2) to S3 for C3 + // Upload V2 apps to S3 for C3 appVersion = "V2" appFileList = testenv.GetAppFileList(appListV2, 2) - testenvInstance.Log.Info("Uploading apps S3 for", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Uploading %s apps to S3", appVersion)) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory for C3") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory for C3") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Search Head Cluster", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Wait for the poll period for the apps to be downloaded time.Sleep(2 * time.Minute) - // Ensure that the cluster-master goes to Ready phase + // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(deployment, testenvInstance) - // Ensure indexers go to Ready phase + // Ensure Indexers go to Ready phase testenv.SingleSiteIndexersReady(deployment, testenvInstance) - // Ensure search head cluster go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(deployment, testenvInstance) // Verify RF SF is met testenv.VerifyRFSFMet(deployment, testenvInstance) - //Verify Apps are not updated - testenvInstance.Log.Info("Verify Apps are not updated by running Splunk CLI commands for app", "version", appVersion) + // Verify apps are not updated + testenvInstance.Log.Info(fmt.Sprintf("Verify %s are not updated", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV1, true, "enabled", false, true) testenvInstance.Log.Info("Get config map for triggering manual update") @@ -1489,22 +1576,22 @@ var _ = Describe("c3appfw test", func() { err = deployment.UpdateCR(config) Expect(err).To(Succeed(), "Unable to update config map") - // Ensure standalone is updating + // Ensure Standalone is updating testenv.VerifyClusterManagerPhase(deployment, testenvInstance, splcommon.PhaseUpdating) - // Ensure that the cluster-master goes to Ready phase + // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(deployment, testenvInstance) - // Ensure indexers go to Ready phase + // Ensure Indexers go to Ready phase testenv.SingleSiteIndexersReady(deployment, testenvInstance) - // Ensure search head cluster go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(deployment, testenvInstance) // Verify RF SF is met testenv.VerifyRFSFMet(deployment, testenvInstance) - //Verify config map set back to off after poll trigger + // Verify config map set back to off after poll trigger testenvInstance.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) config, _ = testenv.GetAppframeworkManualUpdateConfigMap(deployment, testenvInstance.GetName()) Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") @@ -1514,16 +1601,16 @@ var _ = Describe("c3appfw test", func() { testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appFileList, initContDownloadLocationIdxc) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())}, appFileList, initContDownloadLocationShc) - //Verify Apps are copied to location - testenvInstance.Log.Info("Verify Apps are copied to correct location based on Pod KIND for app", "version", appVersion) + // Verify apps are copied to location + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to correct location based on Pod KIND", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV2, true, true) - // Verify apps are not copied in /etc/apps/ on CM and on Deployer (therefore not installed on Deployer and on CM) - testenvInstance.Log.Info("Verify Apps are NOT copied to /etc/apps on CM and Deployer for app", "verison", appVersion) + // Verify apps are not copied in /etc/apps/ on Cluster Manager and on Deployer (therefore not installed on Deployer and on Cluster Manager) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are NOT copied to /etc/apps on Cluster Manager and Deployer", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), masterPodNames, appListV2, false, false) - //Verify Apps are updated - testenvInstance.Log.Info("Verify Apps are installed on the pods by running Splunk CLI commands for app", "version", appVersion) + // Verify apps are updated + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are installed on the pods", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV2, true, "enabled", true, true) }) }) diff --git a/test/m4/appframework/appframework_test.go b/test/m4/appframework/appframework_test.go index 0b6b8e913..c6891ec5f 100644 --- a/test/m4/appframework/appframework_test.go +++ b/test/m4/appframework/appframework_test.go @@ -38,7 +38,6 @@ var _ = Describe("m4appfw test", func() { var err error deployment, err = testenvInstance.NewDeployment(testenv.RandomDNSName(3)) Expect(err).To(Succeed(), "Unable to create deployment") - }) AfterEach(func() { @@ -55,33 +54,38 @@ var _ = Describe("m4appfw test", func() { } }) - Context("Multi Site Indexer Cluster with SHC (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { It("smoke, m4, appframework: can deploy a M4 SVA with App Framework enabled, install apps and upgrade them", func() { /* Test Steps ################## SETUP ################## - * Create 2 App Sources for MC (Monitoring Console) and M4 SVA (CM and SHC Deployer) - * Prepare and deploy MC CRD with app framework and wait for pod to be ready - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ############### VERIFICATION ############## + * Upload V1 apps to S3 for Monitoring Console + * Create app source for Monitoring Console + * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready + * Upload V1 apps to S3 for Indexer Cluster and Search Head Cluster + * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready + ########## INITIAL VERIFICATIONS ########## * Verify bundle push is successful - * Verify apps are copied and installed on MC and on SH and Indexers pods - ############### UPGRADE APPS ############## + * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods + ############# UPGRADE APPS ################ * Upgrade apps in app sources - * Wait for MC and M4 pod to be ready - ############### VERIFICATION ############## + * Wait for Monitoring Console and M4 pod to be ready + ########## UPGRADE VERIFICATIONS ########## * Verify bundle push is successful - * Verify apps are copied and upgraded on MC and on SH and Indexers pods + * Verify apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods */ - // Upload older version of apps (V1) to S3 for MC + //################## SETUP ################## + // Upload V1 apps to S3 for Monitoring Console + appVersion := "V1" appFileList := testenv.GetAppFileList(appListV1, 1) + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps for MC to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for MC + // Create App framework Spec for Monitoring Console volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) volumeSpecMC := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volumeNameMC, testenv.GetS3Endpoint(), testenvInstance.GetIndexSecretName(), "aws", "s3")} appSourceDefaultSpecMC := enterpriseApi.AppSourceDefaultSpec{ @@ -106,22 +110,23 @@ var _ = Describe("m4appfw test", func() { AppFrameworkConfig: appFrameworkSpecMC, } - // Deploy MC CRD + // Deploy Monitoring Console testenvInstance.Log.Info("Deploy Monitoring Console") mcName := deployment.GetName() mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(testenvInstance.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - // Verify MC is ready and stays in ready state + // Verify Monitoring Console is ready and stays in ready state testenv.VerifyMonitoringConsoleReady(deployment, deployment.GetName(), mc, testenvInstance) // Upload V1 apps to S3 for M4 SVA + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for M4", appVersion)) s3TestDir = "m4appfw-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for M4", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec + // Create App framework Spec for M4 volumeName := "appframework-test-volume-" + testenv.RandomDNSName(3) volumeSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volumeName, testenv.GetS3Endpoint(), testenvInstance.GetIndexSecretName(), "aws", "s3")} appSourceDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -138,122 +143,125 @@ var _ = Describe("m4appfw test", func() { } // Deploy M4 CRD - testenvInstance.Log.Info("Deploy Multisite Indexer Cluster with SHC") + testenvInstance.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") siteCount := 3 - indexersPerSite := 1 shReplicas := 3 + indexersPerSite := 1 err = deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpec, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster and SHC with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - // Ensure that the CM goes to Ready phase + // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(deployment, testenvInstance) - // Ensure the indexers of all sites go to Ready phase + // Ensure the Indexers of all sites go to Ready phase testenv.IndexersReady(deployment, testenvInstance, siteCount) - // Ensure cluster configured as multisite + // Ensure Indexer Cluster configured as Multisite testenv.IndexerClusterMultisiteStatus(deployment, testenvInstance, siteCount) - // Ensure SHC go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(deployment, testenvInstance) // Verify RF SF is met testenv.VerifyRFSFMet(deployment, testenvInstance) - // Verify MC is Ready and stays in ready state + // Verify Monitoring Console is ready and stays in ready state testenv.VerifyMonitoringConsoleReady(deployment, deployment.GetName(), mc, testenvInstance) - // Verify apps are downloaded by init-container on CM, Deployer - appVersion := "V1" + //########## INITIAL VERIFICATIONS ########## + // Verify V1 apps are downloaded on Cluster Manager and Deployer podNames := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName()), fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} initContDownloadLocation := "/init-apps/" + appSourceName - testenvInstance.Log.Info("Verify V1 apps are downloaded by init container for apps", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded on Cluster Manager and Deployer", appVersion)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), podNames, appFileList, initContDownloadLocation) - // Verify apps downloaded by init-container on Monitoring Console + // Verify V1 apps are downloaded on Monitoring Console initContDownloadLocationMCPod := "/init-apps/" + appSourceNameMC mcPodName := fmt.Sprintf(testenv.MonitoringConsolePod, mcName, 0) - testenvInstance.Log.Info("Verify V1 apps are downloaded by init container on MC for apps", "POD", mcPodName, "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded on Monitoring Console pod %s", appVersion, mcPodName)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{mcPodName}, appFileList, initContDownloadLocationMCPod) // Verify bundle push status + testenvInstance.Log.Info(fmt.Sprintf("Verify bundle push status (%s apps)", appVersion)) testenv.VerifyClusterManagerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), siteCount, "") testenv.VerifyDeployerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), shReplicas) // Saving current V1 bundle hash for future comparison clusterManagerBundleHash := testenv.GetClusterManagerBundleHash(deployment) - // Add SHC and Indexer Pods to all Pod Names + // Add Search Head Cluster and Indexer Pods to all Pod Names allPodNames := podNames allPodNames = append(allPodNames, testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount)...) allPodNames = append(allPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) // Verify Apps are copied to correct location for M4 SVA - testenvInstance.Log.Info("Verify apps are copied to correct location for M4 SVA", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to correct location for M4", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV1, true, true) - // Verify Apps are copied to correct location for MC - testenvInstance.Log.Info("Verify apps are copied to correct location for Monitoring Console", "version", appVersion) + // Verify Apps are copied to correct location for Monitoring Console + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to correct location for Monitoring Console", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), []string{mcPodName}, appListV1, true, false) - // Verify apps are not copied in /etc/apps/ on CM and on Deployer (therefore not installed on Deployer and on CM) - testenvInstance.Log.Info("Verify apps are NOT copied to /etc/apps on CM and Deployer for app", "version", appVersion, "App List", appFileList) + // Verify apps are not copied in /etc/apps/ on Cluster Manager and on Deployer (therefore not installed on Deployer and on Cluster Manager) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are NOT copied to /etc/apps on Cluster Manager and Deployer (App list: %s)", appVersion, appFileList)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), podNames, appListV1, false, false) - // Verify apps are installed on M4(cluster-wide) - testenvInstance.Log.Info("Verify apps are installed on the pods by running Splunk CLI commands for app", "version", appVersion) + // Verify apps are installed on M4 (cluster-wide) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are installed on M4 pods (cluster-wide)", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV1, true, "enabled", false, true) - // Verify apps are installed on MC (local install) - testenvInstance.Log.Info("Verify apps are installed on the pods by running Splunk CLI commands for app", "version", appVersion) + // Verify apps are installed on Monitoring Console (local install) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are installed on Monitoring Console pod (local)", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), []string{mcPodName}, appListV1, true, "enabled", false, false) + //############# UPGRADE APPS ################ // Delete apps on S3 - testenvInstance.Log.Info("Delete apps on S3 for", "Version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) uploadedApps = nil - testenvInstance.Log.Info("Testing upgrade scenario") - // Upload newer version of apps to S3 for M4 SVA - appFileList = testenv.GetAppFileList(appListV2, 2) + // Upload V2 apps to S3 for M4 appVersion = "V2" - testenvInstance.Log.Info("Uploading apps S3 for", "version", appVersion) + appFileList = testenv.GetAppFileList(appListV2, 2) + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for M4", appVersion)) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV2) - Expect(err).To(Succeed(), "Unable to upload newer version of apps for M4 to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for M4", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload new version of apps for Monitoring Console + // Upload V2 apps for Monitoring Console + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), "Unable to upload newer version of apps for MC to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Wait for the poll period for the apps to be downloaded time.Sleep(2 * time.Minute) - // Ensure that the cluster-manager goes to Ready phase + // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(deployment, testenvInstance) - // Ensure the indexers of all sites go to Ready phase + // Ensure the Indexers of all sites go to Ready phase testenv.IndexersReady(deployment, testenvInstance, siteCount) - // Ensure cluster configured as multisite + // Ensure cluster configured as Multisite testenv.IndexerClusterMultisiteStatus(deployment, testenvInstance, siteCount) - // Ensure SHC go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(deployment, testenvInstance) // Verify RF SF is met testenv.VerifyRFSFMet(deployment, testenvInstance) - // Verify MC is Ready and stays in ready state + // Verify Monitoring Console is Ready and stays in ready state testenv.VerifyMonitoringConsoleReady(deployment, deployment.GetName(), mc, testenvInstance) - // Verify apps are downloaded by init-container for M4 SVA - testenvInstance.Log.Info("Verify V2 apps are downloaded by init container for M4 for apps", "version", appVersion) + //########## UPGRADE VERIFICATIONS ########## + // Verify V2 apps are downloaded on Cluster Manager and Deployer + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded on Cluster Manager and Deployer", appVersion)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), podNames, appFileList, initContDownloadLocation) - // Verify apps downloaded by init-container for Monitoring Console - testenvInstance.Log.Info("Verify V2 apps are downloaded by init container for MC for apps", "POD", mcPodName, "version", appVersion) + // Verify V2 apps are downloaded on Monitoring Console + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded on Monitoring Console pod %s", appVersion, mcPodName)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{mcPodName}, appFileList, initContDownloadLocationMCPod) // Verify bundle push status and compare bundle hash with previous V1 bundle @@ -261,61 +269,59 @@ var _ = Describe("m4appfw test", func() { testenv.VerifyDeployerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), shReplicas) // Verify apps are copied to location for M4 SVA - testenvInstance.Log.Info("Verify apps are copied to correct location based on Pod KIND for app", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to correct location on M4 pods", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV2, true, true) // Verify apps are copied to location for Monitoring Console - testenvInstance.Log.Info("Verify apps are copied to correct location for Monitoring Console", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to correct location on Monitoring Console", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), []string{mcPodName}, appListV2, true, false) - // Verify apps are not copied in /etc/apps/ on CM and on Deployer (therefore not installed on Deployer and on CM) - testenvInstance.Log.Info("Verify apps are NOT copied to /etc/apps on CM and Deployer for app", "version", appVersion, "App List", appFileList) + // Verify apps are not copied in /etc/apps/ on Cluster Manager and on Deployer (therefore not installed on Deployer and on Cluster Manager) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are NOT copied to /etc/apps on Cluster Manager and Deployer", appFileList)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), podNames, appListV2, false, false) - // Verify apps are installed on M4(cluster-wide) - testenvInstance.Log.Info("Verify apps are installed on the pods by running Splunk CLI commands for app", "version", appVersion) + // Verify apps are updated on M4(cluster-wide) + testenvInstance.Log.Info(fmt.Sprintf("Verify apps have been updated to %s on M4 pods", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV2, true, "enabled", true, true) - // Verify apps are installed on MC (local install) - testenvInstance.Log.Info("Verify apps are installed on the pods by running Splunk CLI commands for app", "version", appVersion) + // Verify apps are updated on Monitoring Console (local install) + testenvInstance.Log.Info(fmt.Sprintf("Verify apps have been updated to %s on Monitoring Console", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), []string{mcPodName}, appListV2, true, "enabled", true, false) }) }) - Context("Multi Site Indexer Cluster with SHC (m4) with App Framework", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled, install apps and downgrade them", func() { /* Test Steps ################## SETUP ################## - * Create 2 App Sources for MC and M4 SVA (CM and SHC Deployer) - * Prepare and deploy MC CRD with app config and wait for pod to be ready - * Prepare and deploy M4 CRD with app config and wait for pods to be ready - ############### VERIFICATION ############## + * Upload V2 apps to S3 for Monitoring Console + * Create app source for Monitoring Console + * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready + * Upload V2 apps to S3 for Indexer Cluster and Search Head Cluster + * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready + ########## INITIAL VERIFICATIONS ########## * Verify bundle push is successful - * Verify apps are copied and installed on MC and on SH and Indexers pods - ############### DOWNGRADE APPS ############ + * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods + ############ DOWNGRADE APPS ############### * Downgrade apps in app sources - * Wait for MC and M4 to be ready - ############### VERIFICATION ############## + * Wait for Monitoring Console and M4 to be ready + ########## DOWNGRADE VERIFICATIONS ######## * Verify bundle push is successful - * Verify apps are copied and downgraded on MC and on SH and Indexers pods + * Verify apps are copied and downgraded on Monitoring Console and on Search Heads and Indexers pods */ - // Upload V2 version of apps to S3 for M4 SVA + //################## SETUP ################## + // Upload V2 version of apps to S3 for Monitoring Console appVersion := "V2" - s3TestDir = "m4appfw-" + testenv.RandomDNSName(4) appFileList := testenv.GetAppFileList(appListV2, 2) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV2) - Expect(err).To(Succeed(), "Unable to upload newer version of apps for M4 to S3 test directory") - - // Upload V2 verison of apps to S3 for Monitoring Console - uploadedApps = append(uploadedApps, uploadedFiles...) + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), "Unable to upload newer version of apps for MC to S3 test directory") + uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV2) + Expect(err).To(Succeed(), "Unable to upload newer version of apps for Monitoring Console to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for MC + // Create App framework Spec for Monitoring Console volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) volumeSpecMC := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volumeNameMC, testenv.GetS3Endpoint(), testenvInstance.GetIndexSecretName(), "aws", "s3")} appSourceDefaultSpecMC := enterpriseApi.AppSourceDefaultSpec{ @@ -340,15 +346,22 @@ var _ = Describe("m4appfw test", func() { AppFrameworkConfig: appFrameworkSpecMC, } - // Deploy MC CRD + // Deploy Monitoring Console testenvInstance.Log.Info("Deploy Monitoring Console") mcName := deployment.GetName() mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(testenvInstance.GetName(), mcName, mcSpec) Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - // Verify MC is ready and stays in ready state + // Verify Monitoring Console is ready and stays in ready state testenv.VerifyMonitoringConsoleReady(deployment, deployment.GetName(), mc, testenvInstance) + // Upload V2 version of apps to S3 for M4 + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for M4", appVersion)) + s3TestDir = "m4appfw-" + testenv.RandomDNSName(4) + uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV2) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for M4", appVersion)) + uploadedApps = append(uploadedApps, uploadedFiles...) + // Create App framework Spec for M4 volumeName := "appframework-test-volume-" + testenv.RandomDNSName(3) volumeSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volumeName, testenv.GetS3Endpoint(), testenvInstance.GetIndexSecretName(), "aws", "s3")} @@ -366,174 +379,187 @@ var _ = Describe("m4appfw test", func() { } // Deploy M4 CRD - testenvInstance.Log.Info("Deploy Multisite Indexer Cluster with SHC") + testenvInstance.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") siteCount := 3 shReplicas := 3 indexersPerSite := 1 err = deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpec, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster and SHC with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - // Ensure that the CM goes to Ready phase + // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(deployment, testenvInstance) - // Ensure the indexers of all sites go to Ready phase + // Ensure the Indexers of all sites go to Ready phase testenv.IndexersReady(deployment, testenvInstance, siteCount) - // Ensure cluster configured as multisite + // Ensure cluster configured as Multisite testenv.IndexerClusterMultisiteStatus(deployment, testenvInstance, siteCount) - // Ensure SHC go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(deployment, testenvInstance) // Verify RF SF is met testenv.VerifyRFSFMet(deployment, testenvInstance) - // Verify MC is Ready and stays in ready state + // Verify Monitoring Console is Ready and stays in ready state testenv.VerifyMonitoringConsoleReady(deployment, deployment.GetName(), mc, testenvInstance) - // Verify apps are downloaded by init-container on CM, Deployer + //########## INITIAL VERIFICATIONS ########## + // Verify V2 apps are downloaded on Cluster Manager and Deployer initContDownloadLocation := "/init-apps/" + appSourceName podNames := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName()), fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - testenvInstance.Log.Info("Verify apps are downloaded by init container for apps", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded for M4", appVersion)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), podNames, appFileList, initContDownloadLocation) - // Verify apps are downloaded by init-container on Monitoring Console + // Verify V2 apps are downloaded on Monitoring Console initContDownloadLocationMCPod := "/init-apps/" + appSourceNameMC mcPodName := fmt.Sprintf(testenv.MonitoringConsolePod, mcName, 0) - testenvInstance.Log.Info("Verify apps are downloaded by init container on MC pod for apps", "POD", mcPodName, "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded for Monitoring Console POD %s", appVersion, mcPodName)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{mcPodName}, appFileList, initContDownloadLocationMCPod) // Verify bundle push status + testenvInstance.Log.Info(fmt.Sprintf("Verify bundle push status (%s apps)", appVersion)) testenv.VerifyClusterManagerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), siteCount, "") testenv.VerifyDeployerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), shReplicas) // Saving current V1 bundle hash for future comparison clusterManagerBundleHash := testenv.GetClusterManagerBundleHash(deployment) - // Verify apps are copied to location - // Add SHC and Indexer Pods to all Pod Names + // Add Search Head Cluster and Indexer Pods to all Pod Names allPodNames := podNames allPodNames = append(allPodNames, testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount)...) allPodNames = append(allPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) - testenvInstance.Log.Info("Verify apps are copied to correct location based on Pod KIND for app", "version", appVersion) + + // Verify V2 apps are copied to location on M4 + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to correct location on M4", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV2, true, true) - // Verify Apps are copied to correct location for MC - testenvInstance.Log.Info("Verify apps are copied to correct location for Monitoring Console", "version", appVersion) + // Verify V2 apps are copied to correct location for Monitoring Console + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to correct location for Monitoring Console", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), []string{mcPodName}, appListV2, true, false) - // Verify apps are not copied in /etc/apps/ on CM and on Deployer (therefore not installed on Deployer and on CM) - testenvInstance.Log.Info("Verify apps are NOT copied to /etc/apps on CM and Deployer for app", "version", appVersion, "App List", appFileList) + // Verify V2 apps are not copied in /etc/apps/ on Cluster Manager and on Deployer (therefore not installed on Deployer and on Cluster Manager) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are NOT copied to /etc/apps on Cluster Manager and Deployer (App list: %s)", appVersion, appFileList)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), podNames, appListV2, false, false) - // Verify apps are installed on (cluster-wide) - testenvInstance.Log.Info("Verify newer version of apps are installed on the pods", "version", appVersion) + // Verify V2 apps are installed on (cluster-wide) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are installed on the pods", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV2, true, "enabled", true, true) - // Verify apps are installed on MC (local install) - testenvInstance.Log.Info("Verify apps are installed on the pods by running Splunk CLI commands for app", "version", appVersion) + // Verify apps are installed on Monitoring Console (local install) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are installed on Monitoring Console pod", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), []string{mcPodName}, appListV2, true, "enabled", true, false) - // Delete apps on S3 - testenvInstance.Log.Info("Delete apps on S3 for", "Version", appVersion) + //############# DOWNGRADE APPS ################ + // Delete V2 apps on S3 + testenvInstance.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) uploadedApps = nil - // Upload older version of apps (V1) to S3 for M4 + // Upload V1 apps to S3 for M4 appVersion = "V1" + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for M4", appVersion)) appFileList = testenv.GetAppFileList(appListV1, 1) - testenvInstance.Log.Info("Uploading apps S3 for", "version", appVersion) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload older version of apps to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for M4", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Upload older version of apps (V1) to S3 for Monitoring Console + // Upload V1 apps to S3 for Monitoring Console + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload older version of apps to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Wait for the poll period for the apps to be downloaded time.Sleep(2 * time.Minute) - // Ensure that the CM goes to Ready phase + // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(deployment, testenvInstance) - // Ensure the indexers of all sites go to Ready phase + // Ensure the Indexers of all sites go to Ready phase testenv.IndexersReady(deployment, testenvInstance, siteCount) - // Ensure cluster configured as multisite + // Ensure cluster configured as Multisite testenv.IndexerClusterMultisiteStatus(deployment, testenvInstance, siteCount) - // Ensure SHC go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(deployment, testenvInstance) // Verify RF SF is met testenv.VerifyRFSFMet(deployment, testenvInstance) - // Verify MC is Ready and stays in ready state + // Verify Monitoring Console is Ready and stays in ready state testenv.VerifyMonitoringConsoleReady(deployment, deployment.GetName(), mc, testenvInstance) - // Verify apps are downloaded by init-container - testenvInstance.Log.Info("Verify apps are downloaded by init container for M4 for apps", " version", appVersion) + //########## DOWNGRADE VERIFICATIONS ######## + // Verify V1 apps are downloaded on Cluster Manager and Deployer + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded on Cluster Manager and Deployer", appVersion)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), podNames, appFileList, initContDownloadLocation) - testenvInstance.Log.Info("Verify apps are downloaded by init container on MC pod for apps", "POD", mcPodName, "version", appVersion) + + // Verify V1 apps are downloaded on Monitoring Console + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded on Monitoring Console pod %s", appVersion, mcPodName)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{mcPodName}, appFileList, initContDownloadLocationMCPod) // Verify bundle push status + testenvInstance.Log.Info(fmt.Sprintf("Verify bundle push status (%s apps)", appVersion)) testenv.VerifyClusterManagerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), siteCount, clusterManagerBundleHash) testenv.VerifyDeployerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), shReplicas) // Verify apps are copied to correct location for M4 SVA - testenvInstance.Log.Info("Verify apps are copied to correct location based on Pod KIND for app", " version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to correct location for M4", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV1, true, true) // Verify apps are copied to correct location for Monitoring Console - testenvInstance.Log.Info("Verify apps are copied to correct location for Monitoring Console", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to correct location for Monitoring Console", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), []string{mcPodName}, appListV2, true, false) - // Verify apps are not copied in /etc/apps/ on CM and on Deployer (therefore not installed on Deployer and on CM) - testenvInstance.Log.Info("Verify apps are NOT copied to /etc/apps on CM and Deployer for app", " version", appVersion) + // Verify apps are not copied in /etc/apps/ on Cluster Manager and on Deployer (therefore not installed on Deployer and on Cluster Manager) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are NOT copied to /etc/apps on Cluster Manager and Deployer", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), podNames, appListV1, false, false) - // Verify apps are downgraded on M4(cluster-wide) - testenvInstance.Log.Info("Verify apps are downgraded on the pods by running Splunk CLI commands for app", " version", appVersion) + // Verify apps are downgraded on M4 (cluster-wide) + testenvInstance.Log.Info(fmt.Sprintf("Verify apps have been downgraded to %s on the M4 pods", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV1, true, "enabled", false, true) - // Verify apps are installed on MC (local install) - testenvInstance.Log.Info("Verify apps are installed on the pods by running Splunk CLI commands for app", "version", appVersion) + // Verify apps are downgraded on Monitoring Console (local install) + testenvInstance.Log.Info(fmt.Sprintf("Verify apps have been downgraded to %s on Monitoring Console", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), []string{mcPodName}, appListV1, true, "enabled", false, false) }) }) - Context("Multi Site Indexer Cluster with SHC (m4) with App Framework", func() { - It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled, install apps, scale up IDXC and SHC, install apps on new pods, scale down", func() { + Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { + It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { /* Test Steps ################## SETUP ################## - * Create App Sources for M4 SVA (CM and SHC Deployer) + * Upload V1 apps to S3 for M4 + * Create app source for M4 SVA (Cluster Manager and Deployer) * Prepare and deploy M4 CRD with app config and wait for pods to be ready - ############### VERIFICATION ############## + ########### INITIAL VERIFICATIONS ######### * Verify bundle push is sucessful - * Verify apps are copied and installed on MC and also on SH and Indexers pods + * Verify apps are copied and installed on Monitoring Console and also on Search Heads and Indexers pods ############### SCALING UP ################ - * Scale up indexers and SHC - * Wait for M4 to be ready - ############### VERIFICATION ############## + * Scale up Indexers and Search Head Cluster + * Wait for Monitoring Console and M4 to be ready + ######### SCALING UP VERIFICATIONS ######## * Verify bundle push is sucessful - * Verify apps are copied and installed on new SH and Indexers pods - ############### SCALING DOWN ################ - * Scale down indexers and SHC - * Wait for M4 to be ready - ############### VERIFICATION ############## + * Verify apps are copied and installed on new Search Heads and Indexers pods + ############### SCALING DOWN ############## + * Scale down Indexers and Search Head Cluster + * Wait for Monitoring Console and M4 to be ready + ######### SCALING DOWN VERIFICATIONS ###### * Verify bundle push is sucessful - * Verify apps are still copied and installed on all SH and Indexers pods + * Verify apps are still copied and installed on all Search Heads and Indexers pods */ - // Upload V1 apps to S3 + //################## SETUP ################## + // Upload V1 apps to S3 for M4 + appVersion := "V1" + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for M4", appVersion)) appFileList := testenv.GetAppFileList(appListV1, 1) s3TestDir = "m4appfw-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for M4", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec for M4 @@ -553,33 +579,33 @@ var _ = Describe("m4appfw test", func() { } // Deploy M4 CRD - testenvInstance.Log.Info("Deploy Multisite Indexer Cluster with SHC") + testenvInstance.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") siteCount := 3 indexersPerSite := 1 shReplicas := 3 err = deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpec, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the CM goes to Ready phase + // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(deployment, testenvInstance) - // Ensure the indexers of all sites go to Ready phase + // Ensure the Indexers of all sites go to Ready phase testenv.IndexersReady(deployment, testenvInstance, siteCount) - // Ensure cluster configured as multisite + // Ensure cluster configured as Multisite testenv.IndexerClusterMultisiteStatus(deployment, testenvInstance, siteCount) - // Ensure SHC go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(deployment, testenvInstance) // Verify RF SF is met testenv.VerifyRFSFMet(deployment, testenvInstance) - // Verify apps are downloaded by init-container on CM, Deployer - appVersion := "V1" + //########### INITIAL VERIFICATIONS ######### + // Verify apps are downloaded on Cluster Manager and Deployer initContDownloadLocation := "/init-apps/" + appSourceName podNames := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName()), fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - testenvInstance.Log.Info("Verify apps are downloaded by init container on Cluster Master and Deployer for apps", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded on Cluster Master and Deployer", appVersion)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), podNames, appFileList, initContDownloadLocation) // Verify bundle push status @@ -588,35 +614,36 @@ var _ = Describe("m4appfw test", func() { // Verify apps are copied to correct location allPodNames := testenv.DumpGetPods(testenvInstance.GetName()) - testenvInstance.Log.Info("Verify apps are copied to correct location based on Pod KIND for app", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to correct location", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV1, true, true) - // Verify apps are not copied in /etc/apps/ on CM and on Deployer (therefore not installed on Deployer and on CM) + // Verify apps are not copied in /etc/apps/ on Cluster Manager and on Deployer (therefore not installed on Deployer and on Cluster Manager) managerPodNames := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName()), fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - testenvInstance.Log.Info("Verify apps are NOT copied to /etc/apps on CM and Deployer for app", "version", appVersion, "App List", appFileList) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are NOT copied to /etc/apps on Cluster Manager and Deployer (App list: %s)", appVersion, appFileList)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), managerPodNames, appListV1, false, false) - // Verify apps are installed on M4(cluster-wide) - testenvInstance.Log.Info("Verify apps are installed on the pods by running Splunk CLI commands for app", "version", appVersion) + // Verify apps are installed on Monitoring Console and M4(cluster-wide) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are installed cluster-wide on the pods", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV1, true, "enabled", false, true) - // Get instance of current SHC CR with latest config + //############### SCALING UP ################ + // Get instance of current Search Head Cluster CR with latest config shc := &enterpriseApi.SearchHeadCluster{} shcName := deployment.GetName() + "-shc" err = deployment.GetInstance(shcName, shc) Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - // Scale up SHC + // Scale up Search Head Cluster defaultSHReplicas := shc.Spec.Replicas scaledSHReplicas := defaultSHReplicas + 1 - testenvInstance.Log.Info("Scaling up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) + testenvInstance.Log.Info("Scale up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - // Update Replicas of SHC + // Update Replicas of Search Head Cluster shc.Spec.Replicas = int32(scaledSHReplicas) err = deployment.UpdateCR(shc) Expect(err).To(Succeed(), "Failed to scale up Search Head Cluster") - // Ensure SHC scales up and go to ScalingUp phase + // Ensure Search Head Cluster scales up and go to ScalingUp phase testenv.VerifySearchHeadClusterPhase(deployment, testenvInstance, splcommon.PhaseScalingUp) // Get instance of current Indexer CR with latest config @@ -626,7 +653,7 @@ var _ = Describe("m4appfw test", func() { Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") defaultIndexerReplicas := idxc.Spec.Replicas scaledIndexerReplicas := defaultIndexerReplicas + 1 - testenvInstance.Log.Info("Scaling up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) + testenvInstance.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) // Update Replicas of Indexer Cluster idxc.Spec.Replicas = int32(scaledIndexerReplicas) @@ -636,48 +663,51 @@ var _ = Describe("m4appfw test", func() { // Ensure Indexer cluster scales up and go to ScalingUp phase testenv.VerifyIndexerClusterPhase(deployment, testenvInstance, splcommon.PhaseScalingUp, idxcName) - // Ensure SHC go to Ready phase - testenv.SearchHeadClusterReady(deployment, testenvInstance) - // Ensure Indexer cluster go to Ready phase testenv.IndexersReady(deployment, testenvInstance, siteCount) + // Ensure Search Head Cluster go to Ready phase + testenv.SearchHeadClusterReady(deployment, testenvInstance) + // Verify RF SF is met testenv.VerifyRFSFMet(deployment, testenvInstance) - // Verify bundle push status + //######### SCALING UP VERIFICATIONS ######## + // Verify bundle push status. Bundle hash not compared as scaleup does not involve new config + testenvInstance.Log.Info("Verify bundle push status after scaling up") testenv.VerifyClusterManagerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), int(scaledIndexerReplicas), "") testenv.VerifyDeployerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), siteCount) - // Verify apps are copied to correct location + // Verify V1 apps are copied to correct location allPodNames = testenv.DumpGetPods(testenvInstance.GetName()) - testenvInstance.Log.Info("Verify apps are copied to correct location based on Pod KIND after scaling up of indexers and SH", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to correct location after scaling up of Indexers and Search Heads", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV1, true, true) - // Verify apps are not copied in /etc/apps/ on CM and on Deployer (therefore not installed on Deployer and on CM) - testenvInstance.Log.Info("Verify apps are NOT copied to /etc/apps on CM and Deployer after scaling up of indexers and SH", "version", appVersion) + // Verify V1 apps are not copied in /etc/apps/ on Cluster Manager and on Deployer (therefore not installed on Deployer and on Cluster Manager) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are NOT copied to /etc/apps on Cluster Manager and Deployer after scaling up of Indexers and Search Heads", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), managerPodNames, appListV1, false, false) - // Verify apps are installed cluster-wide - testenvInstance.Log.Info("Verify apps are installed on the pods by running Splunk CLI commands after scaling up of indexers and SH", "version", appVersion) + // Verify V1 apps are installed cluster-wide + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are installed on the pods after scaling up of Indexers and Search Heads", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV1, true, "enabled", false, true) - // Get instance of current SHC CR with latest config + //############### SCALING DOWN ############## + // Get instance of current Search Head Cluster CR with latest config shc = &enterpriseApi.SearchHeadCluster{} err = deployment.GetInstance(shcName, shc) Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - // Scale down SHC + // Scale down Search Head Cluster defaultSHReplicas = shc.Spec.Replicas scaledSHReplicas = defaultSHReplicas - 1 testenvInstance.Log.Info("Scaling down Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - // Update Replicas of SHC + // Update Replicas of Search Head Cluster shc.Spec.Replicas = int32(scaledSHReplicas) err = deployment.UpdateCR(shc) Expect(err).To(Succeed(), "Failed to scale down Search Head Cluster") - // Ensure SHC scales down and go to ScalingDown phase + // Ensure Search Head Cluster scales down and go to ScalingDown phase testenv.VerifySearchHeadClusterPhase(deployment, testenvInstance, splcommon.PhaseScalingDown) // Get instance of current Indexer CR with latest config @@ -695,7 +725,7 @@ var _ = Describe("m4appfw test", func() { // Ensure Indexer cluster scales down and go to ScalingDown phase testenv.VerifyIndexerClusterPhase(deployment, testenvInstance, splcommon.PhaseScalingDown, idxcName) - // Ensure SHC go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(deployment, testenvInstance) // Ensure Indexer cluster go to Ready phase @@ -704,47 +734,52 @@ var _ = Describe("m4appfw test", func() { // Verify RF SF is met testenv.VerifyRFSFMet(deployment, testenvInstance) + //######### SCALING DOWN VERIFICATIONS ###### // Verify bundle push status testenv.VerifyClusterManagerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), int(scaledIndexerReplicas), "") testenv.VerifyDeployerBundlePush(deployment, testenvInstance, testenvInstance.GetName(), siteCount) // Verify apps are copied to correct location allPodNames = testenv.DumpGetPods(testenvInstance.GetName()) - testenvInstance.Log.Info("Verify apps are copied to correct location based on Pod KIND after scaling down of indexers and SH", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to correct location based on Pod KIND after scaling down of Indexers and Search Heads", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV1, true, true) - // Verify apps are not copied in /etc/apps/ on CM and on Deployer (therefore not installed on Deployer and on CM) - testenvInstance.Log.Info("Verify apps are NOT copied to /etc/apps on CM and Deployer after scaling down of indexers and SH", "version", appVersion) + // Verify apps are not copied in /etc/apps/ on Cluster Manager and on Deployer (therefore not installed on Deployer and on Cluster Manager) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are NOT copied to /etc/apps on Cluster Manager and Deployer after scaling down of Indexers and Search Heads", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), managerPodNames, appListV1, false, false) - // Verify apps are installed cluster-wide - testenvInstance.Log.Info("Verify apps are installed on the pods by running Splunk CLI commands after scaling down of indexers and SH", "version", appVersion) + // Verify apps are installed cluster-wide after scaling down + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are installed on the pods after scaling down of Indexers and Search Heads", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV1, true, "enabled", false, true) }) }) - Context("Multi Site Indexer Cluster with SHC (m4) with App Framework)", func() { - It("integration, m4, appframework: can deploy a M4 SVA and have apps installed locally on CM and SHC Deployer", func() { + Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework)", func() { + It("integration, m4, appframework: can deploy a M4 SVA and have apps installed locally on Cluster Manager and Deployer", func() { /* Test Steps ################## SETUP #################### - * Create App Source with local scope for M4 SVA (CM and SHC Deployer) + * Upload V1 apps to S3 + * Create app source with local scope for M4 SVA (Cluster Manager and Deployer) * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ################## VERIFICATION ############# - * Verify apps are installed locally on CM and Deployer + ########## INITIAL VERIFICATION ############# + * Verify apps are installed locally on Cluster Manager and Deployer ############### UPGRADE APPS ################ * Upgrade apps in app sources * Wait for pods to be ready - ############### VERIFICATIONS ############### + ########## UPGRADE VERIFICATIONS ############ * Verify bundle push is successful - * Verify apps are copied, installed and upgraded on CM and Deployer + * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer */ - // Upload V1 apps to S3 + //################## SETUP #################### + // Upload V1 apps to S3 for M4 + appVersion := "V1" s3TestDir = "m4appfw-" + testenv.RandomDNSName(4) appFileList := testenv.GetAppFileList(appListV1, 1) + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for M4", appVersion)) uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for M4", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec @@ -763,80 +798,83 @@ var _ = Describe("m4appfw test", func() { AppSources: appSourceSpec, } - // Create Multi site Cluster and SHC, with App Framework enabled on CM and SHC Deployer + // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer siteCount := 3 indexersPerSite := 1 - testenvInstance.Log.Info("Deploy Multisite Indexer Cluster") + testenvInstance.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") err = deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpec, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") + Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - // Ensure that the CM goes to Ready phase + // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(deployment, testenvInstance) - // Ensure the indexers of all sites go to Ready phase + // Ensure the Indexers of all sites go to Ready phase testenv.IndexersReady(deployment, testenvInstance, siteCount) - // Ensure SHC go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(deployment, testenvInstance) - // Verify apps are downloaded by init-container + //########## INITIAL VERIFICATION ############# + // Verify V1 apps are downloaded initContDownloadLocation := "/init-apps/" + appSourceName podNames := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName()), fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - appVersion := "V1" - testenvInstance.Log.Info("Verify apps are downloaded by init container for apps", "version", appVersion) + appFileList = testenv.GetAppFileList(appListV1, 1) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded ", appVersion)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), podNames, appFileList, initContDownloadLocation) - // Verify apps are copied at the correct location on CM and on Deployer (/etc/apps/) - testenvInstance.Log.Info("Verify apps are copied to /etc/apps on CM and Deployer for app", "version", appVersion) + // Verify V1 apps are copied at the correct location on Cluster Manager and on Deployer (/etc/apps/) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to /etc/apps on Cluster Manager and Deployer", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), podNames, appListV1, true, false) - // Verify apps are installed locally on CM and on SHC Deployer - testenvInstance.Log.Info("Verify apps are installed locally on CM and Deployer for app", "version", appVersion) + // Verify V1 apps are installed locally on Cluster Manager and on Deployer + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are installed locally on Cluster Manager and Deployer", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), podNames, appListV1, false, "enabled", false, false) - // Verify apps are not copied in the apps folder on CM and /etc/shcluster/ on Deployer (therefore not installed on peers and on SH) - testenvInstance.Log.Info("Verify apps are not copied to "+splcommon.ManagerAppsLoc+" on CM and "+splcommon.SHCluster+" on Deployer for app", "version", appVersion) + // Verify V1 apps are not copied in the apps folder on Cluster Manager and /etc/shcluster/ on Deployer (therefore not installed on Indexers and on Search Heads) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are not copied to "+splcommon.ManagerAppsLoc+" on Cluster Manager and "+splcommon.SHCluster+" on Deployer", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), podNames, appListV1, false, true) - // Delete apps on S3 - testenvInstance.Log.Info("Delete apps on S3 for", "Version", appVersion) + //############### UPGRADE APPS ################ + // Delete V1 apps on S3 + testenvInstance.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) uploadedApps = nil - // Upload newer version of apps (V2) to S3 - appFileList = testenv.GetAppFileList(appListV2, 2) + // Upload V2 apps to S3 appVersion = "V2" + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3", appVersion)) + appFileList = testenv.GetAppFileList(appListV2, 2) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV2) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - testenvInstance.Log.Info("Uploading apps S3 for", "version", appVersion) // Wait for the poll period for the apps to be downloaded time.Sleep(2 * time.Minute) - // Ensure that the CM goes to Ready phase + // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(deployment, testenvInstance) - // Ensure the indexers of all sites go to Ready phase + // Ensure the Indexers of all sites go to Ready phase testenv.IndexersReady(deployment, testenvInstance, siteCount) - // Ensure SHC go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(deployment, testenvInstance) - // Verify apps are downloaded by init-container - testenvInstance.Log.Info("Verify apps are downloaded by init container on CM and Deployer", "version", appVersion) + //########## UPGRADE VERIFICATIONS ############ + // Verify V2 apps are downloaded on Cluster Manager and Deployer + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded on Cluster Manager and Deployer", appVersion)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), podNames, appFileList, initContDownloadLocation) - // Verify apps are copied at the correct location on CM and on Deployer (/etc/apps/) - testenvInstance.Log.Info("Verify apps are copied to /etc/apps on CM and Deployer for app", "version", appVersion) + // Verify V2 apps are copied at the correct location on Cluster Manager and on Deployer (/etc/apps/) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to /etc/apps on Cluster Manager and Deployer", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), podNames, appListV2, true, false) - // Verify apps are not copied in the apps folder on CM and /etc/shcluster/ on Deployer (therefore not installed on peers and on SH) - testenvInstance.Log.Info("Verify apps are not copied to "+splcommon.ManagerAppsLoc+" on CM and "+splcommon.SHCluster+" on Deployer for app", "version", appVersion) + // Verify V2 apps are not copied in the apps folder on Cluster Manager and /etc/shcluster/ on Deployer (therefore not installed on Indexers and on Search Heads) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are not copied to "+splcommon.ManagerAppsLoc+" on Cluster Manager and "+splcommon.SHCluster+" on Deployer", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), podNames, appListV2, false, true) - // Verify apps are installed locally on CM and on SHC Deployer - testenvInstance.Log.Info("Verify apps are installed locally on CM and Deployer for app", "version", appVersion) + // Verify V2 apps are installed locally on Cluster Manager and on Deployer + testenvInstance.Log.Info("Verify apps have been updated to %s on Cluster Manager and Deployer", appVersion) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), podNames, appListV2, true, "enabled", true, false) }) }) @@ -845,10 +883,11 @@ var _ = Describe("m4appfw test", func() { It("integration, m4, appframework: can deploy a M4 SVA with App Framework enabled for manual poll", func() { // Upload V1 apps to S3 + appVersion := "V1" s3TestDir = "m4appfw-" + testenv.RandomDNSName(4) appFileList := testenv.GetAppFileList(appListV1, 1) uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for M4", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec @@ -880,10 +919,10 @@ var _ = Describe("m4appfw test", func() { err = deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpec, true, "", "") Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") - // Ensure that the cluster-master goes to Ready phase + // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(deployment, testenvInstance) - // Ensure the indexers of all sites go to Ready phase + // Ensure the Indexers of all sites go to Ready phase testenv.IndexersReady(deployment, testenvInstance, siteCount) // Ensure cluster configured as multisite @@ -895,46 +934,45 @@ var _ = Describe("m4appfw test", func() { // Verify RF SF is met testenv.VerifyRFSFMet(deployment, testenvInstance) - // Verify Apps are downloaded by init-container - appVersion := "V1" + // Verify V1 apps are downloaded initContDownloadLocation := "/init-apps/" + appSourceName podNames := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName()), fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - testenvInstance.Log.Info("Verify Apps are downloaded by init container for apps", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded on Cluster Manager and Deployer", appVersion)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), podNames, appFileList, initContDownloadLocation) - //Verify Apps are copied to location + // Verify V1 apps are copied to location allPodNames := testenv.DumpGetPods(testenvInstance.GetName()) - testenvInstance.Log.Info("Verify Apps are copied to correct location based on Pod KIND for app", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to correct location based on Pod KIND", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV1, true, true) // Verify apps are not copied in /etc/apps/ on CM and on Deployer (therefore not installed on Deployer and on CM) - testenvInstance.Log.Info("Verify Apps are NOT copied to /etc/apps on CM and Deployer for app", "verison", appVersion, "App List", appFileList) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are NOT copied to /etc/apps on CM and Deployer for app (App List: %s)", appVersion, appFileList)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), podNames, appListV1, false, false) // Verify Apps are installed cluster-wide - testenvInstance.Log.Info("Verify Apps are installed on the pods by runnign Splunk CLI commands for app", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are installed on the pods", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV1, true, "enabled", false, true) - //Delete apps on S3 for new Apps - testenvInstance.Log.Info("Delete Apps on S3 for Version", appVersion) + // Delete apps on S3 for new Apps + testenvInstance.Log.Info(fmt.Sprintf("Delete %s apps on S3 for Version", appVersion)) testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) uploadedApps = nil - //Upload new Versioned Apps to S3 + // Upload V2 apps to S3 appFileList = testenv.GetAppFileList(appListV2, 2) appVersion = "V2" - testenvInstance.Log.Info("Uploading apps S3 for", "verison", appVersion) + testenvInstance.Log.Info("Uploading %s apps to S3", appVersion) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV2) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for M4", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Wait for the poll period for the apps to be downloaded time.Sleep(2 * time.Minute) - // Ensure that the cluster-master goes to Ready phase + // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(deployment, testenvInstance) - // Ensure the indexers of all sites go to Ready phase + // Ensure the Indexers of all sites go to Ready phase testenv.IndexersReady(deployment, testenvInstance, siteCount) // Ensure cluster configured as multisite @@ -947,7 +985,7 @@ var _ = Describe("m4appfw test", func() { testenv.VerifyRFSFMet(deployment, testenvInstance) //Verify Apps are not updated cluster-wide - testenvInstance.Log.Info("Verify Apps are not updated on the pods by running Splunk CLI commands for app", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are not updated on the pods", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV1, true, "enabled", false, true) testenvInstance.Log.Info("Get config map for triggering manual update") @@ -960,43 +998,43 @@ var _ = Describe("m4appfw test", func() { err = deployment.UpdateCR(config) Expect(err).To(Succeed(), "Unable to update config map") - // Ensure standalone is updating + // Ensure Standalone is updating testenv.VerifyClusterManagerPhase(deployment, testenvInstance, splcommon.PhaseUpdating) - // Ensure that the cluster-master goes to Ready phase + // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(deployment, testenvInstance) - // Ensure the indexers of all sites go to Ready phase + // Ensure the Indexers of all sites go to Ready phase testenv.IndexersReady(deployment, testenvInstance, siteCount) - // Ensure cluster configured as multisite + // Ensure Indexer cluster configured as multisite testenv.IndexerClusterMultisiteStatus(deployment, testenvInstance, siteCount) - // Ensure search head cluster go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(deployment, testenvInstance) // Verify RF SF is met testenv.VerifyRFSFMet(deployment, testenvInstance) //Verify config map set back to off after poll trigger - testenvInstance.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify config map set back to off after poll trigger for %s app", appVersion)) config, _ = testenv.GetAppframeworkManualUpdateConfigMap(deployment, testenvInstance.GetName()) Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - // Verify Apps are downloaded by init-container - testenvInstance.Log.Info("Verify Apps are downloaded by init container for apps", "version", appVersion) + // Verify apps are downloaded + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded by init container for apps", appVersion)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), podNames, appFileList, initContDownloadLocation) - //Verify Apps are copied to location - testenvInstance.Log.Info("Verify Apps are copied to correct location based on Pod KIND for app", "version", appVersion) + // Verify apps are copied to location + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to correct location based on Pod KIND", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV2, true, true) // Verify apps are not copied in /etc/apps/ on CM and on Deployer (therefore not installed on Deployer and on CM) - testenvInstance.Log.Info("Verify Apps are NOT copied to /etc/apps on CM and Deployer for app", "verison", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are NOT copied to /etc/apps on CM and Deployer for app", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), podNames, appListV2, false, false) - //Verify Apps are installed cluster-wide - testenvInstance.Log.Info("Verify Apps are installed on the pods by running Splunk CLI commands for app", "version", appVersion) + // Verify apps are installed cluster-wide + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are installed on the pods by running Splunk CLI commands for app", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), allPodNames, appListV2, true, "enabled", true, true) }) }) diff --git a/test/s1/appframework/appframework_test.go b/test/s1/appframework/appframework_test.go index 25a49efd4..6b0950c6b 100644 --- a/test/s1/appframework/appframework_test.go +++ b/test/s1/appframework/appframework_test.go @@ -55,28 +55,33 @@ var _ = Describe("s1appfw test", func() { }) Context("Standalone deployment (S1) with App Framework", func() { - It("smoke, s1, appframework: can deploy a standalone instance with App Framework enabled, install apps and upgrade them", func() { + It("smoke, s1, appframework: can deploy a Standalone instance with App Framework enabled, install apps then upgrade them", func() { /* Test Steps - ################## SETUP #################### - * Upload older versions of apps (V1) to S3 - * Create 2 app sources for Monitoring Console and Standalone - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############## VERIFICATION ################# - * Verify apps are copied and installed on Monitoring Console and on Standalone - ############### UPGRADE APPS ################ - * Upload newer versions of apps (V2) on S3 - * Wait for Monitoring Console and Standalone pods to be ready - ############## VERIFICATIONS ################ - * Verify apps are copied, installed and upgraded on Monitoring Console and on standalone + ################## SETUP #################### + * Upload V1 apps to S3 for Monitoring Console + * Create app source for Monitoring Console + * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready + * Upload V1 apps to S3 for Standalone + * Create app source for Standalone + * Prepare and deploy Standalone with app framework and wait for the pod to be ready + ############ INITIAL VERIFICATION ########### + * Verify V1 apps are copied and installed on Monitoring Console and on Standalone + ############## UPGRADE APPS ################# + * Upload V2 apps on S3 + * Wait for Monitoring Console and Standalone pods to be ready + ############ UPGRADE VERIFICATION ########### + * Verify V2 apps are copied, installed and upgraded on Monitoring Console and on Standalone */ - // Upload older versions of apps (V1) to S3 + //################## SETUP #################### + // Upload V1 apps to S3 for Monitoring Console + appVersion := "V1" + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) s3TestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) appFileList := testenv.GetAppFileListPhase3(appListV1) uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework spec for Monitoring Console @@ -108,19 +113,19 @@ var _ = Describe("s1appfw test", func() { testenvInstance.Log.Info("Deploy Monitoring Console") mcName := deployment.GetName() mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(testenvInstance.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is Ready and stays in ready state testenv.VerifyMonitoringConsoleReady(deployment, deployment.GetName(), mc, testenvInstance) - // Upload V1 apps to S3 + // Upload V1 apps to S3 for Standalone + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Standalone", appVersion)) s3TestDir = "s1appfw-" + testenv.RandomDNSName(4) - appFileList = testenv.GetAppFileListPhase3(appListV1) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) - // Create App framework Spec for Standalone + // Create App framework spec for Standalone volumeName := "appframework-test-volume-" + testenv.RandomDNSName(3) volumeSpec := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(volumeName, testenv.GetS3Endpoint(), testenvInstance.GetIndexSecretName(), "aws", "s3")} appSourceDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ @@ -155,8 +160,9 @@ var _ = Describe("s1appfw test", func() { } // Deploy Standalone + testenvInstance.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") // Verify App Download State on CR appFileList = testenv.GetAppFileListPhase3(appListV1) @@ -167,7 +173,6 @@ var _ = Describe("s1appfw test", func() { // opLocalAppPathStandalone := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), kind, deployment.GetName(), enterpriseApi.ScopeLocal, appSourceName) // opPod := testenv.GetOperatorPodName(testenvInstance.GetName()) - appVersion := "V1" // testenvInstance.Log.Info("Verify Apps are downloaded on Splunk Operator container for apps", "version", appVersion) // testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{opPod}, appFileList, opLocalAppPathStandalone) @@ -177,102 +182,101 @@ var _ = Describe("s1appfw test", func() { // Verify Monitoring Console is Ready and stays in ready state testenv.VerifyMonitoringConsoleReady(deployment, deployment.GetName(), mc, testenvInstance) - // Verify apps are downloaded by init-container on Standalone Pod and MC + //############ INITIAL VERIFICATION ########### + // Verify V1 apps are downloaded on Standalone and Monitoring Console initContDownloadLocationStandalonePod := "/init-apps/" + appSourceName initContDownloadLocationMCPod := "/init-apps/" + appSourceNameMC standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) mcPodName := fmt.Sprintf(testenv.MonitoringConsolePod, mcName, 0) - testenvInstance.Log.Info("Verify V1 apps are downloaded by init container for Standalone", "POD", standalonePodName, "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded on Standalone pod %s", appVersion, standalonePodName)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{standalonePodName}, appFileList, initContDownloadLocationStandalonePod) - testenvInstance.Log.Info("Verify V1 apps are downloaded by init container for Monitoring Console", "POD", mcPodName, "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded on Monitoring Console pod %s", appVersion, mcPodName)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{mcPodName}, appFileList, initContDownloadLocationMCPod) - podNames := []string{standalonePodName, mcPodName} - // Verify V1 apps are copied to location - testenvInstance.Log.Info("Verify V1 apps are copied to correct location on Pod for app", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to correct location on Standalone and Monitoring Console", appVersion)) + podNames := []string{standalonePodName, mcPodName} testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), podNames, appListV1, true, false) // Verify V1 apps are installed - testenvInstance.Log.Info("Verify V1 apps are installed on the pods by running Splunk CLI commands for app", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are installed on Standalone and Monitoring Console", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), podNames, appListV1, true, "enabled", false, false) + //############## UPGRADE APPS ################# // Delete apps on S3 - testenvInstance.Log.Info("Delete apps on S3 for", "Version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) uploadedApps = nil - // Upload newer version of apps (V2) to S3 for Standalone and Monitoring Console - appFileList = testenv.GetAppFileListPhase3(appListV2) + // Upload V2 apps to S3 for Standalone and Monitoring Console appVersion = "V2" + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Standalone and Monitoring Console", appVersion)) + appFileList = testenv.GetAppFileListPhase3(appListV2) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV2) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Wait for the poll period for the apps to be downloaded time.Sleep(2 * time.Minute) - // Wait for Standalone to be in UPDATING status - testenv.VerifyStandalonePhase(deployment, testenvInstance, deployment.GetName(), splcommon.PhasePending) - // Wait for Standalone to be in READY status testenv.StandaloneReady(deployment, deployment.GetName(), standalone, testenvInstance) // Verify Monitoring Console is Ready and stays in ready state testenv.VerifyMonitoringConsoleReady(deployment, deployment.GetName(), mc, testenvInstance) - // Verify V2 apps are downloaded by init-container - testenvInstance.Log.Info("Verify V2 apps are downloaded by init container for apps", "POD", standalonePodName, "version", appVersion) + //############ UPGRADE VERIFICATION ########### + // Verify V2 apps are downloaded + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded on Standalone pod %s", appVersion, standalonePodName)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{standalonePodName}, appFileList, initContDownloadLocationStandalonePod) - testenvInstance.Log.Info("Verify V2 apps are downloaded by init container for apps", "POD", mcPodName, "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded on Monitoring Console pod %s", appVersion, mcPodName)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{mcPodName}, appFileList, initContDownloadLocationMCPod) // Verify V2 apps are copied to location - testenvInstance.Log.Info("Verify V2 apps are copied to correct location on Pod for app", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to correct location on Standalone and Monitoring Console", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), podNames, appListV2, true, false) // Verify V2 apps are installed - testenvInstance.Log.Info("Verify apps are updated on the pods by running Splunk CLI commands for app", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify apps have been updated to %s on Standalone and Monitoring Console", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), podNames, appListV2, true, "enabled", true, false) }) }) Context("Standalone deployment (S1) with App Framework", func() { - It("smoke, s1, appframework: can deploy a standalone instance with App Framework enabled, install apps and downgrade them", func() { + It("smoke, s1, appframework: can deploy a Standalone instance with App Framework enabled, install apps then downgrade them", func() { /* Test Steps - ################## SETUP #################### - * Upload newer versions of apps (V2) to S3 - * Create 2 app sources for Monitoring Console and Standalone - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############## VERIFICATION ################# - * Verify apps are copied and installed on Monitoring Console and on Standalone - ############# DOWNGRADE APPS ################ - * Upload older versions of apps (V1) on S3 - * Wait for Monitoring Console and Standalone pods to be ready - ############## VERIFICATION ################# - * Verify apps are copied, installed and downgraded on Monitoring Console and also on standalone + ################## SETUP #################### + * Upload V2 apps to S3 for Monitoring Console + * Create app source for Monitoring Console + * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready + * Upload V2 apps to S3 for Standalone + * Create app source for Standalone + * Prepare and deploy Standalone with app framework and wait for the pod to be ready + ############ INITIAL VERIFICATION ########### + * Verify apps are copied and installed on Monitoring Console and on Standalone + ############# DOWNGRADE APPS ################ + * Upload V1 apps on S3 + * Wait for Monitoring Console and Standalone pods to be ready + ########## DOWNGRADE VERIFICATION ########### + * Verify apps are copied, installed and downgraded on Monitoring Console and on Standalone */ - // Delete pre-installed apps on S3 - testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) - uploadedApps = nil - testenvInstance.Log.Info("Testing upgrade scenario") - - // Upload newer versions of apps (V2) to S3 for MC + //################## SETUP #################### + // Upload V2 apps to S3 appVersion := "V2" + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Standalone and Monitoring Console", appVersion)) s3TestDir = "s1appfw-" + testenv.RandomDNSName(4) appFileList := testenv.GetAppFileListPhase3(appListV2) uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV2) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory for S1") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) s3TestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory for Monitoring Console") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec for Monitoring Console @@ -304,7 +308,7 @@ var _ = Describe("s1appfw test", func() { testenvInstance.Log.Info("Deploy Monitoring Console") mcName := deployment.GetName() mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(testenvInstance.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is Ready and stays in ready state testenv.VerifyMonitoringConsoleReady(deployment, deployment.GetName(), mc, testenvInstance) @@ -338,8 +342,9 @@ var _ = Describe("s1appfw test", func() { } // Deploy Standalone + testenvInstance.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") // Wait for Standalone to be in READY status testenv.StandaloneReady(deployment, deployment.GetName(), standalone, testenvInstance) @@ -347,39 +352,42 @@ var _ = Describe("s1appfw test", func() { // Verify Monitoring Console is Ready and stays in ready state testenv.VerifyMonitoringConsoleReady(deployment, deployment.GetName(), mc, testenvInstance) - // Verify apps are downloaded by init-container on Standalone and Monitoring Console + //############ INITIAL VERIFICATION ########### + // Verify V2 apps are downloaded on Standalone Pod and Monitoring Console initContDownloadLocationStandalonePod := "/init-apps/" + appSourceName initContDownloadLocationMCPod := "/init-apps/" + appSourceNameMC standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) mcPodName := fmt.Sprintf(testenv.MonitoringConsolePod, mcName, 0) appFileList = testenv.GetAppFileListPhase3(appListV2) - testenvInstance.Log.Info("Verify V2 apps are downloaded by init container for Standalone", "POD", standalonePodName, "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded for Standalone pod %s", appVersion, standalonePodName)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{standalonePodName}, appFileList, initContDownloadLocationStandalonePod) - testenvInstance.Log.Info("Verify V2 apps are downloaded by init container for Monitoring Console", "POD", mcPodName, "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded for Monitoring Console pod %s", appVersion, mcPodName)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{mcPodName}, appFileList, initContDownloadLocationMCPod) - podNames := []string{standalonePodName, mcPodName} - // Verify V2 apps are copied to location - testenvInstance.Log.Info("Verify V2 apps are copied to correct location on Pod for app", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to correct location on Standalone and Monitoring Console", appVersion)) + podNames := []string{standalonePodName, mcPodName} testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), podNames, appListV2, true, false) // Verify V2 apps are installed - testenvInstance.Log.Info("Verify V2 apps are installed on the pods by running Splunk CLI commands for app", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are installed on Standalone and Monitoring Console", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), podNames, appListV2, true, "enabled", true, false) + //############# DOWNGRADE APPS ################ // Delete apps on S3 + testenvInstance.Log.Info(fmt.Sprintf("Delete %s apps on S3", appVersion)) testenv.DeleteFilesOnS3(testS3Bucket, uploadedApps) uploadedApps = nil - // Upload older version of apps (V1) to S3 for Standalone and Monitoring Console + // Upload V1 apps to S3 for Standalone and Monitoring Console appVersion = "V1" + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Standalone and Monitoring Console", appVersion)) appFileList = testenv.GetAppFileListPhase3(appListV1) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Wait for the poll period for the apps to be downloaded @@ -400,18 +408,19 @@ var _ = Describe("s1appfw test", func() { // Verify Monitoring Console is Ready and stays in ready state testenv.VerifyMonitoringConsoleReady(deployment, deployment.GetName(), mc, testenvInstance) - // Verify apps are downloaded by init-container - testenvInstance.Log.Info("Verify V1 apps are downloaded by init container for apps", "POD", standalonePodName, "version", appVersion) + //########## DOWNGRADE VERIFICATION ########### + // Verify apps are downloaded + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded for Standalone pod %s", appVersion, standalonePodName)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{standalonePodName}, appFileList, initContDownloadLocationStandalonePod) - testenvInstance.Log.Info("Verify V1 apps are downloaded by init container for apps", "POD", mcPodName, "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded for Monitoring Console pod %s ", appVersion, mcPodName)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{mcPodName}, appFileList, initContDownloadLocationMCPod) // Verify apps are copied to location - testenvInstance.Log.Info("Verify V1 apps are copied to correct location on Pod for app", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to correct location on Standalone and Monitoring Console", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), podNames, appListV1, true, false) // Verify apps are downgraded - testenvInstance.Log.Info("Verify V1 apps are installed on the pods by running Splunk CLI commands for app", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify apps have been downgraded to %s on Standalone and Monitoring Console", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), podNames, appListV1, true, "enabled", false, false) }) }) @@ -425,25 +434,34 @@ var _ = Describe("s1appfw test", func() { * Create 2 app sources for Monitoring Console and Standalone * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready * Prepare and deploy Standalone CRD with app framework and wait for the pod to be ready - ################# VERIFICATION ############### + ########## INITIAL VERIFICATION ############# * Verify apps are copied and installed on Monitoring Console and Standalone - ################## SCALING UP ################ + ############### SCALING UP ################## * Scale up Standalone * Wait for Monitoring Console and Standalone to be ready - ################# VERIFICATION ############### + ########### SCALING UP VERIFICATION ######### * Verify apps are copied and installed on new Standalone pod - ################ SCALING DOWN ################ + ############## SCALING DOWN ################# * Scale down Standalone * Wait for Monitoring Console and Standalone to be ready - ################# VERIFICATION ############### + ########### SCALING DOWN VERIFICATION ####### * Verify apps are still copied and installed on Standalone */ - // Upload apps to S3 for Monitoring Console - s3TestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) + //################## SETUP #################### + // Upload V1 apps to S3 for Standalone and Monitoring Console + appVersion := "V1" appFileList := testenv.GetAppFileListPhase3(appListV1) + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Monitoring Console", appVersion)) + s3TestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Monitoring Console", appVersion)) + uploadedApps = append(uploadedApps, uploadedFiles...) + + testenvInstance.Log.Info(fmt.Sprintf("Upload %s apps to S3 for Standalone", appVersion)) + s3TestDir = "s1appfw-" + testenv.RandomDNSName(4) + uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Standalone", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec for Monitoring Console @@ -472,9 +490,10 @@ var _ = Describe("s1appfw test", func() { } // Deploy Monitoring Console + testenvInstance.Log.Info("Deploy Monitoring Console") mcName := deployment.GetName() mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(testenvInstance.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") // Verify Monitoring Console is Ready and stays in ready state testenv.VerifyMonitoringConsoleReady(deployment, deployment.GetName(), mc, testenvInstance) @@ -514,6 +533,7 @@ var _ = Describe("s1appfw test", func() { } // Deploy Standalone + testenvInstance.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(deployment.GetName(), spec) Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") @@ -523,30 +543,30 @@ var _ = Describe("s1appfw test", func() { // Verify Monitoring Console is Ready and stays in ready state testenv.VerifyMonitoringConsoleReady(deployment, deployment.GetName(), mc, testenvInstance) - // Verify apps are downloaded by init-container on Standalone Pod and Monitoring Console - appVersion := "V1" + //########## INITIAL VERIFICATION ############# + // Verify apps are downloaded on Standalone and Monitoring Console initContDownloadLocationStandalonePod := "/init-apps/" + appSourceName initContDownloadLocationMCPod := "/init-apps/" + appSourceNameMC standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) mcPodName := fmt.Sprintf(testenv.MonitoringConsolePod, mcName, 0) appFileList = testenv.GetAppFileListPhase3(appListV1) - testenvInstance.Log.Info("Verify Apps are downloaded by init container for Standalone", "POD", standalonePodName, "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded for Standalone pod %s", appVersion, standalonePodName)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{standalonePodName}, appFileList, initContDownloadLocationStandalonePod) - testenvInstance.Log.Info("Verify Apps are downloaded by init container for Monitoring Console", "POD", mcPodName, "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded for Monitoring Console pod %s", appVersion, mcPodName)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{mcPodName}, appFileList, initContDownloadLocationMCPod) - podNames := []string{standalonePodName, mcPodName} - // Verify apps are copied to location - testenvInstance.Log.Info("Verify Apps are copied to correct location on Pod for app", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to correct location on Standalone and Monitoring Console", appVersion)) + podNames := []string{standalonePodName, mcPodName} testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), podNames, appListV1, true, true) // Verify apps are installed - testenvInstance.Log.Info("Verify Apps are installed on the pods by running Splunk CLI commands for app", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are installed on Standalone and Monitoring Console", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), podNames, appListV1, true, "enabled", false, false) + //############### SCALING UP ################## // Scale up Standalone instance - testenvInstance.Log.Info("Scaling Up Standalone CR") + testenvInstance.Log.Info("Scale up Standalone") scaledReplicaCount := 2 standalone = &enterpriseApi.Standalone{} err = deployment.GetInstance(deployment.GetName(), standalone) @@ -569,32 +589,34 @@ var _ = Describe("s1appfw test", func() { podNames = append(podNames, fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 1)) standalonePods := []string{standalonePodName, fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 1)} - // Verify apps are downloaded by init-container - testenvInstance.Log.Info("Verify Apps are downloaded by init container on Standalone after scaling up", "POD", standalonePods, "version", appVersion) + //########### SCALING UP VERIFICATION ######### + // Verify apps are downloaded + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded after scaling up on Standalone pods %s", appVersion, standalonePods)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), standalonePods, appFileList, initContDownloadLocationStandalonePod) - testenvInstance.Log.Info("Verify Apps are downloaded by init container on Monitoring Console after scaling up", "POD", mcPodName, "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded after scaling up on Monitoring Console pod %s", appVersion, mcPodName)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{mcPodName}, appFileList, initContDownloadLocationMCPod) // Verify apps are copied to location - testenvInstance.Log.Info("Verify Apps are copied to correct location on all Pods after scaling up", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to correct location on all pods after scaling up", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), podNames, appListV1, true, false) // Verify apps are installed - testenvInstance.Log.Info("Verify Apps are installed on the pods after scaling up", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are installed on all pods after scaling up", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), podNames, appListV1, true, "enabled", false, false) + //############## SCALING DOWN ################# // Scale down Standalone instance - testenvInstance.Log.Info("Scaling down Standalone CR") + testenvInstance.Log.Info("Scale down Standalone") scaledReplicaCount = 1 standalone = &enterpriseApi.Standalone{} err = deployment.GetInstance(deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") + Expect(err).To(Succeed(), "Failed to get instance of Standalone after scaling down") standalone.Spec.Replicas = int32(scaledReplicaCount) err = deployment.UpdateCR(standalone) Expect(err).To(Succeed(), "Failed to scale down Standalone") - // Ensure standalone is scaling down + // Ensure Standalone is scaling down testenv.VerifyStandalonePhase(deployment, testenvInstance, deployment.GetName(), splcommon.PhaseScalingDown) // Wait for Standalone to be in READY status @@ -605,18 +627,19 @@ var _ = Describe("s1appfw test", func() { podNames = testenv.DumpGetPods(testenvInstance.GetName()) - // Verify apps are downloaded by init-container - testenvInstance.Log.Info("Verify Apps are downloaded by init container for Standalone after scaling down ", "POD", standalonePodName, "version", appVersion) + //########### SCALING DOWN VERIFICATION ####### + // Verify apps are downloaded + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded after scaling down for Standalone pod %s", appVersion, standalonePodName)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{standalonePodName}, appFileList, initContDownloadLocationStandalonePod) - testenvInstance.Log.Info("Verify Apps are downloaded by init container for Monitoring Console after scaling down ", "POD", mcPodName, "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded after scaling down for Monitoring Console pod %s", appVersion, mcPodName)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{mcPodName}, appFileList, initContDownloadLocationMCPod) // Verify apps are copied to location - testenvInstance.Log.Info("Verify Apps are copied to correct location on all Pods after scaling down ", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to correct location on all Pods after scaling down ", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), podNames, appListV1, true, false) // Verify apps are installed - testenvInstance.Log.Info("Verify Apps are installed on the pods after scaling down ", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are still installed on the pods after scaling down", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), podNames, appListV1, true, "enabled", false, false) // Delete apps on S3 for new Apps @@ -664,16 +687,29 @@ var _ = Describe("s1appfw test", func() { Context("Standalone deployment (S1) with App Framework", func() { It("s1, integration, appframework: can deploy a Standalone and have ES app installed", func() { - // Upload ES app to S3 - esApp := []string{"SplunkEnterpriseSecuritySuite"} - appFileList := testenv.GetAppFileListPhase3(esApp) + /* Test Steps + ################## SETUP #################### + * Upload ES app to S3 + * Create App Source for Standalone + * Prepare and deploy Standalone and wait for the pod to be ready + ################## VERIFICATION ############# + * Verify ES app is installed on Standalone + */ + + //################## SETUP #################### // Download ES App from S3 + testenvInstance.Log.Info("Download ES app from S3") + esApp := []string{"SplunkEnterpriseSecuritySuite"} + appFileList := testenv.GetAppFileListPhase3(esApp) err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download ES app file") + Expect(err).To(Succeed(), "Unable to download ES app") - // Upload ES app to S3 + // Create local directory for file download s3TestDir = "s1appfw-" + testenv.RandomDNSName(4) + + // Upload ES app to S3 + testenvInstance.Log.Info("Upload ES app on S3") uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload ES app to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -694,8 +730,6 @@ var _ = Describe("s1appfw test", func() { VolList: volumeSpec, AppSources: appSourceSpec, } - - // Create Standalone spec with App Framework enabled and some extra config to have ES installed correctly spec := enterpriseApi.StandaloneSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: splcommon.Spec{ @@ -706,6 +740,8 @@ var _ = Describe("s1appfw test", func() { AppFrameworkConfig: appFrameworkSpec, } + // Deploy Standalone + testenvInstance.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(deployment.GetName(), spec) Expect(err).To(Succeed(), "Unable to deploy Standalone with App framework") @@ -723,32 +759,48 @@ var _ = Describe("s1appfw test", func() { // Ensure Standalone goes to Ready phase testenv.StandaloneReady(deployment, deployment.GetName(), standalone, testenvInstance) - // Verify apps are downloaded by init-container + //################## VERIFICATION ############# + // Verify ES app is downloaded + testenvInstance.Log.Info("Verify ES app is downloaded on Standalone") initContDownloadLocation := "/init-apps/" + appSourceName podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) testenvInstance.Log.Info("Verify Apps are downloaded on Pod", "version", appVersion, "Pod Name", podName) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{podName}, appFileList, initContDownloadLocation) - // Verify apps are installed locally + // Verify ES app is installed + testenvInstance.Log.Info("Verify ES app is installed on Standalone") standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), standalonePod, esApp, false, "enabled", false, false) }) }) Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1, appframework: can deploy a standalone instance with App Framework enabled and install around 350MB of apps at once", func() { + It("integration, s1, appframework: can deploy a Standalone instance with App Framework enabled and install around 350MB of apps at once", func() { + + /* Test Steps + ################## SETUP #################### + * Create app source for Standalone + * Add more apps than usual on S3 for this test + * Prepare and deploy Standalone with app framework and wait for the pod to be ready + ############### VERIFICATION ################ + * Verify apps are copied, installed on Standalone + */ + + //################## SETUP #################### // Creating a bigger list of apps to be installed than the default one appList := append(appListV1, testenv.RestartNeededApps...) appFileList := testenv.GetAppFileListPhase3(appList) // Download apps from S3 - err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, testenv.GetAppFileListPhase3(testenv.RestartNeededApps)) + testenvInstance.Log.Info("Download bigger amount of apps from S3 for this test") + err := testenv.DownloadFilesFromS3(testDataS3Bucket, s3AppDirV1, downloadDirV1, testenv.GetAppFileListPhase3(appList)) Expect(err).To(Succeed(), "Unable to download apps files") // Upload apps to S3 + testenvInstance.Log.Info("Upload bigger amount of apps to S3 for this test") s3TestDir = "s1appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, testenv.GetAppFileListPhase3(appList), downloadDirV1) + uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") uploadedApps = append(uploadedApps, uploadedFiles...) @@ -781,23 +833,28 @@ var _ = Describe("s1appfw test", func() { AppFrameworkConfig: appFrameworkSpec, } - // Create Standalone Deployment with App Framework + // Deploy Standalone + testenvInstance.Log.Info("Deploy Standalone") standalone, err := deployment.DeployStandaloneWithGivenSpec(deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with App framework") + Expect(err).To(Succeed(), "Unable to deploy Standalone instance") // Wait for Standalone to be in READY status testenv.StandaloneReady(deployment, deployment.GetName(), standalone, testenvInstance) - // Verify apps are downloaded by init-container + //############### VERIFICATION ################ + // Verify apps are downloaded + testenvInstance.Log.Info("Verify apps are downloaded for Standalone") initContDownloadLocation := "/init-apps/" + appSourceName podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{podName}, appFileList, initContDownloadLocation) - // Verify apps are copied to location + // Verify apps are copied to correct location + testenvInstance.Log.Info("Verify apps are copied to correct location on Standalone") testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), []string{podName}, appList, true, false) // Verify apps are installed + testenvInstance.Log.Info("Verify apps are installed on Standalone") testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), []string{podName}, appList, true, "enabled", false, false) }) }) @@ -805,11 +862,12 @@ var _ = Describe("s1appfw test", func() { Context("appframework Standalone deployment (S1) with App Framework", func() { It("s1, smoke, appframework: can deploy a standalone instance with App Framework enabled for manual poll", func() { - // Upload V1 versions of apps (V1) to S3 + // Upload V1 apps to S3 + appVersion := "V1" s3TestDir := "s1appfw-mc-" + testenv.RandomDNSName(4) appFileList := testenv.GetAppFileListPhase3(appListV1) uploadedFiles, err := testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Create App framework Spec @@ -855,7 +913,7 @@ var _ = Describe("s1appfw test", func() { // kind := standalone.Kind // opLocalAppPathStandalone := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), kind, deployment.GetName(), enterpriseApi.ScopeLocal, appSourceName) // opPod := testenv.GetOperatorPodName(testenvInstance.GetName()) - appVersion := "V1" + // testenvInstance.Log.Info("Verify Apps are downloaded on Splunk Operator container for apps", "version", appVersion) // testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{opPod}, appFileList, opLocalAppPathStandalone) @@ -865,15 +923,15 @@ var _ = Describe("s1appfw test", func() { // Verify Apps are downloaded by init-container initContDownloadLocation := "/init-apps/" + appSourceName podName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - testenvInstance.Log.Info("Verify Apps are downloaded by init container for apps", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded by init container for apps", appVersion)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{podName}, appFileList, initContDownloadLocation) - //Verify Apps are copied to location - testenvInstance.Log.Info("Verify Apps are copied to correct location on standalone(/etc/apps/) for app", "version", appVersion) + // Verify Apps are copied to location + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to correct location on standalone(/etc/apps/)", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), []string{podName}, appListV1, true, true) //Verify Apps are installed - testenvInstance.Log.Info("Verify Apps are installed Locally on CM and Deployer by running Splunk CLI commands for app", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are installed Locally on CM and Deployer", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), []string{podName}, appListV1, true, "enabled", false, false) //Delete apps on S3 for new Apps @@ -884,7 +942,7 @@ var _ = Describe("s1appfw test", func() { appVersion = "V2" appFileList = testenv.GetAppFileListPhase3(appListV2) uploadedFiles, err = testenv.UploadFilesToS3(testS3Bucket, s3TestDir, appFileList, downloadDirV2) - Expect(err).To(Succeed(), "Unable to upload apps to S3 test directory") + Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory", appVersion)) uploadedApps = append(uploadedApps, uploadedFiles...) // Wait for the poll period for the apps to be downloaded @@ -894,7 +952,7 @@ var _ = Describe("s1appfw test", func() { testenv.StandaloneReady(deployment, deployment.GetName(), standalone, testenvInstance) //Verify Apps are not updated - testenvInstance.Log.Info("Verify Apps are not auto installed on standalone by running Splunk CLI commands for app", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are not auto installed on standalone", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), []string{podName}, appListV1, true, "enabled", false, false) testenvInstance.Log.Info("Get config map for triggering manual update") @@ -921,20 +979,20 @@ var _ = Describe("s1appfw test", func() { testenv.StandaloneReady(deployment, deployment.GetName(), standalone, testenvInstance) //Verify config map set back to off after poll trigger - testenvInstance.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify config map set back to off after poll trigger for %s app", appVersion)) config, _ = testenv.GetAppframeworkManualUpdateConfigMap(deployment, testenvInstance.GetName()) Expect(strings.Contains(config.Data["Standalone"], "status: off")).To(Equal(true), "Config map update not complete") // Verify Apps are downloaded by init-container - testenvInstance.Log.Info("Verify Apps are downloaded by init container for apps", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are downloaded", appVersion)) testenv.VerifyAppsDownloadedOnContainer(deployment, testenvInstance, testenvInstance.GetName(), []string{podName}, appFileList, initContDownloadLocation) //Verify Apps are copied to location - testenvInstance.Log.Info("Verify Apps are copied to correct location on standalone(/etc/apps/) for app", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are copied to correct location on standalone(/etc/apps/)", appVersion)) testenv.VerifyAppsCopied(deployment, testenvInstance, testenvInstance.GetName(), []string{podName}, appListV2, true, true) //Verify Apps are updated - testenvInstance.Log.Info("Verify Apps are installed Locally on standalone by running Splunk CLI commands for app", "version", appVersion) + testenvInstance.Log.Info(fmt.Sprintf("Verify %s apps are installed Locally on standalone", appVersion)) testenv.VerifyAppInstalled(deployment, testenvInstance, testenvInstance.GetName(), []string{podName}, appListV2, true, "enabled", true, false) }) }) diff --git a/test/testenv/appframework_utils.go b/test/testenv/appframework_utils.go index c9bd6b131..11d10eff7 100644 --- a/test/testenv/appframework_utils.go +++ b/test/testenv/appframework_utils.go @@ -106,11 +106,11 @@ func GetPodInstalledAppVersion(deployment *Deployment, podName string, ns string } } filePath := fmt.Sprintf("/opt/splunk/%s/%s/default/app.conf", path, appname) - logf.Log.Info("Check Version for app", "AppName", appname, "config", filePath) + logf.Log.Info("Check app version", "App", appname, "Conf file", filePath) confline, err := GetConfLineFromPod(podName, filePath, ns, "version", "launcher", true) if err != nil { - logf.Log.Error(err, "Failed to get Version from pod", "Pod Name", podName) + logf.Log.Error(err, "Failed to get version from pod", "Pod Name", podName) return "", err } version := strings.TrimSpace(strings.Split(confline, "=")[1]) @@ -127,7 +127,7 @@ func GetPodAppInstallStatus(deployment *Deployment, podName string, ns string, a logf.Log.Error(err, "Failed to execute command on pod", "pod", podName, "command", command, "stdin", stdin) return "", err } - logf.Log.Info("Command executed on pod", "pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) + logf.Log.Info("Command executed", "on pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) return strings.TrimSuffix(stdout, "\n"), nil } @@ -141,10 +141,10 @@ func GetPodAppbtoolStatus(deployment *Deployment, podName string, ns string, app logf.Log.Error(err, "Failed to execute command on pod", "pod", podName, "command", command, "stdin", stdin) return "", err } - logf.Log.Info("Command executed on pod", "pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) + logf.Log.Info("Command executed", "on pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) if len(stdout) > 0 { - if strings.Contains(strings.Split(stdout, "\n")[0], "Application is disabled") { + if strings.Contains(strings.Split(stdout, "\n")[0], "App is disabled") { return "DISABLED", nil } return "ENABLED", nil diff --git a/test/testenv/s3utils.go b/test/testenv/s3utils.go index 192c8a182..f809f4940 100644 --- a/test/testenv/s3utils.go +++ b/test/testenv/s3utils.go @@ -195,10 +195,10 @@ func GetFilesInPathOnS3(bucket string, path string) []string { // DownloadFilesFromS3 download given list of files from S3 to the given directory func DownloadFilesFromS3(testDataS3Bucket string, s3AppDir string, downloadDir string, appList []string) error { for _, key := range appList { - logf.Log.Info("Downloading app from S3", "App Name", key) + logf.Log.Info("Downloading file from S3", "File name", key) _, err := DownloadFileFromS3(testDataS3Bucket, key, s3AppDir, downloadDir) if err != nil { - logf.Log.Error(err, "Unable to downlaod file", "File Name", key) + logf.Log.Error(err, "Unable to download file", "File Name", key) return err } } @@ -209,19 +209,19 @@ func DownloadFilesFromS3(testDataS3Bucket string, s3AppDir string, downloadDir s func UploadFilesToS3(testS3Bucket string, s3TestDir string, applist []string, downloadDir string) ([]string, error) { var uploadedFiles []string for _, key := range applist { - logf.Log.Info("Uploading app to s3", "App Name", key) + logf.Log.Info("Uploading file to S3", "File name", key) fileLocation := filepath.Join(downloadDir, key) fileBody, err := os.Open(fileLocation) if err != nil { - logf.Log.Error(err, "Unable to open app file", "App Name", key) + logf.Log.Error(err, "Unable to open file", "File name", key) return nil, err } fileName, err := UploadFileToS3(testS3Bucket, key, s3TestDir, fileBody) if err != nil { - logf.Log.Error(err, "Unable to upload file", "File Name", key) + logf.Log.Error(err, "Unable to upload file", "File name", key) return nil, err } - logf.Log.Info("App upload to test S3", "App Name", fileName) + logf.Log.Info("File upload to test S3", "File name", fileName) uploadedFiles = append(uploadedFiles, fileName) } return uploadedFiles, nil diff --git a/test/testenv/util.go b/test/testenv/util.go index 57c9f3719..5f8bf9524 100644 --- a/test/testenv/util.go +++ b/test/testenv/util.go @@ -622,7 +622,7 @@ func GetConfLineFromPod(podName string, filePath string, ns string, configName s } continue } else if strings.HasPrefix(line, configName) { - logf.Log.Info("Configuration found.", "Config", configName, "Line", line) + logf.Log.Info(fmt.Sprintf("Configuration %s found at line %s", configName, line)) config = line break } @@ -641,7 +641,7 @@ func ExecuteCommandOnPod(deployment *Deployment, podName string, stdin string) ( logf.Log.Error(err, "Failed to execute command on pod", "pod", podName, "command", command) return "", err } - logf.Log.Info("Command executed on pod", "pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) + logf.Log.Info("Command executed", "on pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) return stdout, nil } diff --git a/test/testenv/verificationutils.go b/test/testenv/verificationutils.go index 3a8b53ccb..703f2e719 100644 --- a/test/testenv/verificationutils.go +++ b/test/testenv/verificationutils.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -66,7 +66,7 @@ func VerifyMonitoringConsoleReady(deployment *Deployment, mcName string, monitor if err != nil { return splcommon.PhaseError } - testenvInstance.Log.Info("Waiting for Monitoring Console STATUS to be ready", "instance", monitoringConsole.ObjectMeta.Name, "Phase", monitoringConsole.Status.Phase) + testenvInstance.Log.Info("Waiting for Monitoring Console phase to be ready", "instance", monitoringConsole.ObjectMeta.Name, "Phase", monitoringConsole.Status.Phase) DumpGetPods(testenvInstance.GetName()) return monitoringConsole.Status.Phase }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(splcommon.PhaseReady)) @@ -78,14 +78,14 @@ func VerifyMonitoringConsoleReady(deployment *Deployment, mcName string, monitor }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(splcommon.PhaseReady)) } -// StandaloneReady verify Standlone is in ReadyStatus and does not flip-flop +// StandaloneReady verify Standalone is in ReadyStatus and does not flip-flop func StandaloneReady(deployment *Deployment, deploymentName string, standalone *enterpriseApi.Standalone, testenvInstance *TestEnv) { gomega.Eventually(func() splcommon.Phase { err := deployment.GetInstance(deploymentName, standalone) if err != nil { return splcommon.PhaseError } - testenvInstance.Log.Info("Waiting for standalone STATUS to be ready", "instance", standalone.ObjectMeta.Name, "Phase", standalone.Status.Phase) + testenvInstance.Log.Info("Waiting for Standalone phase to be ready", "instance", standalone.ObjectMeta.Name, "Phase", standalone.Status.Phase) DumpGetPods(testenvInstance.GetName()) return standalone.Status.Phase }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(splcommon.PhaseReady)) @@ -106,7 +106,7 @@ func SearchHeadClusterReady(deployment *Deployment, testenvInstance *TestEnv) { if err != nil { return splcommon.PhaseError } - testenvInstance.Log.Info("Waiting for search head cluster STATUS to be ready", "instance", shc.ObjectMeta.Name, "Phase", shc.Status.Phase) + testenvInstance.Log.Info("Waiting for Search head cluster phase to be ready", "instance", shc.ObjectMeta.Name, "Phase", shc.Status.Phase) DumpGetPods(testenvInstance.GetName()) return shc.Status.Phase }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(splcommon.PhaseReady)) @@ -116,7 +116,7 @@ func SearchHeadClusterReady(deployment *Deployment, testenvInstance *TestEnv) { if err != nil { return splcommon.PhaseError } - testenvInstance.Log.Info("Waiting for Deployer STATUS to be ready", "instance", shc.ObjectMeta.Name, "Phase", shc.Status.DeployerPhase) + testenvInstance.Log.Info("Waiting for Deployer phase to be ready", "instance", shc.ObjectMeta.Name, "Phase", shc.Status.DeployerPhase) DumpGetPods(testenvInstance.GetName()) return shc.Status.DeployerPhase }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(splcommon.PhaseReady)) @@ -126,7 +126,7 @@ func SearchHeadClusterReady(deployment *Deployment, testenvInstance *TestEnv) { if err != nil { return splcommon.PhaseError } - testenvInstance.Log.Info("Waiting for search head cluster STATUS to be ready", "instance", shc.ObjectMeta.Name, "Phase", shc.Status.Phase) + testenvInstance.Log.Info("Waiting for Search Head Cluster phase to be ready", "instance", shc.ObjectMeta.Name, "Phase", shc.Status.Phase) DumpGetPods(testenvInstance.GetName()) return shc.Status.Phase }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(splcommon.PhaseReady)) @@ -147,7 +147,7 @@ func SingleSiteIndexersReady(deployment *Deployment, testenvInstance *TestEnv) { if err != nil { return splcommon.PhaseError } - testenvInstance.Log.Info("Waiting for indexer instance's status to be ready", "instance", instanceName, "Phase", idc.Status.Phase) + testenvInstance.Log.Info("Waiting for indexer instance's phase to be ready", "instance", instanceName, "Phase", idc.Status.Phase) DumpGetPods(testenvInstance.GetName()) return idc.Status.Phase }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(splcommon.PhaseReady)) @@ -168,7 +168,7 @@ func ClusterManagerReady(deployment *Deployment, testenvInstance *TestEnv) { if err != nil { return splcommon.PhaseError } - testenvInstance.Log.Info("Waiting for "+splcommon.ClusterManager+" instance status to be ready", "instance", cm.ObjectMeta.Name, "Phase", cm.Status.Phase) + testenvInstance.Log.Info("Waiting for "+splcommon.ClusterManager+" phase to be ready", "instance", cm.ObjectMeta.Name, "Phase", cm.Status.Phase) DumpGetPods(testenvInstance.GetName()) // Test ClusterManager Phase to see if its ready return cm.Status.Phase @@ -195,7 +195,7 @@ func IndexersReady(deployment *Deployment, testenvInstance *TestEnv, siteCount i if err != nil { return splcommon.PhaseError } - testenvInstance.Log.Info("Waiting for indexer site instance status to be ready", "instance", instanceName, "Phase", idc.Status.Phase) + testenvInstance.Log.Info("Waiting for indexer site instance phase to be ready", "instance", instanceName, "Phase", idc.Status.Phase) DumpGetPods(testenvInstance.GetName()) return idc.Status.Phase }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(splcommon.PhaseReady)) @@ -222,10 +222,10 @@ func IndexerClusterMultisiteStatus(deployment *Deployment, testenvInstance *Test command := []string{"/bin/sh"} stdout, stderr, err := deployment.PodExecCommand(podName, command, stdin, false) if err != nil { - testenvInstance.Log.Error(err, "Failed to execute command on pod", "pod", podName, "command", command) + testenvInstance.Log.Error(err, "Failed to execute command", "on pod", podName, "command", command) return map[string][]string{} } - testenvInstance.Log.Info("Command executed on pod", "pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) + testenvInstance.Log.Info("Command executed", "on pod", podName, "command", command, "stdin", stdin, "stdout", stdout, "stderr", stderr) siteIndexerResponse := ClusterMasterSitesResponse{} json.Unmarshal([]byte(stdout), &siteIndexerResponse) siteIndexerStatus := map[string][]string{} @@ -252,7 +252,7 @@ func VerifyRFSFMet(deployment *Deployment, testenvInstance *TestEnv) { func VerifyNoDisconnectedSHPresentOnCM(deployment *Deployment, testenvInstance *TestEnv) { gomega.Consistently(func() bool { shStatus := CheckSearchHeadRemoved(deployment) - testenvInstance.Log.Info("Verifying no SH in DISCONNECTED state present on CM", "Status", shStatus) + testenvInstance.Log.Info("Verifying no Search Head in DISCONNECTED state present on Cluster Manager", "Status", shStatus) return shStatus }, ConsistentDuration, ConsistentPollInterval).Should(gomega.Equal(true)) } @@ -261,7 +261,7 @@ func VerifyNoDisconnectedSHPresentOnCM(deployment *Deployment, testenvInstance * func VerifyNoSHCInNamespace(deployment *Deployment, testenvInstance *TestEnv) { gomega.Eventually(func() bool { shcStatus := SHCInNamespace(testenvInstance.GetName()) - testenvInstance.Log.Info("Verifying no SHC is present in namespace", "Status", shcStatus) + testenvInstance.Log.Info("Verifying no Search Head Cluster is present in namespace", "Status", shcStatus) return shcStatus }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(false)) } @@ -310,7 +310,7 @@ func VerifyServiceAccountConfiguredOnPod(deployment *Deployment, ns string, podN restResponse := PodDetailsStruct{} err = json.Unmarshal([]byte(output), &restResponse) if err != nil { - logf.Log.Error(err, "Failed to parse cluster searchheads") + logf.Log.Error(err, "Failed to parse cluster Search heads") return false } logf.Log.Info("Service Account on Pod", "FOUND", restResponse.Spec.ServiceAccount, "EXPECTED", serviceAccount) @@ -419,7 +419,7 @@ func VerifyStandalonePhase(deployment *Deployment, testenvInstance *TestEnv, crN if err != nil { return splcommon.PhaseError } - testenvInstance.Log.Info("Waiting for standalone status", "instance", standalone.ObjectMeta.Name, "Expected", phase, " Actual Phase", standalone.Status.Phase) + testenvInstance.Log.Info("Waiting for Standalone status", "instance", standalone.ObjectMeta.Name, "Expected", phase, " Actual Phase", standalone.Status.Phase) DumpGetPods(testenvInstance.GetName()) return standalone.Status.Phase }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(phase)) @@ -433,7 +433,7 @@ func VerifyMonitoringConsolePhase(deployment *Deployment, testenvInstance *TestE if err != nil { return splcommon.PhaseError } - testenvInstance.Log.Info("Waiting for monitoring console CR status", "instance", mc.ObjectMeta.Name, "Expected", phase, " Actual Phase", mc.Status.Phase) + testenvInstance.Log.Info("Waiting for Monitoring Console CR status", "instance", mc.ObjectMeta.Name, "Expected", phase, " Actual Phase", mc.Status.Phase) DumpGetPods(testenvInstance.GetName()) return mc.Status.Phase }, deployment.GetTimeout(), PollInterval).Should(gomega.Equal(phase)) @@ -617,7 +617,7 @@ func VerifyAppInstalled(deployment *Deployment, testenvInstance *TestEnv, ns str for _, podName := range pods { for _, appName := range apps { status, versionInstalled, err := GetPodAppStatus(deployment, podName, ns, appName, clusterWideInstall) - logf.Log.Info("App info returned for app", "App-name", appName, "status", status, "versionInstalled", versionInstalled, "error", err) + logf.Log.Info("App details", "App", appName, "Status", status, "Version", versionInstalled, "Error", err) gomega.Expect(err).To(gomega.Succeed(), "Unable to get app status on pod ") comparison := strings.EqualFold(status, statusCheck) //Check the app is installed on specific pods and un-installed on others for cluster-wide install @@ -625,7 +625,7 @@ func VerifyAppInstalled(deployment *Deployment, testenvInstance *TestEnv, ns str if clusterWideInstall { if strings.Contains(podName, "-indexer-") || strings.Contains(podName, "-search-head-") { check = true - testenvInstance.Log.Info("App Install Check", "Pod Name", podName, "App Name", appName, "Expected", check, "Found", comparison, "Cluster Install Scope", clusterWideInstall) + testenvInstance.Log.Info("App Install Check", "Pod", podName, "App", appName, "Expected", check, "Found", comparison, "Scope:cluster", clusterWideInstall) gomega.Expect(comparison).Should(gomega.Equal(check)) } } else { @@ -635,7 +635,7 @@ func VerifyAppInstalled(deployment *Deployment, testenvInstance *TestEnv, ns str } else { check = true } - testenvInstance.Log.Info("App Install Check", "Pod Name", podName, "App Name", appName, "Expected", check, "Found", comparison, "Cluster Install Scope", clusterWideInstall) + testenvInstance.Log.Info("App Install Check", "Pod", podName, "App", appName, "Expected", check, "Found", comparison, "Scope:cluster", clusterWideInstall) gomega.Expect(comparison).Should(gomega.Equal(check)) } @@ -648,7 +648,7 @@ func VerifyAppInstalled(deployment *Deployment, testenvInstance *TestEnv, ns str } else { expectedVersion = AppInfo[appName]["V1"] } - testenvInstance.Log.Info("Verify app Version", "Pod Name", podName, "App Name", appName, "Expected Version", expectedVersion, "Version Installed", versionInstalled, "Updated", checkupdated) + testenvInstance.Log.Info("Verify app", "On pod", podName, "App name", appName, "Expected version", expectedVersion, "Version installed", versionInstalled, "Updated", checkupdated) gomega.Expect(versionInstalled).Should(gomega.Equal(expectedVersion)) } } @@ -670,7 +670,6 @@ func VerifyAppsCopied(deployment *Deployment, testenvInstance *TestEnv, ns strin path = splcommon.PeerAppsLoc } } - testenvInstance.Log.Info("Verifying App in Directory", "Directory Name", path, "Pod Name", podName) VerifyAppsInFolder(deployment, testenvInstance, ns, podName, apps, path, checkAppDirectory) } } @@ -684,7 +683,7 @@ func VerifyAppsInFolder(deployment *Deployment, testenvInstance *TestEnv, ns str for _, app := range apps { folderName := app + "/" found := CheckStringInSlice(appList, folderName) - logf.Log.Info("Check App in Directory", "Pod Name", podName, "App-name", folderName, "Path", path, "status", found) + logf.Log.Info("App check", "On pod", podName, "check app", folderName, "is in path", path, "Status", found) if found != checkAppDirectory { return false }