From 3bfa25fc4dfbeadf4470fcf01a81dcc704a76e6e Mon Sep 17 00:00:00 2001 From: Niki Maslarski Date: Thu, 25 Apr 2019 10:45:59 +0300 Subject: [PATCH 1/2] Introduce basic cf services performance test [finishes #163462432] Signed-off-by: Georgi Lozev --- Makefile | 8 ++ .../performance/performance_suite_test.go | 75 ++++++++++++++++++ .../services_command_performance_test.go | 78 +++++++++++++++++++ 3 files changed, 161 insertions(+) create mode 100644 integration/shared/performance/performance_suite_test.go create mode 100644 integration/shared/performance/services_command_performance_test.go diff --git a/Makefile b/Makefile index 6acd38cdda0..71ada8d28bb 100644 --- a/Makefile +++ b/Makefile @@ -106,6 +106,14 @@ integration-isolated-shared: integration-shared-isolated integration-shared-isolated: build integration-cleanup ## Run all parallel-enabled integration tests that are shared between v6 and v7 $(ginkgo_int) -nodes $(NODES) integration/shared/isolated +ip: integration-performance +integration-performance: build integration-cleanup integration-shared-performance + +isp: integration-shared-performance +integration-performance-shared: integration-shared-performance +integration-shared-performance: build integration-cleanup + $(ginkgo_int) integration/shared/performance + ivi: integration-versioned-isolated integration-isolated-versioned: integration-versioned-isolated integration-versioned-isolated: build integration-cleanup ## Run all parallel-enabled integration tests, both versioned and shared across versions diff --git a/integration/shared/performance/performance_suite_test.go b/integration/shared/performance/performance_suite_test.go new file mode 100644 index 00000000000..d36919b1360 --- /dev/null +++ b/integration/shared/performance/performance_suite_test.go @@ -0,0 +1,75 @@ +package performance_test + +import ( + "fmt" + "testing" + "time" + + "code.cloudfoundry.org/cli/integration/helpers" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +const ( + CFEventuallyTimeout = 300 * time.Second + CFConsistentlyTimeout = 500 * time.Millisecond +) + +var ( + // Suite Level + apiURL string + skipSSLValidation bool + perfOrg string + perfSpace string + + // Per Test Level + homeDir string +) + +// TODO: Performance tests should probably run serially and in the same order every time + +func TestPerformance(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Performance Integration Suite") +} + +var _ = SynchronizedBeforeSuite(func() []byte { + GinkgoWriter.Write([]byte("==============================Global FIRST Node Synchronized Before Each==============================")) + GinkgoWriter.Write([]byte("==============================End of Global FIRST Node Synchronized Before Each==============================")) + + return nil +}, func(_ []byte) { + GinkgoWriter.Write([]byte(fmt.Sprintf("==============================Global Node %d Synchronized Before Each==============================", GinkgoParallelNode()))) + // Ginkgo Globals + SetDefaultEventuallyTimeout(CFEventuallyTimeout) + SetDefaultConsistentlyDuration(CFConsistentlyTimeout) + + // Setup common environment variables + helpers.TurnOffColors() + perfOrg, perfSpace = helpers.SetupReadOnlyOrgAndSpace() + + GinkgoWriter.Write([]byte(fmt.Sprintf("==============================End of Global Node %d Synchronized Before Each==============================", GinkgoParallelNode()))) +}) + +var _ = SynchronizedAfterSuite(func() { + GinkgoWriter.Write([]byte(fmt.Sprintf("==============================Global Node %d Synchronized After Each==============================", GinkgoParallelNode()))) + homeDir = helpers.SetHomeDir() + helpers.SetAPI() + helpers.LoginCF() + helpers.QuickDeleteOrg(perfOrg) + helpers.DestroyHomeDir(homeDir) + GinkgoWriter.Write([]byte(fmt.Sprintf("==============================End of Global Node %d Synchronized After Each==============================", GinkgoParallelNode()))) +}, func() {}) + +var _ = BeforeEach(func() { + GinkgoWriter.Write([]byte("==============================Global Before Each==============================")) + homeDir = helpers.SetHomeDir() + apiURL, skipSSLValidation = helpers.SetAPI() + GinkgoWriter.Write([]byte("==============================End of Global Before Each==============================")) +}) + +var _ = AfterEach(func() { + GinkgoWriter.Write([]byte("==============================Global After Each==============================\n")) + helpers.DestroyHomeDir(homeDir) + GinkgoWriter.Write([]byte("==============================End of Global After Each==============================")) +}) diff --git a/integration/shared/performance/services_command_performance_test.go b/integration/shared/performance/services_command_performance_test.go new file mode 100644 index 00000000000..98d0a627a63 --- /dev/null +++ b/integration/shared/performance/services_command_performance_test.go @@ -0,0 +1,78 @@ +package performance_test + +import ( + "fmt" + "os" + "strconv" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gexec" + + "code.cloudfoundry.org/cli/integration/helpers" +) + +var _ = Describe("services command performance", func() { + const ( + serviceName = "service" + servicePlan = "service-plan" + ) + + var ( + broker helpers.ServiceBroker + currentExecution int + maxExecutions = getEnvOrDefault("MAX_EXECUTIONS", 10) + numberOfServices = getEnvOrDefault("NUMBER_OF_SERVICE_INSTANCES", 15) + ) + + BeforeEach(func() { + helpers.LoginCF() + helpers.TargetOrgAndSpace(perfOrg, perfSpace) + + currentExecution++ + if os.Getenv("SKIP_PERF_SETUP") == "true" || currentExecution > 1 { + return + } + + domain := helpers.DefaultSharedDomain() + broker = helpers.CreateBroker(domain, serviceName, servicePlan) + + Eventually(helpers.CF("enable-service-access", serviceName)).Should(Exit(0)) + + for i := 0; i < numberOfServices; i++ { + Eventually(helpers.CF("create-service", serviceName, servicePlan, fmt.Sprintf("instance-%d", i))).Should(Exit(0)) + } + }) + + AfterEach(func() { + if currentExecution == maxExecutions { + for i := 0; i < numberOfServices; i++ { + Eventually(helpers.CF("delete-service", fmt.Sprintf("instance-%d", i), "-f")).Should(Exit(0)) + } + broker.Destroy() + } + }) + + Measure("services command", func(b Benchmarker) { + b.Time("cf services", func() { + fmt.Printf("cf services...") + session := helpers.CF("services") + session.Wait() + fmt.Printf(" DONE.\n") + Expect(session).Should(Exit(0)) + }) + }, maxExecutions) +}) + +func getEnvOrDefault(key string, defaultValue int) int { + val, ok := os.LookupEnv(key) + if !ok { + return defaultValue + } + + value, err := strconv.Atoi(val) + if err == nil { + return value + } + return defaultValue +} From 27bced5cb16bdd9f41bd6c4a8ad1665bb01e7708 Mon Sep 17 00:00:00 2001 From: Florent Flament Date: Fri, 26 Apr 2019 17:14:03 +0200 Subject: [PATCH 2/2] Have performance test displaying more information Displays number of samples as well as services instances used during the test. [finishes #163462432] --- .../shared/performance/services_command_performance_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/integration/shared/performance/services_command_performance_test.go b/integration/shared/performance/services_command_performance_test.go index 98d0a627a63..a124f72a14b 100644 --- a/integration/shared/performance/services_command_performance_test.go +++ b/integration/shared/performance/services_command_performance_test.go @@ -34,6 +34,10 @@ var _ = Describe("services command performance", func() { return } + /* Display some useful information */ + fmt.Printf("Number of samples (MAX_EXECUTIONS): %d\n", maxExecutions) + fmt.Printf("Number of service instances (NUMBER_OF_SERVICE_INSTANCES): %d\n", numberOfServices) + domain := helpers.DefaultSharedDomain() broker = helpers.CreateBroker(domain, serviceName, servicePlan)