Merge pull request cloudfoundry#1626 from cloudfoundry-incubator/reba…

…se_services_command_performance_test

Create cf services command performance test

[finishes #165674740]

Makefile

@@ -106,6 +106,14 @@ integration-isolated-shared: integration-shared-isolated
 integration-shared-isolated: build integration-cleanup ## Run all parallel-enabled integration tests that are shared between v6 and v7
 	$(ginkgo_int) -nodes $(NODES) integration/shared/isolated
 
+ip: integration-performance
+integration-performance: build integration-cleanup integration-shared-performance
+
+isp: integration-shared-performance
+integration-performance-shared: integration-shared-performance
+integration-shared-performance: build integration-cleanup
+	$(ginkgo_int) integration/shared/performance
+
 ivi: integration-versioned-isolated
 integration-isolated-versioned: integration-versioned-isolated
 integration-versioned-isolated: build integration-cleanup ## Run all parallel-enabled integration tests, both versioned and shared across versions

integration/shared/performance/performance_suite_test.go

@@ -0,0 +1,75 @@
+package performance_test
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"code.cloudfoundry.org/cli/integration/helpers"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+const (
+	CFEventuallyTimeout   = 300 * time.Second
+	CFConsistentlyTimeout = 500 * time.Millisecond
+)
+
+var (
+	// Suite Level
+	apiURL            string
+	skipSSLValidation bool
+	perfOrg           string
+	perfSpace         string
+
+	// Per Test Level
+	homeDir string
+)
+
+// TODO: Performance tests should probably run serially and in the same order every time
+
+func TestPerformance(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Performance Integration Suite")
+}
+
+var _ = SynchronizedBeforeSuite(func() []byte {
+	GinkgoWriter.Write([]byte("==============================Global FIRST Node Synchronized Before Each=============================="))
+	GinkgoWriter.Write([]byte("==============================End of Global FIRST Node Synchronized Before Each=============================="))
+
+	return nil
+}, func(_ []byte) {
+	GinkgoWriter.Write([]byte(fmt.Sprintf("==============================Global Node %d Synchronized Before Each==============================", GinkgoParallelNode())))
+	// Ginkgo Globals
+	SetDefaultEventuallyTimeout(CFEventuallyTimeout)
+	SetDefaultConsistentlyDuration(CFConsistentlyTimeout)
+
+	// Setup common environment variables
+	helpers.TurnOffColors()
+	perfOrg, perfSpace = helpers.SetupReadOnlyOrgAndSpace()
+
+	GinkgoWriter.Write([]byte(fmt.Sprintf("==============================End of Global Node %d Synchronized Before Each==============================", GinkgoParallelNode())))
+})
+
+var _ = SynchronizedAfterSuite(func() {
+	GinkgoWriter.Write([]byte(fmt.Sprintf("==============================Global Node %d Synchronized After Each==============================", GinkgoParallelNode())))
+	homeDir = helpers.SetHomeDir()
+	helpers.SetAPI()
+	helpers.LoginCF()
+	helpers.QuickDeleteOrg(perfOrg)
+	helpers.DestroyHomeDir(homeDir)
+	GinkgoWriter.Write([]byte(fmt.Sprintf("==============================End of Global Node %d Synchronized After Each==============================", GinkgoParallelNode())))
+}, func() {})
+
+var _ = BeforeEach(func() {
+	GinkgoWriter.Write([]byte("==============================Global Before Each=============================="))
+	homeDir = helpers.SetHomeDir()
+	apiURL, skipSSLValidation = helpers.SetAPI()
+	GinkgoWriter.Write([]byte("==============================End of Global Before Each=============================="))
+})
+
+var _ = AfterEach(func() {
+	GinkgoWriter.Write([]byte("==============================Global After Each==============================\n"))
+	helpers.DestroyHomeDir(homeDir)
+	GinkgoWriter.Write([]byte("==============================End of Global After Each=============================="))
+})

integration/shared/performance/services_command_performance_test.go

@@ -0,0 +1,82 @@
+package performance_test
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	. "github.com/onsi/gomega/gexec"
+
+	"code.cloudfoundry.org/cli/integration/helpers"
+)
+
+var _ = Describe("services command performance", func() {
+	const (
+		serviceName = "service"
+		servicePlan = "service-plan"
+	)
+
+	var (
+		broker           helpers.ServiceBroker
+		currentExecution int
+		maxExecutions    = getEnvOrDefault("MAX_EXECUTIONS", 10)
+		numberOfServices = getEnvOrDefault("NUMBER_OF_SERVICE_INSTANCES", 15)
+	)
+
+	BeforeEach(func() {
+		helpers.LoginCF()
+		helpers.TargetOrgAndSpace(perfOrg, perfSpace)
+
+		currentExecution++
+		if os.Getenv("SKIP_PERF_SETUP") == "true" || currentExecution > 1 {
+			return
+		}
+
+		/* Display some useful information */
+		fmt.Printf("Number of samples (MAX_EXECUTIONS): %d\n", maxExecutions)
+		fmt.Printf("Number of service instances (NUMBER_OF_SERVICE_INSTANCES): %d\n", numberOfServices)
+
+		domain := helpers.DefaultSharedDomain()
+		broker = helpers.CreateBroker(domain, serviceName, servicePlan)
+
+		Eventually(helpers.CF("enable-service-access", serviceName)).Should(Exit(0))
+
+		for i := 0; i < numberOfServices; i++ {
+			Eventually(helpers.CF("create-service", serviceName, servicePlan, fmt.Sprintf("instance-%d", i))).Should(Exit(0))
+		}
+	})
+
+	AfterEach(func() {
+		if currentExecution == maxExecutions {
+			for i := 0; i < numberOfServices; i++ {
+				Eventually(helpers.CF("delete-service", fmt.Sprintf("instance-%d", i), "-f")).Should(Exit(0))
+			}
+			broker.Destroy()
+		}
+	})
+
+	Measure("services command", func(b Benchmarker) {
+		b.Time("cf services", func() {
+			fmt.Printf("cf services...")
+			session := helpers.CF("services")
+			session.Wait()
+			fmt.Printf(" DONE.\n")
+			Expect(session).Should(Exit(0))
+		})
+	}, maxExecutions)
+})
+
+func getEnvOrDefault(key string, defaultValue int) int {
+	val, ok := os.LookupEnv(key)
+	if !ok {
+		return defaultValue
+	}
+
+	value, err := strconv.Atoi(val)
+	if err == nil {
+		return value
+	}
+	return defaultValue
+}