Add openshift e2e test (#2675)

* Add Openshift e2e tests

* Make distro configurable

* avoid patch version checks

* make route test re-entrant

* infer TENANT_ID from circleci env

* enable openshift-3.9-rhel-e2e

.circleci/config.yml

@@ -353,6 +353,37 @@ jobs:
           path: /go/src/github.com/Azure/acs-engine/_logs
       - store_artifacts:
           path: /go/src/github.com/Azure/acs-engine/_output
+  openshift-3.9-rhel-e2e:
+    working_directory: /go/src/github.com/Azure/acs-engine
+    docker:
+      - image: registry.svc.ci.openshift.org/ci/acs-engine-tests:v3.9
+        environment:
+          GOPATH: /go
+    steps:
+      - checkout
+      - run: |
+          echo 'export TIMEOUT=30m' >> $BASH_ENV
+          echo 'export DISTRO=openshift39_rhel' >> $BASH_ENV
+          echo 'export LOCATION=eastus' >> $BASH_ENV
+          echo 'export ORCHESTRATOR_RELEASE=3.9' >> $BASH_ENV
+          echo 'export CLUSTER_DEFINITION=examples/openshift.json' >> $BASH_ENV
+          echo 'export CREATE_VNET=false' >> $BASH_ENV
+          echo 'export CLEANUP_ON_EXIT=false' >> $BASH_ENV
+          echo 'export RETAIN_SSH=false' >> $BASH_ENV
+          echo 'export SUBSCRIPTION_ID=${SUBSCRIPTION_ID_E2E_KUBERNETES}' >> $BASH_ENV
+          echo 'export CLIENT_ID=${SERVICE_PRINCIPAL_CLIENT_ID_E2E_KUBERNETES}' >> $BASH_ENV
+          echo 'export CLIENT_SECRET=${SERVICE_PRINCIPAL_CLIENT_SECRET_E2E_KUBERNETES}' >> $BASH_ENV
+      - run:
+          name: compile
+          command: make build-binary
+      - run:
+          name: ginkgo openshift e2e tests
+          command: make test-openshift
+          no_output_timeout: "30m"
+      - store_artifacts:
+          path: /go/src/github.com/Azure/acs-engine/_logs
+      - store_artifacts:
+          path: /go/src/github.com/Azure/acs-engine/_output
 workflows:
   version: 2
   build_and_test_pr:
@@ -428,6 +459,12 @@ workflows:
           filters:
             branches:
               ignore: master
+      - openshift-3.9-rhel-e2e:
+          requires:
+            - pr-e2e-hold
+          filters:
+            branches:
+              ignore: master
       - swarm-e2e:
           requires:
             - pr-e2e-hold
@@ -488,6 +525,10 @@ workflows:
           filters:
             branches:
               only: master
+      - openshift-3.9-rhel-e2e:
+          filters:
+            branches:
+              only: master
       - swarm-e2e:
           requires:
             - test

.gitignore

@@ -27,6 +27,7 @@ test/acs-engine-test/report/TestReport.json
 
 # I have no idea why these get generated when I run the e2e test
 test/e2e/kubernetes/translations/
+test/e2e/openshift/translations/
 
 # test outputs
 cmd/_test_output

Makefile

@@ -101,7 +101,7 @@ ifneq ($(GIT_BASEDIR),)
 endif
 
 test: generate
-	ginkgo -skipPackage test/e2e/dcos,test/e2e/kubernetes -r .
+	ginkgo -skipPackage test/e2e/dcos,test/e2e/kubernetes,test/e2e/openshift -r .
 
 .PHONY: test-style
 test-style:

scripts/ginkgo.coverage.sh

@@ -25,7 +25,7 @@ hash goveralls 2>/dev/null || go get github.com/mattn/goveralls
 hash godir 2>/dev/null || go get github.com/Masterminds/godir
 
 generate_cover_data() {
-  ginkgo -skipPackage test/e2e/dcos,test/e2e/kubernetes -cover -r .
+  ginkgo -skipPackage test/e2e/dcos,test/e2e/kubernetes,test/e2e/openshift -cover -r .
   echo "" > ${coveragetxt}  
   find . -type f -name "*.coverprofile" | while read -r file;  do cat $file >> ${coveragetxt} && mv $file ${coverdir}; done
   echo "mode: $covermode" >"$profile"

test.mk

@@ -28,5 +28,8 @@ test-kubernetes:
 test-dcos:
 	@ORCHESTRATOR=dcos go run ./test/e2e/runner.go
 
+test-openshift:
+	@ORCHESTRATOR=openshift go run ./test/e2e/runner.go
+
 test-azure-constants:
 	./scripts/azure-const.sh

test/e2e/config/config.go

@@ -38,6 +38,7 @@ const (
 	dcosOrchestrator       = "dcos"
 	swarmModeOrchestrator  = "swarmmode"
 	swarmOrchestrator      = "swarm"
+	openShiftOrchestrator  = "openshift"
 )
 
 // ParseConfig will parse needed environment variables for running the tests
@@ -54,9 +55,24 @@ func ParseConfig() (*Config, error) {
 
 // GetKubeConfig returns the absolute path to the kubeconfig for c.Location
 func (c *Config) GetKubeConfig() string {
-	file := fmt.Sprintf("kubeconfig.%s.json", c.Location)
-	kubeconfig := filepath.Join(c.CurrentWorkingDir, "_output", c.Name, "kubeconfig", file)
-	return kubeconfig
+	var kubeconfigPath string
+
+	switch {
+	case c.IsKubernetes():
+		file := fmt.Sprintf("kubeconfig.%s.json", c.Location)
+		kubeconfigPath = filepath.Join(c.CurrentWorkingDir, "_output", c.Name, "kubeconfig", file)
+
+	case c.IsOpenShift():
+		artifactsDir := filepath.Join(c.CurrentWorkingDir, "_output", c.Name)
+		masterTarball := filepath.Join(artifactsDir, "master.tar.gz")
+		out, err := exec.Command("tar", "-xzf", masterTarball, "-C", artifactsDir).CombinedOutput()
+		if err != nil {
+			log.Fatalf("Cannot untar master tarball: %v: %v", out, err)
+		}
+		kubeconfigPath = filepath.Join(artifactsDir, "etc", "origin", "master", "admin.kubeconfig")
+	}
+
+	return kubeconfigPath
 }
 
 // SetKubeConfig will set the KUBECONIFG env var
@@ -149,6 +165,11 @@ func (c *Config) IsSwarm() bool {
 	return c.Orchestrator == swarmOrchestrator
 }
 
+// IsOpenShift will return true if the ORCHESTRATOR env var is set to openshift
+func (c *Config) IsOpenShift() bool {
+	return c.Orchestrator == openShiftOrchestrator
+}
+
 // SetRandomRegion sets Location to a random region
 func (c *Config) SetRandomRegion() {
 	var regions []string

test/e2e/engine/template.go

@@ -1,6 +1,8 @@
 package engine
 
 import (
+	"crypto/rand"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -30,6 +32,9 @@ type Config struct {
 	OutputDirectory       string `envconfig:"OUTPUT_DIR" default:"_output"`
 	CreateVNET            bool   `envconfig:"CREATE_VNET" default:"false"`
 	EnableKMSEncryption   bool   `envconfig:"ENABLE_KMS_ENCRYPTION" default:"false"`
+	Distro                string `envconfig:"DISTRO"`
+	SubscriptionID        string `envconfig:"SUBSCRIPTION_ID"`
+	TenantID              string `envconfig:"TENANT_ID"`
 
 	ClusterDefinitionPath     string // The original template we want to use to build the cluster from.
 	ClusterDefinitionTemplate string // This is the template after we splice in the environment variables
@@ -85,12 +90,37 @@ func Build(cfg *config.Config, subnetID string) (*Engine, error) {
 			Secret:   config.ClientSecret,
 		}
 	}
+	if cfg.IsOpenShift() {
+		// azProfile
+		cs.ContainerService.Properties.AzProfile = &vlabs.AzProfile{
+			TenantID:       config.TenantID,
+			SubscriptionID: config.SubscriptionID,
+			ResourceGroup:  cfg.Name,
+			Location:       cfg.Location,
+		}
+		// openshiftConfig
+		pass, err := generateRandomString(32)
+		if err != nil {
+			return nil, err
+		}
+		cs.ContainerService.Properties.OrchestratorProfile.OpenShiftConfig = &vlabs.OpenShiftConfig{
+			ClusterUsername: "test-user",
+			ClusterPassword: pass,
+		}
+		// master and agent config
+		cs.ContainerService.Properties.MasterProfile.Distro = vlabs.Distro(config.Distro)
+		cs.ContainerService.Properties.MasterProfile.ImageRef = nil
+		for i := range cs.ContainerService.Properties.AgentPoolProfiles {
+			cs.ContainerService.Properties.AgentPoolProfiles[i].Distro = vlabs.Distro(config.Distro)
+			cs.ContainerService.Properties.AgentPoolProfiles[i].ImageRef = nil
+		}
+	}
 
 	if config.MasterDNSPrefix != "" {
 		cs.ContainerService.Properties.MasterProfile.DNSPrefix = config.MasterDNSPrefix
 	}
 
-	if !cfg.IsKubernetes() && config.AgentDNSPrefix != "" {
+	if !cfg.IsKubernetes() && !cfg.IsOpenShift() && config.AgentDNSPrefix != "" {
 		for idx, pool := range cs.ContainerService.Properties.AgentPoolProfiles {
 			pool.DNSPrefix = fmt.Sprintf("%v-%v", config.AgentDNSPrefix, idx)
 		}
@@ -236,3 +266,17 @@ func ParseOutput(path string) (*api.ContainerService, error) {
 	}
 	return containerService, nil
 }
+
+func generateRandomBytes(n int) ([]byte, error) {
+	b := make([]byte, n)
+	_, err := rand.Read(b)
+	if err != nil {
+		return nil, err
+	}
+	return b, nil
+}
+
+func generateRandomString(s int) (string, error) {
+	b, err := generateRandomBytes(s)
+	return base64.URLEncoding.EncodeToString(b), err
+}

test/e2e/openshift/OWNERS

@@ -0,0 +1,4 @@
+approvers:
+- jim-minter
+- kargakis
+- pweil-

test/e2e/openshift/node/node.go

@@ -0,0 +1,29 @@
+package node
+
+import (
+	"errors"
+	"fmt"
+	"log"
+	"os/exec"
+	"regexp"
+	"strings"
+)
+
+// Version returns the version of an OpenShift cluster.
+func Version() (string, error) {
+	cmd := exec.Command("oc", "version")
+	fmt.Printf("\n$ %s\n", strings.Join(cmd.Args, " "))
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		log.Printf("Error trying to run 'oc version':%s", string(out))
+		return "", err
+	}
+	exp := regexp.MustCompile(`(openshift\s)+(v\d+.\d+.\d+)+`)
+	for _, line := range strings.Split(string(out), "\n") {
+		if strings.HasPrefix(line, "openshift") {
+			s := exp.FindStringSubmatch(line)
+			return s[2], nil
+		}
+	}
+	return "", errors.New("cannot find openshift version")
+}

test/e2e/openshift/openshift_suite_test.go

@@ -0,0 +1,13 @@
+package openshift_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"testing"
+)
+
+func TestOpenShift(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "OpenShift Suite")
+}