Update run-e2e-test script to setup cluster

Makefile

@@ -56,9 +56,13 @@ test-integration:
 	chmod +x ${AWS_K8S_TESTER_PATH}
 	${AWS_K8S_TESTER_PATH} csi test integration --terminate-on-exit=true --timeout=20m ${PR_NUM_FLAG} ${VPC_ID_FLAG}
 
-.PHONY: test-e2e
-test-e2e:
-	./hack/run-e2e-test
+.PHONY: test-e2e-single-az
+test-e2e-single-az:
+	AWS_REGION=us-west-2 AWS_AVAILABILITY_ZONES=us-west-2a GINKGO_FOCUS="\[ebs-csi-e2e\] \[single-az\]" ./hack/run-e2e-test
+
+.PHONY: test-e2e-multi-az
+test-e2e-multi-az:
+	AWS_REGION=us-west-2 AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c GINKGO_FOCUS="\[ebs-csi-e2e\] \[multi-az\]" ./hack/run-e2e-test
 
 .PHONY: image-release
 image-release:

docs/README.md

@@ -122,34 +122,21 @@ In order to make sure that the driver complies with the CSI specification, run:
 make test-sanity
 ```
 
-To execute integration tests, run:
+See [Ingetration Testing](../tests/integration/README.md) for more details. To execute integration tests, run:
 
 ```
 make test-integration
 ```
 
 **Note**: EC2 instance is required to run integration test, since it is exercising the actual flow of creating EBS volume, attaching it and read/write on the disk.
 
-To execute e2e tests:
+See [E2E Testing](../tests/e2e/README.md) for more details. To execute e2e tests, run:
 
-Some tests marked with `[env]` require specific environmental variables to be set, if not set these tests will be skipped.
-
-```
-export AWS_AVAILABILITY_ZONES="us-west-2a,us-west-2b"
-```
-
-Replacing `us-west-2a,us-west-2b` with the AZ(s) where your Kubernetes worker nodes are located.
-
-These tests also rely on having proper [AWS credentials](https://docs.aws.amazon.com/amazonswf/latest/awsrbflowguide/set-up-creds.html) set either via environmental variables or a credentials file.
-
-Finally run:
 ```
-export KUBECONFIG=~/.kube/config
-make test-e2e
+make test-e2e-single-az // executes single az test suite
+make test-e2e-multi-az // executes multi az test suite
 ```
 
-**Note**: By default `make test-e2e` will run 32 tests concurrently, set `GINKGO_NODES` to change the parallelism.
-
 ### Build and Publish Container Image
 
 Build image and push it with latest tag:

hack/additional-policies.yaml

@@ -0,0 +1,22 @@
+  additionalPolicies:
+    node: |
+      [
+        {
+          "Effect": "Allow",
+          "Action": [
+            "ec2:AttachVolume",
+            "ec2:CreateSnapshot",
+            "ec2:CreateTags",
+            "ec2:CreateVolume",
+            "ec2:DeleteSnapshot",
+            "ec2:DeleteTags",
+            "ec2:DeleteVolume",
+            "ec2:DescribeInstances",
+            "ec2:DescribeSnapshots",
+            "ec2:DescribeTags",
+            "ec2:DescribeVolumes",
+            "ec2:DetachVolume"
+          ],
+          "Resource": "*"
+        }
+      ]

hack/feature-gates.yaml

@@ -0,0 +1,8 @@
+  kubeAPIServer:
+    featureGates:
+      CSIDriverRegistry: "true"
+      CSINodeInfo: "true"
+  kubelet:
+    featureGates:
+      CSIDriverRegistry: "true"
+      CSINodeInfo: "true"

hack/run-e2e-test

@@ -14,8 +14,93 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -euo pipefail
+set -uo pipefail
 
+OS_ARCH=$(go env GOOS)-amd64
+TEST_ID=$RANDOM
+CLUSTER_NAME=test-cluster-$TEST_ID
+TEST_DIR=/tmp/ebs-e2e-test
+BASE_DIR=$(dirname $0)
+REGION=${AWS_REGION-us-east-1}
+ZONES=${AWS_AVAILABILITY_ZONES-us-east-1a,us-east-1b,us-east-1c}
+FOCUS=${GINKGO_FOCUS-"[ebs-csi-e2e]"}
 NODES=${GINKGO_NODES:-32}
 
-ginkgo -p -nodes=$NODES -v --focus ebs-csi-e2e tests/e2e
+echo "Testing in region: $REGION and zones: $ZONES"
+
+KOPS_DOWNLOAD_URL=https://github.com/kubernetes/kops/releases/download/1.11.0/kops-$OS_ARCH
+KOPS_PATH=$TEST_DIR/kops
+KOPS_STATE_FILE=s3://k8s-kops-csi-e2e
+
+# Download kops if not yet
+if [[ ! -e $KOPS_PATH ]]; then
+    mkdir -p $TEST_DIR 
+    echo "Downloading KOPS from $KOPS_DOWNLOAD_URL to $KOPS_PATH"
+    curl -L -X GET $KOPS_DOWNLOAD_URL -o $KOPS_PATH
+fi
+
+chmod +x $KOPS_PATH
+
+echo "Creating cluster $CLUSTER_NAME"
+CLUSTER_YAML_PATH=$TEST_DIR/$CLUSTER_NAME.yaml
+$KOPS_PATH create cluster --state $KOPS_STATE_FILE \
+    --zones $ZONES \
+    --node-count=3 \
+    --kubernetes-version=1.13.0 \
+        $CLUSTER_NAME.k8s.local
+$KOPS_PATH get cluster --state $KOPS_STATE_FILE $CLUSTER_NAME.k8s.local -o yaml > $CLUSTER_YAML_PATH
+cat $BASE_DIR/feature-gates.yaml >> $CLUSTER_YAML_PATH
+cat $BASE_DIR/additional-policies.yaml >> $CLUSTER_YAML_PATH
+$KOPS_PATH replace --state $KOPS_STATE_FILE -f $CLUSTER_YAML_PATH
+$KOPS_PATH update cluster --state $KOPS_STATE_FILE $CLUSTER_NAME.k8s.local --yes
+
+# Wait for cluster creation
+while [[ 1 ]]; do
+    $KOPS_PATH validate cluster --state $KOPS_STATE_FILE
+    ret=$?
+    if [[ $ret -eq 0 ]]; then
+        break
+    else
+        echo "Waiting cluster to be created"
+        sleep 30
+    fi
+done;
+
+# Push test driver image
+eval $(aws ecr get-login --region $REGION --no-include-email)
+AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
+IMAGE_TAG=$TEST_ID
+IMAGE_NAME=$AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/aws-ebs-csi-driver:$IMAGE_TAG
+docker build -t $IMAGE_NAME .
+docker push $IMAGE_NAME
+
+# Update manifest files
+cp deploy/kubernetes/*.yaml $TEST_DIR
+sed -i '' "s/image: amazon\/aws-ebs-csi-driver:latest/image: $AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com\/aws-ebs-csi-driver:$IMAGE_TAG/g" $TEST_DIR/controller.yaml
+sed -i '' "s/image: amazon\/aws-ebs-csi-driver:latest/image: $AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com\/aws-ebs-csi-driver:$IMAGE_TAG/g" $TEST_DIR/node.yaml
+
+echo "Deploying driver"
+kubectl create -f https://raw.githubusercontent.com/kubernetes/csi-api/release-1.13/pkg/crd/manifests/csinodeinfo.yaml --validate=false
+kubectl apply -f $TEST_DIR/controller.yaml 
+kubectl apply -f $TEST_DIR/node.yaml 
+kubectl apply -f $TEST_DIR/secret.yaml
+
+# Run the test
+go install -v github.com/onsi/ginkgo/ginkgo
+export KUBECONFIG=$HOME/.kube/config
+ginkgo -p -nodes=$NODES -v --focus="$FOCUS" tests/e2e
+TEST_PASS=$?
+
+echo "Removing driver"
+kubectl delete -f $TEST_DIR/controller.yaml
+kubectl delete -f $TEST_DIR/node.yaml
+kubectl delete -f $TEST_DIR/secret.yaml
+
+echo "Deleting cluster $CLUSTER_NAME"
+$KOPS_PATH delete cluster --name $CLUSTER_NAME.k8s.local --state $KOPS_STATE_FILE --yes
+
+rm -rf $TEST_DIR
+
+if [[ $TEST_PASS -ne 0 ]]; then
+    exit 1
+fi

tests/e2e/README.md

@@ -0,0 +1,24 @@
+## E2E Testing
+E2E test verifies the funcitonality of EBS CSI driver in the context of Kubernetes. It exercises driver feature e2e including static provisioning, dynamic provisioning, volume scheduling, mount options, etc.
+
+### Requirements
+1. AWS credential is [configured](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html)
+1. AWS CLI v1.16+
+1. Kubectl v1.13+
+1. Docker CLI v18.09+
+1. curl
+1. sed
+1. Golang 1.11+
+
+### Notes
+Some tests marked with `[env]` require specific environmental variables to be set, if not set these tests will be skipped.
+
+```
+export AWS_AVAILABILITY_ZONES="us-west-2a,us-west-2b"
+```
+
+Replacing `us-west-2a,us-west-2b` with the AZ(s) where your Kubernetes worker nodes are located.
+
+By default `make test-e2e-` targets will run 32 tests concurrently, set `GINKGO_NODES` to change the parallelism.
+
+