diff --git a/Makefile b/Makefile index 915c597780..7c876f253a 100644 --- a/Makefile +++ b/Makefile @@ -56,9 +56,13 @@ test-integration: chmod +x ${AWS_K8S_TESTER_PATH} ${AWS_K8S_TESTER_PATH} csi test integration --terminate-on-exit=true --timeout=20m ${PR_NUM_FLAG} ${VPC_ID_FLAG} -.PHONY: test-e2e -test-e2e: - ./hack/run-e2e-test +.PHONY: test-e2e-single-az +test-e2e-single-az: + AWS_REGION=us-west-2 AWS_AVAILABILITY_ZONES=us-west-2a GINKGO_FOCUS="\[ebs-csi-e2e\] \[single-az\]" ./hack/run-e2e-test + +.PHONY: test-e2e-multi-az +test-e2e-multi-az: + AWS_REGION=us-west-2 AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c GINKGO_FOCUS="\[ebs-csi-e2e\] \[multi-az\]" ./hack/run-e2e-test .PHONY: image-release image-release: diff --git a/docs/README.md b/docs/README.md index d17906883e..cfba82952e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -122,15 +122,16 @@ In order to make sure that the driver complies with the CSI specification, run: make test-sanity ``` -To execute integration tests, run: +See [Ingetration Testing](../tests/integration/README.md) for more details. To execute integration tests, run: ``` make test-integration ``` -To execute e2e tests, run: +See [E2E Testing](../tests/e2e/README.md) for more details. To execute e2e tests, run: ``` -make test-e2e +make test-e2e-single-az // executes single az test suite +make test-e2e-multi-az // executes multi az test suite ``` **Note**: EC2 instance is required to run integration test, since it is exercising the actual flow of creating EBS volume, attaching it and read/write on the disk. diff --git a/hack/additional-policies.yaml b/hack/additional-policies.yaml new file mode 100644 index 0000000000..b183a57776 --- /dev/null +++ b/hack/additional-policies.yaml @@ -0,0 +1,22 @@ + additionalPolicies: + node: | + [ + { + "Effect": "Allow", + "Action": [ + "ec2:AttachVolume", + "ec2:CreateSnapshot", + "ec2:CreateTags", + "ec2:CreateVolume", + "ec2:DeleteSnapshot", + "ec2:DeleteTags", + "ec2:DeleteVolume", + "ec2:DescribeInstances", + "ec2:DescribeSnapshots", + "ec2:DescribeTags", + "ec2:DescribeVolumes", + "ec2:DetachVolume" + ], + "Resource": "*" + } + ] diff --git a/hack/feature-gates.yaml b/hack/feature-gates.yaml new file mode 100644 index 0000000000..19f0e33a97 --- /dev/null +++ b/hack/feature-gates.yaml @@ -0,0 +1,8 @@ + kubeAPIServer: + featureGates: + CSIDriverRegistry: "true" + CSINodeInfo: "true" + kubelet: + featureGates: + CSIDriverRegistry: "true" + CSINodeInfo: "true" diff --git a/hack/run-e2e-test b/hack/run-e2e-test index f4f94cd886..c3f2afe3f8 100755 --- a/hack/run-e2e-test +++ b/hack/run-e2e-test @@ -14,6 +14,93 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -euo pipefail +set -uo pipefail -ginkgo -p -v --focus ebs-csi-e2e tests/e2e +OS_ARCH=$(go env GOOS)-amd64 +TEST_ID=$RANDOM +CLUSTER_NAME=test-cluster-$TEST_ID +TEST_DIR=/tmp/ebs-e2e-test +BASE_DIR=$(dirname $0) +REGION=${AWS_REGION-us-east-1} +ZONES=${AWS_AVAILABILITY_ZONES-us-east-1a,us-east-1b,us-east-1c} +FOCUS=${GINKGO_FOCUS-"[ebs-csi-e2e]"} + +echo "Testing in region: $REGION and zones: $ZONES" + +KOPS_DOWNLOAD_URL=https://github.com/kubernetes/kops/releases/download/1.11.0/kops-$OS_ARCH +KOPS_PATH=$TEST_DIR/kops +KOPS_STATE_FILE=s3://k8s-kops-csi-e2e + +# Download kops if not yet +if [[ ! -e $KOPS_PATH ]]; then + mkdir -p $TEST_DIR + echo "Downloading KOPS from $KOPS_DOWNLOAD_URL to $KOPS_PATH" + curl -L -X GET $KOPS_DOWNLOAD_URL -o $KOPS_PATH +fi + +chmod +x $KOPS_PATH + +echo "Creating cluster $CLUSTER_NAME" +CLUSTER_YAML_PATH=$TEST_DIR/$CLUSTER_NAME.yaml +$KOPS_PATH create cluster --state $KOPS_STATE_FILE \ + --zones $ZONES \ + --node-count=3 \ + --kubernetes-version=1.13.0 \ + $CLUSTER_NAME.k8s.local +$KOPS_PATH get cluster --state $KOPS_STATE_FILE $CLUSTER_NAME.k8s.local -o yaml > $CLUSTER_YAML_PATH +cat $BASE_DIR/feature-gates.yaml >> $CLUSTER_YAML_PATH +cat $BASE_DIR/additional-policies.yaml >> $CLUSTER_YAML_PATH +$KOPS_PATH replace --state $KOPS_STATE_FILE -f $CLUSTER_YAML_PATH +$KOPS_PATH update cluster --state $KOPS_STATE_FILE $CLUSTER_NAME.k8s.local --yes + +# Wait for cluster creation +while [[ 1 ]]; do + $KOPS_PATH validate cluster --state $KOPS_STATE_FILE + ret=$? + if [[ $ret -eq 0 ]]; then + break + else + echo "Waiting cluster to be created" + sleep 30 + fi +done; + +# Push test driver image +eval $(aws ecr get-login --region $REGION --no-include-email) +AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) +IMAGE_TAG=$TEST_ID +IMAGE_NAME=$AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/aws-ebs-csi-driver:$IMAGE_TAG +docker build -t $IMAGE_NAME . +docker push $IMAGE_NAME + +# Update manifest files +cp deploy/kubernetes/*.yaml $TEST_DIR +sed -i '' "s/image: amazon\/aws-ebs-csi-driver:latest/image: $AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com\/aws-ebs-csi-driver:$IMAGE_TAG/g" $TEST_DIR/controller.yaml +sed -i '' "s/image: amazon\/aws-ebs-csi-driver:latest/image: $AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com\/aws-ebs-csi-driver:$IMAGE_TAG/g" $TEST_DIR/node.yaml + +echo "Deploying driver" +kubectl create -f https://raw.githubusercontent.com/kubernetes/csi-api/release-1.13/pkg/crd/manifests/csinodeinfo.yaml --validate=false +kubectl apply -f $TEST_DIR/controller.yaml +kubectl apply -f $TEST_DIR/node.yaml +kubectl apply -f $TEST_DIR/secret.yaml + +# Run the test +go install -v github.com/onsi/ginkgo/ginkgo +export KUBECONFIG=$HOME/.kube/config +ginkgo -p -v --focus="$FOCUS" tests/e2e +TEST_PASS=$? + +echo "Removing driver" +kubectl delete -f $TEST_DIR/controller.yaml +kubectl delete -f $TEST_DIR/node.yaml +kubectl delete -f $TEST_DIR/secret.yaml + +echo "Deleting cluster $CLUSTER_NAME" +$KOPS_PATH delete cluster --name $CLUSTER_NAME.k8s.local --state $KOPS_STATE_FILE --yes + +rm -rf $TEST_DIR + +if [[ $TEST_PASS -ne 0 ]]; then + exit 1 +fi + diff --git a/tests/e2e/README.md b/tests/e2e/README.md new file mode 100644 index 0000000000..4bbffe74d2 --- /dev/null +++ b/tests/e2e/README.md @@ -0,0 +1,12 @@ +## E2E Testing +E2E test verifies the funcitonality of EBS CSI driver in the context of Kubernetes. It exercises driver feature e2e including static provisioning, dynamic provisioning, volume scheduling, mount options, etc. + +### Requirements +1. AWS credential is [configured](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html) +1. AWS CLI v1.16+ +1. Kubectl v1.13+ +1. Docker CLI v18.09+ +1. curl +1. sed +1. Golang 1.11+ +