Merge pull request kubernetes-sigs#2138 from sedefsavas/capi-e2e

k8s-ci-robot · web-flow · commit 634f30eb5a3a · 2021-01-22T03:59:26.000-08:00
🌱Add CAPI E2E tests
diff --git a/Makefile b/Makefile
@@ -118,6 +118,8 @@ LDFLAGS := $(shell source ./hack/version.sh; version::ldflags)
 E2E_UNMANAGED_FOCUS ?= "functional tests - unmanaged"
 # Instead, you can run a quick smoke test, it should run fast (9 minutes)...
 # E2E_UNMANAGED_FOCUS := "Create cluster with name having"
+# For running CAPI e2e tests: E2E_UNMANAGED_FOCUS := "Cluster API E2E tests"
+USE_EXISTING_CLUSTER ?= "false"
 
 GINKGO_NODES ?= 2
 GINKGO_ARGS ?=
@@ -135,7 +137,7 @@ test: ## Run tests
 
 .PHONY: test-e2e ## Run e2e tests using clusterctl
 test-e2e: $(GINKGO) $(KIND) $(SSM_PLUGIN) $(KUSTOMIZE) e2e-image ## Run e2e tests
-	time $(GINKGO) -trace -progress -v -tags=e2e -focus=$(E2E_UNMANAGED_FOCUS) $(GINKGO_ARGS) ./test/e2e/suites/unmanaged/... -- -config-path="$(E2E_CONF_PATH)" -artifacts-folder="$(ARTIFACTS)" --data-folder="$(E2E_DATA_DIR)" $(E2E_ARGS)
+	time $(GINKGO) -trace -progress -v -tags=e2e -focus=$(E2E_UNMANAGED_FOCUS) $(GINKGO_ARGS) ./test/e2e/suites/unmanaged/... -- -config-path="$(E2E_CONF_PATH)" -artifacts-folder="$(ARTIFACTS)" --data-folder="$(E2E_DATA_DIR)" $(E2E_ARGS) -use-existing-cluster=$(USE_EXISTING_CLUSTER)
 
 .PHONY: test-e2e-eks ## Run EKS e2e tests using clusterctl
 test-e2e-eks: $(GINKGO) $(KIND) $(SSM_PLUGIN) $(KUSTOMIZE) e2e-image ## Run eks e2e tests
diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/cluster_api_controller.go b/cmd/clusterawsadm/cloudformation/bootstrap/cluster_api_controller.go
@@ -120,6 +120,7 @@ func (t Template) controllersPolicy() *iamv1.PolicyDocument {
 				"elasticloadbalancing:DeregisterInstancesFromLoadBalancer",
 				"elasticloadbalancing:RemoveTags",
 				"autoscaling:DescribeAutoScalingGroups",
+				"autoscaling:DescribeInstanceRefreshes",
 				"ec2:CreateLaunchTemplate",
 				"ec2:CreateLaunchTemplateVersion",
 				"ec2:DescribeLaunchTemplates",
@@ -137,10 +138,23 @@ func (t Template) controllersPolicy() *iamv1.PolicyDocument {
 				"autoscaling:CreateAutoScalingGroup",
 				"autoscaling:UpdateAutoScalingGroup",
 				"autoscaling:CreateOrUpdateTags",
+				"autoscaling:StartInstanceRefresh",
 				"autoscaling:DeleteAutoScalingGroup",
 				"autoscaling:DeleteTags",
 			},
 		},
+		{
+			Effect: iamv1.EffectAllow,
+			Resource: iamv1.Resources{
+				"arn:*:iam::*:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling",
+			},
+			Action: iamv1.Actions{
+				"iam:CreateServiceLinkedRole",
+			},
+			Condition: iamv1.Conditions{
+				iamv1.StringLike: map[string]string{"iam:AWSServiceName": "autoscaling.amazonaws.com"},
+			},
+		},
 		{
 			Effect: iamv1.EffectAllow,
 			Resource: iamv1.Resources{
diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/customsuffix.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/customsuffix.yaml
@@ -192,6 +192,7 @@ Resources:
           - elasticloadbalancing:DeregisterInstancesFromLoadBalancer
           - elasticloadbalancing:RemoveTags
           - autoscaling:DescribeAutoScalingGroups
+          - autoscaling:DescribeInstanceRefreshes
           - ec2:CreateLaunchTemplate
           - ec2:CreateLaunchTemplateVersion
           - ec2:DescribeLaunchTemplates
@@ -205,11 +206,20 @@ Resources:
           - autoscaling:CreateAutoScalingGroup
           - autoscaling:UpdateAutoScalingGroup
           - autoscaling:CreateOrUpdateTags
+          - autoscaling:StartInstanceRefresh
           - autoscaling:DeleteAutoScalingGroup
           - autoscaling:DeleteTags
           Effect: Allow
           Resource:
           - arn:aws:autoscaling:*:*:autoScalingGroup:*:autoScalingGroupName/*
+        - Action:
+          - iam:CreateServiceLinkedRole
+          Condition:
+            StringLike:
+              iam:AWSServiceName: autoscaling.amazonaws.com
+          Effect: Allow
+          Resource:
+          - arn:*:iam::*:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling
         - Action:
           - iam:CreateServiceLinkedRole
           Condition:
diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/default.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/default.yaml
@@ -192,6 +192,7 @@ Resources:
           - elasticloadbalancing:DeregisterInstancesFromLoadBalancer
           - elasticloadbalancing:RemoveTags
           - autoscaling:DescribeAutoScalingGroups
+          - autoscaling:DescribeInstanceRefreshes
           - ec2:CreateLaunchTemplate
           - ec2:CreateLaunchTemplateVersion
           - ec2:DescribeLaunchTemplates
@@ -205,11 +206,20 @@ Resources:
           - autoscaling:CreateAutoScalingGroup
           - autoscaling:UpdateAutoScalingGroup
           - autoscaling:CreateOrUpdateTags
+          - autoscaling:StartInstanceRefresh
           - autoscaling:DeleteAutoScalingGroup
           - autoscaling:DeleteTags
           Effect: Allow
           Resource:
           - arn:aws:autoscaling:*:*:autoScalingGroup:*:autoScalingGroupName/*
+        - Action:
+          - iam:CreateServiceLinkedRole
+          Condition:
+            StringLike:
+              iam:AWSServiceName: autoscaling.amazonaws.com
+          Effect: Allow
+          Resource:
+          - arn:*:iam::*:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling
         - Action:
           - iam:CreateServiceLinkedRole
           Condition:
diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_all_secret_backends.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_all_secret_backends.yaml
@@ -198,6 +198,7 @@ Resources:
           - elasticloadbalancing:DeregisterInstancesFromLoadBalancer
           - elasticloadbalancing:RemoveTags
           - autoscaling:DescribeAutoScalingGroups
+          - autoscaling:DescribeInstanceRefreshes
           - ec2:CreateLaunchTemplate
           - ec2:CreateLaunchTemplateVersion
           - ec2:DescribeLaunchTemplates
@@ -211,11 +212,20 @@ Resources:
           - autoscaling:CreateAutoScalingGroup
           - autoscaling:UpdateAutoScalingGroup
           - autoscaling:CreateOrUpdateTags
+          - autoscaling:StartInstanceRefresh
           - autoscaling:DeleteAutoScalingGroup
           - autoscaling:DeleteTags
           Effect: Allow
           Resource:
           - arn:aws:autoscaling:*:*:autoScalingGroup:*:autoScalingGroupName/*
+        - Action:
+          - iam:CreateServiceLinkedRole
+          Condition:
+            StringLike:
+              iam:AWSServiceName: autoscaling.amazonaws.com
+          Effect: Allow
+          Resource:
+          - arn:*:iam::*:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling
         - Action:
           - iam:CreateServiceLinkedRole
           Condition:
diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_bootstrap_user.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_bootstrap_user.yaml
@@ -197,6 +197,7 @@ Resources:
           - elasticloadbalancing:DeregisterInstancesFromLoadBalancer
           - elasticloadbalancing:RemoveTags
           - autoscaling:DescribeAutoScalingGroups
+          - autoscaling:DescribeInstanceRefreshes
           - ec2:CreateLaunchTemplate
           - ec2:CreateLaunchTemplateVersion
           - ec2:DescribeLaunchTemplates
@@ -210,11 +211,20 @@ Resources:
           - autoscaling:CreateAutoScalingGroup
           - autoscaling:UpdateAutoScalingGroup
           - autoscaling:CreateOrUpdateTags
+          - autoscaling:StartInstanceRefresh
           - autoscaling:DeleteAutoScalingGroup
           - autoscaling:DeleteTags
           Effect: Allow
           Resource:
           - arn:aws:autoscaling:*:*:autoScalingGroup:*:autoScalingGroupName/*
+        - Action:
+          - iam:CreateServiceLinkedRole
+          Condition:
+            StringLike:
+              iam:AWSServiceName: autoscaling.amazonaws.com
+          Effect: Allow
+          Resource:
+          - arn:*:iam::*:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling
         - Action:
           - iam:CreateServiceLinkedRole
           Condition:
diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_different_instance_profiles.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_different_instance_profiles.yaml
@@ -192,6 +192,7 @@ Resources:
           - elasticloadbalancing:DeregisterInstancesFromLoadBalancer
           - elasticloadbalancing:RemoveTags
           - autoscaling:DescribeAutoScalingGroups
+          - autoscaling:DescribeInstanceRefreshes
           - ec2:CreateLaunchTemplate
           - ec2:CreateLaunchTemplateVersion
           - ec2:DescribeLaunchTemplates
@@ -205,11 +206,20 @@ Resources:
           - autoscaling:CreateAutoScalingGroup
           - autoscaling:UpdateAutoScalingGroup
           - autoscaling:CreateOrUpdateTags
+          - autoscaling:StartInstanceRefresh
           - autoscaling:DeleteAutoScalingGroup
           - autoscaling:DeleteTags
           Effect: Allow
           Resource:
           - arn:aws:autoscaling:*:*:autoScalingGroup:*:autoScalingGroupName/*
+        - Action:
+          - iam:CreateServiceLinkedRole
+          Condition:
+            StringLike:
+              iam:AWSServiceName: autoscaling.amazonaws.com
+          Effect: Allow
+          Resource:
+          - arn:*:iam::*:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling
         - Action:
           - iam:CreateServiceLinkedRole
           Condition:
diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_default_roles.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_default_roles.yaml
@@ -192,6 +192,7 @@ Resources:
           - elasticloadbalancing:DeregisterInstancesFromLoadBalancer
           - elasticloadbalancing:RemoveTags
           - autoscaling:DescribeAutoScalingGroups
+          - autoscaling:DescribeInstanceRefreshes
           - ec2:CreateLaunchTemplate
           - ec2:CreateLaunchTemplateVersion
           - ec2:DescribeLaunchTemplates
@@ -205,11 +206,20 @@ Resources:
           - autoscaling:CreateAutoScalingGroup
           - autoscaling:UpdateAutoScalingGroup
           - autoscaling:CreateOrUpdateTags
+          - autoscaling:StartInstanceRefresh
           - autoscaling:DeleteAutoScalingGroup
           - autoscaling:DeleteTags
           Effect: Allow
           Resource:
           - arn:aws:autoscaling:*:*:autoScalingGroup:*:autoScalingGroupName/*
+        - Action:
+          - iam:CreateServiceLinkedRole
+          Condition:
+            StringLike:
+              iam:AWSServiceName: autoscaling.amazonaws.com
+          Effect: Allow
+          Resource:
+          - arn:*:iam::*:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling
         - Action:
           - iam:CreateServiceLinkedRole
           Condition:
diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_enable.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_enable.yaml
@@ -192,6 +192,7 @@ Resources:
           - elasticloadbalancing:DeregisterInstancesFromLoadBalancer
           - elasticloadbalancing:RemoveTags
           - autoscaling:DescribeAutoScalingGroups
+          - autoscaling:DescribeInstanceRefreshes
           - ec2:CreateLaunchTemplate
           - ec2:CreateLaunchTemplateVersion
           - ec2:DescribeLaunchTemplates
@@ -205,11 +206,20 @@ Resources:
           - autoscaling:CreateAutoScalingGroup
           - autoscaling:UpdateAutoScalingGroup
           - autoscaling:CreateOrUpdateTags
+          - autoscaling:StartInstanceRefresh
           - autoscaling:DeleteAutoScalingGroup
           - autoscaling:DeleteTags
           Effect: Allow
           Resource:
           - arn:aws:autoscaling:*:*:autoScalingGroup:*:autoScalingGroupName/*
+        - Action:
+          - iam:CreateServiceLinkedRole
+          Condition:
+            StringLike:
+              iam:AWSServiceName: autoscaling.amazonaws.com
+          Effect: Allow
+          Resource:
+          - arn:*:iam::*:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling
         - Action:
           - iam:CreateServiceLinkedRole
           Condition:
diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_extra_statements.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_extra_statements.yaml
@@ -192,6 +192,7 @@ Resources:
           - elasticloadbalancing:DeregisterInstancesFromLoadBalancer
           - elasticloadbalancing:RemoveTags
           - autoscaling:DescribeAutoScalingGroups
+          - autoscaling:DescribeInstanceRefreshes
           - ec2:CreateLaunchTemplate
           - ec2:CreateLaunchTemplateVersion
           - ec2:DescribeLaunchTemplates
@@ -205,11 +206,20 @@ Resources:
           - autoscaling:CreateAutoScalingGroup
           - autoscaling:UpdateAutoScalingGroup
           - autoscaling:CreateOrUpdateTags
+          - autoscaling:StartInstanceRefresh
           - autoscaling:DeleteAutoScalingGroup
           - autoscaling:DeleteTags
           Effect: Allow
           Resource:
           - arn:aws:autoscaling:*:*:autoScalingGroup:*:autoScalingGroupName/*
+        - Action:
+          - iam:CreateServiceLinkedRole
+          Condition:
+            StringLike:
+              iam:AWSServiceName: autoscaling.amazonaws.com
+          Effect: Allow
+          Resource:
+          - arn:*:iam::*:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling
         - Action:
           - iam:CreateServiceLinkedRole
           Condition:
diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_ssm_secret_backend.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_ssm_secret_backend.yaml
@@ -192,6 +192,7 @@ Resources:
           - elasticloadbalancing:DeregisterInstancesFromLoadBalancer
           - elasticloadbalancing:RemoveTags
           - autoscaling:DescribeAutoScalingGroups
+          - autoscaling:DescribeInstanceRefreshes
           - ec2:CreateLaunchTemplate
           - ec2:CreateLaunchTemplateVersion
           - ec2:DescribeLaunchTemplates
@@ -205,11 +206,20 @@ Resources:
           - autoscaling:CreateAutoScalingGroup
           - autoscaling:UpdateAutoScalingGroup
           - autoscaling:CreateOrUpdateTags
+          - autoscaling:StartInstanceRefresh
           - autoscaling:DeleteAutoScalingGroup
           - autoscaling:DeleteTags
           Effect: Allow
           Resource:
           - arn:aws:autoscaling:*:*:autoScalingGroup:*:autoScalingGroupName/*
+        - Action:
+          - iam:CreateServiceLinkedRole
+          Condition:
+            StringLike:
+              iam:AWSServiceName: autoscaling.amazonaws.com
+          Effect: Allow
+          Resource:
+          - arn:*:iam::*:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling
         - Action:
           - iam:CreateServiceLinkedRole
           Condition:
diff --git a/test/e2e/data/e2e_conf.yaml b/test/e2e/data/e2e_conf.yaml
@@ -105,26 +105,36 @@ providers:
         targetName: "cluster-template-limit-az.yaml"
       - sourcePath: "./infrastructure-aws/cluster-template-spot-instances.yaml"
         targetName: "cluster-template-spot-instances.yaml"
-
+      - sourcePath: "./infrastructure-aws/cluster-template-mhc.yaml"
+        targetName: "cluster-template-mhc.yaml"
+      - sourcePath: "./infrastructure-aws/cluster-template-machine-pool.yaml"
+        targetName: "cluster-template-machine-pool.yaml"
 variables:
-  KUBERNETES_VERSION: "v1.19.4"
+  KUBERNETES_VERSION: "v1.20.1"
   CNI: "../../data/cni/calico.yaml"
   EXP_CLUSTER_RESOURCE_SET: "true"
   AWS_CONTROL_PLANE_MACHINE_TYPE: t3.large
   AWS_NODE_MACHINE_TYPE: t3.large
   AWS_SSH_KEY_NAME: "cluster-api-provider-aws-sigs-k8s-io"
-  CONFORMANCE_CI_ARTIFACTS_KUBERNETES_VERSION: "v1.19.4"
+  CONFORMANCE_CI_ARTIFACTS_KUBERNETES_VERSION: "v1.20.1"
   CONFORMANCE_WORKER_MACHINE_COUNT: "5"
   CONFORMANCE_CONTROL_PLANE_MACHINE_COUNT: "1"
+  EXP_MACHINE_POOL: "true"
+  ETCD_VERSION_UPGRADE_TO: "3.4.13-0"
+  COREDNS_VERSION_UPGRADE_TO: "1.7.1"
+  KUBERNETES_VERSION_UPGRADE_TO: "v1.20.1"
+  KUBERNETES_VERSION_UPGRADE_FROM: "v1.19.6"
 
 intervals:
-  default/wait-cluster: ["20m", "10s"]
-  default/wait-control-plane: ["10m", "10s"]
-  default/wait-worker-nodes: ["10m", "10s"]
-  conformance/wait-control-plane: ["30m", "10s"]
-  conformance/wait-worker-nodes: ["30m", "10s"]
-  default/wait-controllers: ["3m", "10s"]
-  default/wait-delete-cluster: ["20m", "10s"]
-  default/wait-machine-upgrade: ["20m", "10s"]
-  default/wait-machine-status: ["20m", "10s"]
+  default/wait-cluster: ["30m", "10s"]
+  default/wait-control-plane: ["40m", "10s"]
+  default/wait-worker-nodes: ["40m", "10s"]
+  conformance/wait-control-plane: ["40m", "10s"]
+  conformance/wait-worker-nodes: ["40m", "10s"]
+  default/wait-controllers: ["40m", "10s"]
+  default/wait-delete-cluster: ["40m", "10s"]
+  default/wait-machine-upgrade: ["40m", "10s"]
+  default/wait-machine-status: ["40m", "10s"]
   default/wait-infra-subnets: ["5m", "30s"]
+  default/wait-machine-pool-nodes: ["40m", "10s"]
+  default/wait-machine-pool-upgrade: [ "50m", "10s" ]
diff --git a/test/e2e/data/infrastructure-aws/cluster-template-machine-pool.yaml b/test/e2e/data/infrastructure-aws/cluster-template-machine-pool.yaml
diff --git a/test/e2e/data/infrastructure-aws/cluster-template-mhc.yaml b/test/e2e/data/infrastructure-aws/cluster-template-mhc.yaml
diff --git a/test/e2e/suites/unmanaged/unmanaged_CAPI_test.go b/test/e2e/suites/unmanaged/unmanaged_CAPI_test.go
diff --git a/test/e2e/suites/unmanaged/unmanaged_test.go b/test/e2e/suites/unmanaged/unmanaged_test.go
