Update to v1beta1

* Graduate CRDs to v1beta1
* IAM types moved to /iam/api/v1beta1
* Graduate clusterawsadm configuration types to v1beta1
* Add conversions for v1alpha3 & v1alpha4 --> v1beta1 CRDs
* Add conversion for clusterawsadm configuration from v1alpha1 to v1beta1
* Improve error handling for clusterawsadm when incorrect configuration is passed
* Add defaulter-gen to all types
* Split up /api/**/conversion.go into ones for separate types
* Controllers will now recover from panics, allowing webhooks to continue running
* Remove kustomize generated e2e test flavors and add to Makefile targets
* Ensure CRD docs generation actually occurs
* Rudimentary build caching for API go files
* conversion-gen was forked into hack/tools/third_party to remove incorrect duplicate static function warnings
* API packages given consistent naming and set throughout the codebase. golangci-lint configuration updated to match

Co-authored-by: Shivani Singhal <sishivani@vmware.com>
Co-authored-by: Naadir Jeewa <jeewan@vmware.com>
Co-authored-by: Sedef Savas <ssavas@vmware.com>

.gitignore

@@ -31,12 +31,8 @@ kubeconfig
 # Example and binary output directory
 /out
 
-# bazel
-/bazel-bin
-/bazel-cluster-api-provider-aws
-/bazel-genfiles
-/bazel-out
-/bazel-testlogs
+# build
+.build
 
 # vscode
 .vscode

.golangci.yml

@@ -22,7 +22,6 @@ linters:
   - ifshort
   - importas
   - ineffassign
-  - maligned
   - misspell
   - nakedret
   - nilerr
@@ -53,9 +52,8 @@ linters-settings:
     # Maximum length of variable declaration measured in number of characters, after which linter won't suggest using short syntax.
     max-decl-chars: 50
   importas:
-    no-unaliased: true
+    no-unaliased: false
     alias:
-      # Kubernetes
       - pkg: k8s.io/api/core/v1
         alias: corev1
       - pkg: k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1
@@ -66,9 +64,88 @@ linters-settings:
         alias: apierrors
       - pkg: k8s.io/apimachinery/pkg/util/errors
         alias: kerrors
-      # Controller Runtime
+      - pkg: sigs.k8s.io/controller-runtime/pkg/conversion
+        alias: ctrlconversion
+      - pkg: sigs.k8s.io/cluster-api/util/conversion
+        alias: utilconversion
+      - pkg: k8s.io/apimachinery/pkg/conversion
+        alias: apiconversion
+      - pkg: sigs.k8s.io/cluster-api-provider-aws/api/v1beta1
+        alias: infrav1
+      - pkg: sigs.k8s.io/cluster-api-provider-aws/exp/api/v1alpha3
+        alias: expinfrav1alpha3
+      - pkg: sigs.k8s.io/cluster-api-provider-aws/exp/api/v1alpha4
+        alias: expinfrav1alpha4
+      - pkg: sigs.k8s.io/cluster-api-provider-aws/exp/api/v1beta1
+        alias: expinfrav1
+      - pkg: sigs.k8s.io/cluster-api-provider-aws/api/v1alpha4
+        alias: infrav1alpha4
+      - pkg: sigs.k8s.io/cluster-api-provider-aws/api/v1alpha3
+        alias: infrav1alpha3
+      - pkg: k8s.io/client-go/kubernetes/scheme
+        alias: cgscheme
+      - pkg: k8s.io/client-go/tools/record
+        alias: cgrecord
+      - pkg: sigs.k8s.io/cluster-api-provider-aws/bootstrap/eks/api/v1alpha3
+        alias: eksbootstrapv1alpha3
+      - pkg: sigs.k8s.io/cluster-api-provider-aws/bootstrap/eks/api/v1alpha4
+        alias: eksbootstrapv1alpha4
+      - pkg: sigs.k8s.io/cluster-api-provider-aws/bootstrap/eks/api/v1beta1
+        alias: eksbootstrapv1
+      - pkg: sigs.k8s.io/cluster-api-provider-aws/controlplane/eks/api/v1alpha3
+        alias: ekscontrolplanev1alpha3
+      - pkg: sigs.k8s.io/cluster-api-provider-aws/controlplane/eks/api/v1alpha4
+        alias: ekscontrolplanev1alpha4
+      - pkg: sigs.k8s.io/cluster-api-provider-aws/controlplane/eks/api/v1beta1
+        alias: ekscontrolplanev1
+      - pkg: "sigs.k8s.io/cluster-api-provider-aws/bootstrap/eks/controllers"
+        alias: eksbootstrapcontrollers
+      - pkg: "sigs.k8s.io/cluster-api-provider-aws/controlplane/eks/controllers"
+        alias: ekscontrolplanecontrollers
+      - pkg: "sigs.k8s.io/cluster-api-provider-aws/exp/controllers"
+        alias: expcontrollers
+      - pkg: "k8s.io/apimachinery/pkg/runtime"
+        alias: runtime
+      - pkg: "k8s.io/apimachinery/pkg/runtime/serializer"
+        alias: runtimeserializer
+      - pkg: "k8s.io/apimachinery/pkg/runtime/serializer/yaml"
+        alias: yamlserializer
+      - pkg:  "sigs.k8s.io/cluster-api/api/v1alpha3"
+        alias: clusterv1alpha3
+      - pkg:  "sigs.k8s.io/cluster-api/api/v1alpha4"
+        alias: clusterv1alpha4
+      - pkg: "sigs.k8s.io/cluster-api/api/v1beta1"
+        alias: clusterv1
+      - pkg: "sigs.k8s.io/cluster-api/util/defaulting"
+        alias: utildefaulting
       - pkg: sigs.k8s.io/controller-runtime
         alias: ctrl
+      - pkg: "k8s.io/apimachinery/pkg/api/errors"
+        alias: apierrors
+      - pkg: "sigs.k8s.io/controller-runtime/pkg/log"
+        alias: logf
+      - pkg: "github.com/google/gofuzz"
+        alias: fuzz
+      - pkg: "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/logs"
+        alias: awslogs
+      - pkg: "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/metrics"
+        alias: awsmetrics
+      - pkg: "sigs.k8s.io/cluster-api/errors"
+        alias: capierrors
+      - pkg: "github.com/aws/aws-sdk-go/aws/client"
+        alias: awsclient
+      - pkg:  "github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1"
+        alias: amazoncni
+      - pkg: "github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
+        alias: rgapi
+      - pkg: "crypto/rand"
+        alias: crand
+      - pkg: "sigs.k8s.io/controller-runtime/pkg/client"
+        alias: crclient
+      - pkg: "k8s.io/apimachinery/pkg/types"
+        alias: apimachinerytypes
+      - pkg: "sigs.k8s.io/cluster-api/exp/api/v1beta1"
+        alias: expclusterv1
   staticcheck:
     go: "1.16"
   stylecheck:
@@ -128,10 +205,10 @@ issues:
     - linters:
       - revive
       source: _ "embed"
-    # This directive allows the variable in defaults.go files to have underscore   
+    # This directive allows the variable in defaults.go files to have underscore
     - linters:
       - revive
-      text: "var-naming: don't use underscores in Go names; func (.+) should be (.+)" 
+      text: "var-naming: don't use underscores in Go names; func (.+) should be (.+)"
       path: .*/defaults.go
     # Disable unparam "always receives" which might not be really
     # useful when building libraries.
@@ -140,7 +217,7 @@ issues:
       text: always receives
     - linters:
       - unparam
-      text: (.+) - (`t`|`g`) is unused  
+      text: (.+) - (`t`|`g`) is unused
     - path: _test\.go
       text: cyclomatic complexity
     # Append should be able to assign to a different var/slice.
@@ -155,4 +232,4 @@ run:
   - ".*conversion.*\\.go$"
   skip-dirs:
   - third_party
-  allow-parallel-runners: true
+  allow-parallel-runners: true

Makefile

@@ -25,6 +25,9 @@ TOOLS_DIR := hack/tools
 TOOLS_DIR_DEPS := $(TOOLS_DIR)/go.sum $(TOOLS_DIR)/go.mod $(TOOLS_DIR)/Makefile
 TOOLS_BIN_DIR := $(TOOLS_DIR)/bin
 
+API_DIRS := cmd/clusterawsadm/api api exp/api controlplane/eks/api bootstrap/eks/api iam/api
+API_SRCS := $(foreach dir, $(API_DIRS), $(call rwildcard,../../$(dir),*.go))
+
 BIN_DIR := bin
 REPO_ROOT := $(shell git rev-parse --show-toplevel)
 GH_ORG_NAME ?= kubernetes-sigs
@@ -123,15 +126,6 @@ E2E_SKIP_EKS_UPGRADE ?= "true"
 # Set EKS_SOURCE_TEMPLATE to override the source template
 EKS_SOURCE_TEMPLATE ?= eks/cluster-template-eks-control-plane-only.yaml
 
-#### We are disable Cluster API Framework tests for the time being for lack of resources
-# With framework tests enables, tests exceed the 4 hour timeout.
-GINKGO_SKIP ?= \[Cluster API Framework\]
-
-# If someone sets an explicit focus for Cluster API Framework, remove the skip
-ifeq ($(findstring \[Cluster API Framework\],$(E2E_FOCUS)),\[Cluster API Framework\])
-  override undefine GINKGO_SKIP
-endif
-
 # Enable Cluster API Framework tests for the purposes of running the PR blocking test
 ifeq ($(findstring \[PR-Blocking\],$(E2E_FOCUS)),\[PR-Blocking\])
   override undefine GINKGO_SKIP
@@ -183,11 +177,11 @@ generate-test-flavors: $(KUSTOMIZE)  ## Generate test template flavors
 	./hack/gen-test-flavors.sh
 
 .PHONY: test-e2e ## Run e2e tests using clusterctl
-test-e2e: $(GINKGO) $(KIND) $(SSM_PLUGIN) $(KUSTOMIZE) e2e-image ## Run e2e tests
+test-e2e: $(GINKGO) $(KIND) $(SSM_PLUGIN) $(KUSTOMIZE) generate-test-flavors e2e-image ## Run e2e tests
 	time $(GINKGO) -tags=e2e $(GINKGO_ARGS) -p ./test/e2e/suites/unmanaged/... -- -config-path="$(E2E_CONF_PATH)" $(E2E_ARGS)
 
 .PHONY: test-e2e-eks ## Run EKS e2e tests using clusterctl
-test-e2e-eks: $(GINKGO) $(KIND) $(SSM_PLUGIN) $(KUSTOMIZE) e2e-image ## Run eks e2e tests
+test-e2e-eks: generate-test-flavors  $(GINKGO) $(KIND) $(SSM_PLUGIN) $(KUSTOMIZE) e2e-image ## Run eks e2e tests
 	time $(GINKGO) -tags=e2e $(GINKGO_ARGS) ./test/e2e/suites/managed/... -- -config-path="$(E2E_EKS_CONF_PATH)" --source-template="$(EKS_SOURCE_TEMPLATE)" $(E2E_ARGS) $(EKS_E2E_ARGS)
 
 .PHONY: e2e-image
@@ -198,7 +192,7 @@ CONFORMANCE_E2E_ARGS ?= -kubetest.config-file=$(KUBETEST_CONF_PATH)
 CONFORMANCE_E2E_ARGS += $(E2E_ARGS)
 CONFORMANCE_GINKGO_ARGS += $(GINKGO_ARGS)
 .PHONY: test-conformance
-test-conformance: $(GINKGO) $(KIND) $(SSM_PLUGIN) $(KUSTOMIZE) e2e-image ## Run clusterctl based conformance test on workload cluster (requires Docker).
+test-conformance: generate-test-flavors $(GINKGO) $(KIND) $(SSM_PLUGIN) $(KUSTOMIZE) e2e-image ## Run clusterctl based conformance test on workload cluster (requires Docker).
 	time $(GINKGO) -tags=e2e -focus="conformance" $(CONFORMANCE_GINKGO_ARGS) ./test/e2e/suites/conformance/... -- -config-path="$(E2E_CONF_PATH)" $(CONFORMANCE_E2E_ARGS)
 
 .PHONY: test-cover
@@ -244,88 +238,99 @@ modules: ## Runs go mod to ensure proper vendoring.
 .PHONY: generate
 generate: ## Generate code
 	$(MAKE) generate-go
-	$(MAKE) generate-manifests
-	$(MAKE) generate-test-flavors
 	$(MAKE) $(CRD_DOCS)
 
-.PHONY: $(CRD_DOCS_DIR)/%
-$(CRD_DOCS_DIR)/%:
+$(CRD_DOCS_DIR)/%: $(API_SRCS)
 	$(MAKE) -C docs/book src/crd/$*
 
 .PHONY: generate-go
 generate-go: $(MOCKGEN)
-	$(MAKE) generate-go-core
-	$(MAKE) generate-go-eks-bootstrap
-	$(MAKE) generate-go-eks-controlplane
+	$(MAKE) generate-go-apis
 	go generate ./...
 
-.PHONY: generate-go-core
-generate-go-core: ## Runs Go related generate targets
-	$(MAKE) -B $(CONTROLLER_GEN) $(DEFAULTER_GEN) $(CONVERSION_GEN)
+.PHONY: generate-go-apis
+generate-go-apis: ## Runs Go related generate targets
+	$(MAKE) .build/generate-go-apis
+
+.build:
+	mkdir -p .build
+
+.build/generate-go-apis: .build $(API_SRCS) $(CONTROLLER_GEN) $(DEFAULTER_GEN) $(CONVERSION_GEN)
 	$(CONTROLLER_GEN) \
 		paths=./api/... \
 		paths=./$(EXP_DIR)/api/... \
-		object:headerFile=./hack/boilerplate/boilerplate.generatego.txt
+		paths=./bootstrap/eks/api/... \
+		paths=./controlplane/eks/api/... \
+		paths=./iam/api/... \
+		output:crd:dir=config/crd/bases \
+		object:headerFile=./hack/boilerplate/boilerplate.generatego.txt \
+		crd:crdVersions=v1 \
+		rbac:roleName=manager-role \
+		webhook
 
 	$(CONTROLLER_GEN) \
-		paths=./cmd/clusterawsadm/api/... \
+		paths=./cmd/... \
 		object:headerFile=./hack/boilerplate/boilerplate.generatego.txt
 
-	$(DEFAULTER_GEN) \
-		--input-dirs=./cmd/clusterawsadm/api/bootstrap/v1alpha1,./cmd/clusterawsadm/api/iam/v1alpha1 \
-		--v=0 $(GEN_OUTPUT_BASE) \
-		--go-header-file=./hack/boilerplate/boilerplate.generatego.txt
+	$(MAKE) defaulters
 
 	$(CONVERSION_GEN) \
 		--input-dirs=./api/v1alpha3 \
+		--input-dirs=./api/v1alpha4 \
+		--input-dirs=./cmd/clusterawsadm/api/bootstrap/v1alpha1 \
 		--extra-peer-dirs=sigs.k8s.io/cluster-api/api/v1alpha3 \
-		--build-tag=ignore_autogenerated_core_v1alpha3 \
+		--extra-peer-dirs=sigs.k8s.io/cluster-api/api/v1alpha4 \
+		--build-tag=ignore_autogenerated_conversions \
 		--output-file-base=zz_generated.conversion $(GEN_OUTPUT_BASE) \
 		--go-header-file=./hack/boilerplate/boilerplate.generatego.txt
 
-	$(CONVERSION_GEN) \
-		--input-dirs=./$(EXP_DIR)/api/v1alpha3 \
-		--output-file-base=zz_generated.conversion \
-		--go-header-file=./hack/boilerplate/boilerplate.generatego.txt $(GEN_OUTPUT_BASE)
-
-.PHONY: generate-go-eks-bootstrap
-generate-go-eks-bootstrap: $(CONTROLLER_GEN)
-	$(CONTROLLER_GEN) \
-		paths=./bootstrap/eks/api/... \
-		object:headerFile=./hack/boilerplate/boilerplate.generatego.txt
 	$(CONVERSION_GEN) \
 		--input-dirs=./bootstrap/eks/api/v1alpha3 \
+		--input-dirs=./bootstrap/eks/api/v1alpha4 \
+		--extra-peer-dirs=sigs.k8s.io/cluster-api-provider-aws/api/v1alpha3 \
+		--extra-peer-dirs=sigs.k8s.io/cluster-api-provider-aws/api/v1alpha4 \
+		--extra-peer-dirs=sigs.k8s.io/cluster-api/api/v1alpha3 \
+		--extra-peer-dirs=sigs.k8s.io/cluster-api/api/v1alpha4 \
+		--build-tag=ignore_autogenerated_conversions \
 		--output-file-base=zz_generated.conversion $(GEN_OUTPUT_BASE) \
 		--go-header-file=./hack/boilerplate/boilerplate.generatego.txt
 
-.PHONY: generate-go-eks-controlplane
-generate-go-eks-controlplane: $(CONTROLLER_GEN) $(CONVERSION_GEN)
-	$(CONTROLLER_GEN) \
-		paths=./controlplane/eks/api/... \
-		object:headerFile=./hack/boilerplate/boilerplate.generatego.txt
 	$(CONVERSION_GEN) \
 		--input-dirs=./controlplane/eks/api/v1alpha3 \
+		--input-dirs=./controlplane/eks/api/v1alpha4 \
+		--extra-peer-dirs=sigs.k8s.io/cluster-api-provider-aws/api/v1alpha3 \
+		--extra-peer-dirs=sigs.k8s.io/cluster-api-provider-aws/api/v1alpha4 \
+		--extra-peer-dirs=sigs.k8s.io/cluster-api/api/v1alpha3 \
+		--extra-peer-dirs=sigs.k8s.io/cluster-api/api/v1alpha4 \
+		--build-tag=ignore_autogenerated_conversions \
 		--output-file-base=zz_generated.conversion $(GEN_OUTPUT_BASE) \
 		--go-header-file=./hack/boilerplate/boilerplate.generatego.txt
 
-.PHONY: generate-manifests
-generate-manifests: $(CONTROLLER_GEN) ## Generate manifests e.g. CRD, RBAC etc.
-	$(CONTROLLER_GEN) \
-		paths=./api/... \
-		paths=./$(EXP_DIR)/api/... \
-		paths=./controlplane/eks/api/...\
-		paths=./bootstrap/eks/api/... \
-		crd:crdVersions=v1 \
-		output:crd:dir=$(CRD_ROOT) \
-		output:webhook:dir=$(WEBHOOK_ROOT) \
-		webhook
-	$(CONTROLLER_GEN) \
-		paths=./controllers/... \
-		paths=./$(EXP_DIR)/controllers/... \
-		paths=./bootstrap/eks/controllers/... \
-		paths=./controlplane/eks/controllers/... \
-		output:rbac:dir=$(RBAC_ROOT) \
-		rbac:roleName=manager-role
+	$(CONVERSION_GEN) \
+		--input-dirs=./$(EXP_DIR)/api/v1alpha3 \
+		--input-dirs=./$(EXP_DIR)/api/v1alpha4 \
+		--extra-peer-dirs=sigs.k8s.io/cluster-api-provider-aws/api/v1alpha3 \
+		--extra-peer-dirs=sigs.k8s.io/cluster-api-provider-aws/api/v1alpha4 \
+		--extra-peer-dirs=sigs.k8s.io/cluster-api/api/v1alpha3 \
+		--extra-peer-dirs=sigs.k8s.io/cluster-api/api/v1alpha4 \
+		--build-tag=ignore_autogenerated_conversions \
+		--output-file-base=zz_generated.conversion $(GEN_OUTPUT_BASE) \
+		--go-header-file=./hack/boilerplate/boilerplate.generatego.txt
+
+		touch $@
+
+.PHONY: defaulters
+defaulters: $(DEFAULTER_GEN)
+	$(DEFAULTER_GEN) \
+		--input-dirs=./api/v1alpha3 \
+		--input-dirs=./api/v1alpha4 \
+		--input-dirs=./api/v1beta1 \
+		--input-dirs=./$(EXP_DIR)/api/v1beta1 \
+		--input-dirs=./cmd/clusterawsadm/api/bootstrap/v1beta1 \
+		--input-dirs=./cmd/clusterawsadm/api/bootstrap/v1alpha1 \
+		--extra-peer-dirs=sigs.k8s.io/cluster-api/api/v1beta1 \
+		--v=0 $(GEN_OUTPUT_BASE) \
+		--go-header-file=./hack/boilerplate/boilerplate.generatego.txt
 
 ## --------------------------------------
 ## Docker
@@ -585,6 +590,7 @@ verify: verify-boilerplate verify-modules verify-gen release-manifests
 
 .PHONY: verify-boilerplate
 verify-boilerplate:
+	-rm ./hack/tools/bin/*.sh
 	./hack/verify-boilerplate.sh
 
 .PHONY: verify-modules