update: delete cluster to clean up Karpenter NodePools and EC2NodeClasses

go.mod

@@ -47,6 +47,7 @@ require (
 	k8s.io/apimachinery v0.29.3
 	k8s.io/cli-runtime v0.29.3
 	k8s.io/client-go v0.29.3
+	k8s.io/kubectl v0.29.3
 	sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3
 	sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3
 	sigs.k8s.io/yaml v1.3.0
@@ -97,6 +98,7 @@ require (
 	github.com/fatih/color v1.13.0 // indirect
 	github.com/felixge/httpsnoop v1.0.3 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/fvbommel/sortorder v1.1.0 // indirect
 	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/go-gorp/gorp/v3 v3.1.0 // indirect
 	github.com/go-logr/logr v1.3.0 // indirect
@@ -197,10 +199,9 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/apiextensions-apiserver v0.29.0 // indirect
 	k8s.io/apiserver v0.29.0 // indirect
-	k8s.io/component-base v0.29.0 // indirect
+	k8s.io/component-base v0.29.3 // indirect
 	k8s.io/klog/v2 v2.110.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
-	k8s.io/kubectl v0.29.0 // indirect
 	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
 	oras.land/oras-go v1.2.4 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect

go.sum

@@ -224,6 +224,8 @@ github.com/foxcpp/go-mockdns v1.0.0 h1:7jBqxd3WDWwi/6WhDvacvH1XsN3rOLXyHM1uhvIx6
 github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
+github.com/fvbommel/sortorder v1.1.0 h1:fUmoe+HLsBTctBDoaBwpQo5N+nrCp8g/BjKb/6ZQmYw=
+github.com/fvbommel/sortorder v1.1.0/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
 github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
 github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
@@ -953,14 +955,14 @@ k8s.io/cli-runtime v0.29.3 h1:r68rephmmytoywkw2MyJ+CxjpasJDQY7AGc3XY2iv1k=
 k8s.io/cli-runtime v0.29.3/go.mod h1:aqVUsk86/RhaGJwDhHXH0jcdqBrgdF3bZWk4Z9D4mkM=
 k8s.io/client-go v0.29.3 h1:R/zaZbEAxqComZ9FHeQwOh3Y1ZUs7FaHKZdQtIc2WZg=
 k8s.io/client-go v0.29.3/go.mod h1:tkDisCvgPfiRpxGnOORfkljmS+UrW+WtXAy2fTvXJB0=
-k8s.io/component-base v0.29.0 h1:T7rjd5wvLnPBV1vC4zWd/iWRbV8Mdxs+nGaoaFzGw3s=
-k8s.io/component-base v0.29.0/go.mod h1:sADonFTQ9Zc9yFLghpDpmNXEdHyQmFIGbiuZbqAXQ1M=
+k8s.io/component-base v0.29.3 h1:Oq9/nddUxlnrCuuR2K/jp6aflVvc0uDvxMzAWxnGzAo=
+k8s.io/component-base v0.29.3/go.mod h1:Yuj33XXjuOk2BAaHsIGHhCKZQAgYKhqIxIjIr2UXYio=
 k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0=
 k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo=
 k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780=
 k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA=
-k8s.io/kubectl v0.29.0 h1:Oqi48gXjikDhrBF67AYuZRTcJV4lg2l42GmvsP7FmYI=
-k8s.io/kubectl v0.29.0/go.mod h1:0jMjGWIcMIQzmUaMgAzhSELv5WtHo2a8pq67DtviAJs=
+k8s.io/kubectl v0.29.3 h1:RuwyyIU42MAISRIePaa8Q7A3U74Q9P4MoJbDFz9o3us=
+k8s.io/kubectl v0.29.3/go.mod h1:yCxfY1dbwgVdEt2zkJ6d5NNLOhhWgTyrqACIoFhpdd4=
 k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI=
 k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 oras.land/oras-go v1.2.4 h1:djpBY2/2Cs1PV87GSJlxv4voajVOMZxqqtq9AB8YNvY=

pkg/application/autoscaling/karpenter/cluster_delete.go

@@ -0,0 +1,111 @@
+package karpenter
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/awslabs/eksdemo/pkg/kubernetes"
+	api_errors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/cli-runtime/pkg/genericclioptions"
+	"k8s.io/cli-runtime/pkg/genericiooptions"
+	"k8s.io/cli-runtime/pkg/printers"
+	"k8s.io/cli-runtime/pkg/resource"
+	"k8s.io/client-go/dynamic"
+	cmdwait "k8s.io/kubectl/pkg/cmd/wait"
+)
+
+func DeleteCustomResources(kubeContext string) error {
+	client, err := kubernetes.DynamicClient(kubeContext)
+	if err != nil {
+		return fmt.Errorf("failed creating kubernetes dynamic client: %w", err)
+	}
+
+	if err := DeleteNodePool(client); err != nil {
+		return err
+	}
+
+	return DeleteEC2NodeClass(client, kubeContext)
+}
+
+// This uses cli-runtime's Resource Builder and kubectl's waiter to simplify the delete and wait
+// Ideally this should use the Dynamic Client and a custom waiter
+func DeleteEC2NodeClass(client dynamic.Interface, kubeContext string) error {
+	getter := genericclioptions.NewConfigFlags(true)
+	getter.Context = &kubeContext
+
+	restMapper, err := getter.ToRESTMapper()
+	if err != nil {
+		return fmt.Errorf("failed creating restMapper: %w", err)
+	}
+
+	gvk := "EC2NodeClass.v1beta1.karpenter.k8s.aws"
+	fullySpecifiedGVK, _ := schema.ParseKindArg(gvk)
+
+	_, err = restMapper.RESTMapping(fullySpecifiedGVK.GroupKind(), fullySpecifiedGVK.Version)
+	if err != nil {
+		if meta.IsNoMatchError(err) {
+			// EC2NodeClass kind doesn't exist, skip deletion
+			return nil
+		}
+		return err
+	}
+
+	infos, err := resource.NewBuilder(getter).Unstructured().ResourceTypes(gvk).SelectAllParam(true).Flatten().Do().Infos()
+	if err != nil {
+		return err
+	}
+
+	for _, info := range infos {
+		fmt.Printf("Deleting Karpenter EC2NodeClass %q\n", info.Name)
+
+		_, err := resource.NewHelper(info.Client, info.Mapping).Delete(info.Namespace, info.Name)
+		if err != nil {
+			return fmt.Errorf("failed to delete EC2NodeClass %q: %w", info.Name, err)
+		}
+	}
+
+	fmt.Println("Waiting up to 30 seconds for EC2NodeClasses to be deleted...")
+	waitOptions := cmdwait.WaitOptions{
+		ResourceFinder: genericclioptions.ResourceFinderForResult(resource.InfoListVisitor(infos)),
+		DynamicClient:  client,
+		Timeout:        30 * time.Second,
+
+		Printer:     &printers.NamePrinter{ShortOutput: false, Operation: "deleted"},
+		ConditionFn: cmdwait.IsDeleted,
+		IOStreams:   genericiooptions.IOStreams{Out: os.Stdout, ErrOut: os.Stderr},
+	}
+
+	return waitOptions.RunWait()
+}
+
+func DeleteNodePool(client dynamic.Interface) error {
+	nodePools := schema.GroupVersionResource{
+		Group:    "karpenter.sh",
+		Version:  "v1beta1",
+		Resource: "nodepools",
+	}
+
+	list, err := client.Resource(nodePools).List(context.Background(), metav1.ListOptions{})
+	if err != nil {
+		if api_errors.IsNotFound(err) {
+			// nodepools resource doesn't exist, skip deletion
+			return nil
+		}
+		return err
+	}
+
+	for _, item := range list.Items {
+		nodePoolName := item.GetName()
+
+		fmt.Printf("Deleting Karpenter NodePool %q\n", nodePoolName)
+		if err := client.Resource(nodePools).Delete(context.Background(), nodePoolName, metav1.DeleteOptions{}); err != nil {
+			fmt.Printf("failed to delete NodePool %q: %s\n", nodePoolName, err)
+		}
+	}
+	return nil
+}

pkg/resource/cluster/options.go

@@ -207,6 +207,10 @@ func (o *ClusterOptions) PreCreate() error {
 func (o *ClusterOptions) PreDelete() error {
 	o.Region = aws.Region()
 
+	if err := karpenter.DeleteCustomResources(o.Common().KubeContext); err != nil {
+		return err
+	}
+
 	cloudformationClient := aws.NewCloudformationClient()
 	stacks, err := cloudformation_stack.NewGetter(cloudformationClient).GetStacksByCluster(o.ClusterName, "")
 	if err != nil {