Feat: allow delete the provisioning resources

chart/templates/terraform_controller.yaml

@@ -24,6 +24,7 @@ spec:
             {{- if .Values.controllerNamespace }}
             - --controller-namespace={{ .Values.controllerNamespace }}
             {{- end }}
+            - --feature-gates=AllowDeleteHalfway={{ .Values.featureGates.allowDeleteHalfway }}
           env:
             - name: CONTROLLER_NAMESPACE
               valueFrom:

chart/values.yaml

@@ -25,3 +25,9 @@ backend:
   namespace: vela-system
 
 githubBlocked: "'false'"
+
+featureGates:
+  # Enable the feature of allowing to delete a configuration whose cloud resources is not fully provisioned, or error happens
+  # This guarantees that the partial cloud resources will be deleted when the configuration is deleted
+  # Default value is true
+  allowDeleteHalfway: true

controllers/configuration/configuration.go

@@ -9,13 +9,15 @@ import (
 	"github.com/pkg/errors"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	apitypes "k8s.io/apimachinery/pkg/types"
+	"k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/klog/v2"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/oam-dev/terraform-controller/api/types"
 	crossplane "github.com/oam-dev/terraform-controller/api/types/crossplane-runtime"
 	"github.com/oam-dev/terraform-controller/api/v1beta1"
 	"github.com/oam-dev/terraform-controller/api/v1beta2"
+	"github.com/oam-dev/terraform-controller/controllers/features"
 	"github.com/oam-dev/terraform-controller/controllers/provider"
 )
 
@@ -82,9 +84,13 @@ func Get(ctx context.Context, k8sClient client.Client, namespacedName apitypes.N
 
 // IsDeletable will check whether the Configuration can be deleted immediately
 // If deletable, it means
+// - feature gate AllowDeleteHalfway is enabled
 // - no external cloud resources are provisioned
 // - it's in force-delete state
 func IsDeletable(ctx context.Context, k8sClient client.Client, configuration *v1beta2.Configuration) (bool, error) {
+	if feature.DefaultFeatureGate.Enabled(features.AllowDeleteHalfway) {
+		return true, nil
+	}
 	if configuration.Spec.ForceDelete != nil && *configuration.Spec.ForceDelete {
 		return true, nil
 	}

controllers/configuration_controller.go

@@ -28,14 +28,14 @@ import (
 	"time"
 
 	"github.com/go-logr/logr"
-	"github.com/oam-dev/terraform-controller/controllers/configuration/backend"
 	"github.com/pkg/errors"
 	batchv1 "k8s.io/api/batch/v1"
 	v1 "k8s.io/api/core/v1"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/klog/v2"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -46,6 +46,8 @@ import (
 	"github.com/oam-dev/terraform-controller/api/v1beta1"
 	"github.com/oam-dev/terraform-controller/api/v1beta2"
 	tfcfg "github.com/oam-dev/terraform-controller/controllers/configuration"
+	"github.com/oam-dev/terraform-controller/controllers/configuration/backend"
+	"github.com/oam-dev/terraform-controller/controllers/features"
 	"github.com/oam-dev/terraform-controller/controllers/provider"
 	"github.com/oam-dev/terraform-controller/controllers/terraform"
 )
@@ -170,6 +172,7 @@ func (r *ConfigurationReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 	if isDeleting {
 		// terraform destroy
 		klog.InfoS("performing Configuration Destroy", "Namespace", req.Namespace, "Name", req.Name, "JobName", meta.DestroyJobName)
+		// if allow to delete halfway, we will not check the status of the apply job.
 
 		_, err := terraform.GetTerraformStatus(ctx, meta.ControllerNamespace, meta.DestroyJobName, terraformContainerName, terraformInitContainerName)
 		if err != nil {
@@ -404,11 +407,20 @@ func (r *ConfigurationReconciler) terraformDestroy(ctx context.Context, configur
 	}
 
 	// Sub-resources can be deleted directly without waiting destroy job is done means:
-	// - Configuration is deletable (no cloud resources are provisioned or force delete is set)
+	// - Configuration is deletable (no cloud resources are provisioned or force delete is set) WHEN not allow to delete halfway.
 	// - OR user want to keep the resource when delete the configuration CR
-	notWaitingDestroyJob := deletable || !meta.DeleteResource
+	// If allowed to delete halfway, there could be parts of cloud resources are provisioned, so we need to wait destroy job is done.
+	notWaitingDestroyJob := deletable && !feature.DefaultFeatureGate.Enabled(features.AllowDeleteHalfway) || !meta.DeleteResource
+	// If the configuration is deletable, and it is caused by AllowDeleteHalfway feature, the apply job may be still running, we should clean it first to avoid data race.
+	needCleanApplyJob := deletable && feature.DefaultFeatureGate.Enabled(features.AllowDeleteHalfway)
 
 	if !notWaitingDestroyJob {
+		if needCleanApplyJob {
+			err := deleteApplyJob(ctx, meta, k8sClient)
+			if err != nil {
+				return err
+			}
+		}
 		if err := k8sClient.Get(ctx, client.ObjectKey{Name: meta.DestroyJobName, Namespace: meta.ControllerNamespace}, &destroyJob); err != nil {
 			if kerrors.IsNotFound(err) {
 				if err := r.Client.Get(ctx, client.ObjectKey{Name: configuration.Name, Namespace: configuration.Namespace}, &v1beta2.Configuration{}); err == nil {
@@ -430,7 +442,7 @@ func (r *ConfigurationReconciler) terraformDestroy(ctx context.Context, configur
 	}
 
 	if configuration.Spec.ForceDelete != nil && *configuration.Spec.ForceDelete {
-		// Try to clean up more sub-resources as possible. Ignore the issues if it hit any.
+		// Try to clean up as more sub-resources as possible. Ignore the issues if it hit any.
 		if err := r.cleanUpSubResources(ctx, configuration, meta); err != nil {
 			klog.Warningf("Failed to clean up sub-resources, but it's ignored as the resources are being forced to delete: %s", err)
 		}

controllers/features/feature_gate.go

@@ -0,0 +1,35 @@
+/*
+Copyright 2023 The KubeVela Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package features
+
+import (
+	"k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/component-base/featuregate"
+)
+
+const (
+	AllowDeleteHalfway featuregate.Feature = "AllowDeleteHalfway"
+)
+
+var defaultFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{
+	AllowDeleteHalfway: {Default: false, PreRelease: featuregate.Alpha},
+}
+
+func init() {
+	runtime.Must(feature.DefaultMutableFeatureGate.Add(defaultFeatureGates))
+}

controllers/terraform/status.go

@@ -35,6 +35,7 @@ func GetTerraformStatus(ctx context.Context, jobNamespace, jobName, containerNam
 	return state, errors.New(errMsg)
 }
 
+// analyzeTerraformLog will analyze the logs of Terraform apply pod, returns true if check is ok.
 func analyzeTerraformLog(logs string, stage types.Stage) (bool, types.ConfigurationState, string) {
 	lines := strings.Split(logs, "\n")
 	for i, line := range lines {

go.mod

@@ -14,12 +14,15 @@ require (
 	github.com/onsi/ginkgo v1.16.4
 	github.com/onsi/gomega v1.19.0
 	github.com/pkg/errors v0.9.1
+	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.7.0
 	gopkg.in/yaml.v2 v2.4.0
 	gotest.tools v2.2.0+incompatible
 	k8s.io/api v0.21.3
 	k8s.io/apimachinery v0.21.3
+	k8s.io/apiserver v0.21.3
 	k8s.io/client-go v0.21.3
+	k8s.io/component-base v0.21.3
 	k8s.io/klog/v2 v2.8.0
 	k8s.io/utils v0.0.0-20210722164352-7f3ee0f31471
 	sigs.k8s.io/controller-runtime v0.9.5
@@ -61,7 +64,6 @@ require (
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.26.0 // indirect
 	github.com/prometheus/procfs v0.6.0 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/zclconf/go-cty v1.10.0 // indirect
 	golang.org/x/net v0.2.0 // indirect
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
@@ -77,7 +79,6 @@ require (
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 	k8s.io/apiextensions-apiserver v0.21.3 // indirect
-	k8s.io/component-base v0.21.3 // indirect
 	k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect
 )

go.sum

@@ -799,6 +799,7 @@ k8s.io/apiextensions-apiserver v0.21.3 h1:+B6biyUWpqt41kz5x6peIsljlsuwvNAp/oFax/
 k8s.io/apiextensions-apiserver v0.21.3/go.mod h1:kl6dap3Gd45+21Jnh6utCx8Z2xxLm8LGDkprcd+KbsE=
 k8s.io/apimachinery v0.21.3 h1:3Ju4nvjCngxxMYby0BimUk+pQHPOQp3eCGChk5kfVII=
 k8s.io/apimachinery v0.21.3/go.mod h1:H/IM+5vH9kZRNJ4l3x/fXP/5bOPJaVP/guptnZPeCFI=
+k8s.io/apiserver v0.21.3 h1:QxAgE1ZPQG5cPlHScHTnLxP9H/kU3zjH1Vnd8G+n5OI=
 k8s.io/apiserver v0.21.3/go.mod h1:eDPWlZG6/cCCMj/JBcEpDoK+I+6i3r9GsChYBHSbAzU=
 k8s.io/client-go v0.21.3 h1:J9nxZTOmvkInRDCzcSNQmPJbDYN/PjlxXT9Mos3HcLg=
 k8s.io/client-go v0.21.3/go.mod h1:+VPhCgTsaFmGILxR/7E1N0S+ryO010QBeNCv5JwRGYU=

main.go

@@ -17,11 +17,12 @@ limitations under the License.
 package main
 
 import (
-	"flag"
 	"os"
 	"time"
 
+	"github.com/spf13/pflag"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apiserver/pkg/util/feature"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/klog/v2"
 	"k8s.io/klog/v2/klogr"
@@ -52,15 +53,16 @@ func main() {
 	var namespace string
 	var controllerNamespace string
 
-	flag.BoolVar(&enableLeaderElection, "enable-leader-election", false, "Enable leader election for controller manager, this will ensure there is only one active controller manager.")
-	flag.DurationVar(&syncPeriod, "informer-re-sync-interval", 10*time.Second, "controller shared informer lister full re-sync period")
-	flag.StringVar(&metricsAddr, "metrics-addr", ":38080", "The address the metric endpoint binds to.")
-	flag.StringVar(&namespace, "namespace", "", "Namespace to watch for resources, defaults to all namespaces")
-	flag.StringVar(&controllerNamespace, "controller-namespace", "", "Namespace to run the terraform jobs")
+	pflag.BoolVar(&enableLeaderElection, "enable-leader-election", false, "Enable leader election for controller manager, this will ensure there is only one active controller manager.")
+	pflag.DurationVar(&syncPeriod, "informer-re-sync-interval", 10*time.Second, "controller shared informer lister full re-sync period")
+	pflag.StringVar(&metricsAddr, "metrics-addr", ":38080", "The address the metric endpoint binds to.")
+	pflag.StringVar(&namespace, "namespace", "", "Namespace to watch for resources, defaults to all namespaces")
+	pflag.StringVar(&controllerNamespace, "controller-namespace", "", "Namespace to run the terraform jobs")
+	feature.DefaultMutableFeatureGate.AddFlag(pflag.CommandLine)
 
 	// embed klog
 	klog.InitFlags(nil)
-	flag.Parse()
+	pflag.Parse()
 
 	ctrl.SetLogger(klogr.New())