[WIP] ROX-7352 unlock pending helm actions

pkg/client/actionclient.go

@@ -63,12 +63,14 @@ type ActionInterface interface {
 	Install(name, namespace string, chrt *chart.Chart, vals map[string]interface{}, opts ...InstallOption) (*release.Release, error)
 	Upgrade(name, namespace string, chrt *chart.Chart, vals map[string]interface{}, opts ...UpgradeOption) (*release.Release, error)
 	Uninstall(name string, opts ...UninstallOption) (*release.UninstallReleaseResponse, error)
+	Rollback(name string, opts ...RollbackOption) error
 	Reconcile(rel *release.Release) error
 }
 
 type GetOption func(*action.Get) error
 type InstallOption func(*action.Install) error
 type UpgradeOption func(*action.Upgrade) error
+type RollbackOption func(*action.Rollback) error
 type UninstallOption func(*action.Uninstall) error
 
 func NewActionClientGetter(acg ActionConfigGetter) ActionClientGetter {
@@ -179,6 +181,16 @@ func (c *actionClient) Upgrade(name, namespace string, chrt *chart.Chart, vals m
 	return rel, nil
 }
 
+func (c *actionClient) Rollback(name string, opts ...RollbackOption) error {
+	rollback := action.NewRollback(c.conf)
+	for _, o := range opts {
+		if err := o(rollback); err != nil {
+			return err
+		}
+	}
+	return rollback.Run(name)
+}
+
 func (c *actionClient) Uninstall(name string, opts ...UninstallOption) (*release.UninstallReleaseResponse, error) {
 	uninstall := action.NewUninstall(c.conf)
 	for _, o := range opts {

pkg/reconciler/internal/conditions/conditions.go

@@ -28,7 +28,9 @@ const (
 	TypeInitialized    = "Initialized"
 	TypeDeployed       = "Deployed"
 	TypeReleaseFailed  = "ReleaseFailed"
+	TypeRollbackFailed = "TypeRollbackFailed"
 	TypeIrreconcilable = "Irreconcilable"
+	TypeReleasePending = "ReleasePending"
 
 	ReasonInstallSuccessful   = status.ConditionReason("InstallSuccessful")
 	ReasonUpgradeSuccessful   = status.ConditionReason("UpgradeSuccessful")
@@ -41,6 +43,7 @@ const (
 	ReasonUpgradeError             = status.ConditionReason("UpgradeError")
 	ReasonReconcileError           = status.ConditionReason("ReconcileError")
 	ReasonUninstallError           = status.ConditionReason("UninstallError")
+	ReasonReleasePending           = status.ConditionReason("ReleasePending")
 )
 
 func Initialized(stat corev1.ConditionStatus, reason status.ConditionReason, message interface{}) status.Condition {
@@ -55,6 +58,14 @@ func ReleaseFailed(stat corev1.ConditionStatus, reason status.ConditionReason, m
 	return newCondition(TypeReleaseFailed, stat, reason, message)
 }
 
+func ReleasePending(stat corev1.ConditionStatus, reason status.ConditionReason, message interface{}) status.Condition {
+	return newCondition(TypeReleasePending, stat, reason, message)
+}
+
+func RollbackFailed(stat corev1.ConditionStatus, reason status.ConditionReason, message interface{}) status.Condition {
+	return newCondition(TypeRollbackFailed, stat, reason, message)
+}
+
 func Irreconcilable(stat corev1.ConditionStatus, reason status.ConditionReason, message interface{}) status.Condition {
 	return newCondition(TypeIrreconcilable, stat, reason, message)
 }

pkg/reconciler/internal/fake/actionclient.go

@@ -53,12 +53,14 @@ type ActionClient struct {
 	Upgrades   []UpgradeCall
 	Uninstalls []UninstallCall
 	Reconciles []ReconcileCall
+	Rollbacks  []RollbackCall
 
 	HandleGet       func() (*release.Release, error)
 	HandleInstall   func() (*release.Release, error)
 	HandleUpgrade   func() (*release.Release, error)
 	HandleUninstall func() (*release.UninstallReleaseResponse, error)
 	HandleReconcile func() error
+	HandleRollback  func() error
 }
 
 func NewActionClient() ActionClient {
@@ -118,6 +120,11 @@ type ReconcileCall struct {
 	Release *release.Release
 }
 
+type RollbackCall struct {
+	Name string
+	Opts []client.RollbackOption
+}
+
 func (c *ActionClient) Get(name string, opts ...client.GetOption) (*release.Release, error) {
 	c.Gets = append(c.Gets, GetCall{name, opts})
 	return c.HandleGet()
@@ -142,3 +149,8 @@ func (c *ActionClient) Reconcile(rel *release.Release) error {
 	c.Reconciles = append(c.Reconciles, ReconcileCall{rel})
 	return c.HandleReconcile()
 }
+
+func (c *ActionClient) Rollback(name string, opts ...client.RollbackOption) error {
+	c.Rollbacks = append(c.Rollbacks, RollbackCall{name, opts})
+	return c.HandleRollback()
+}

pkg/reconciler/reconciler.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"math/rand"
 	"strings"
 	"sync"
 	"time"
@@ -56,7 +57,13 @@ import (
 	"github.com/joelanford/helm-operator/pkg/values"
 )
 
-const uninstallFinalizer = "uninstall-helm-release"
+const (
+	uninstallFinalizer = "uninstall-helm-release"
+	// pendingReleaseTimeout defines the time frame after which a rollback is performed on a given release
+	pendingReleaseTimeout = time.Second * 30
+	// pendingRetryInterval defines the time waiting until the current reconciliation finished to avoid spamming helm actions
+	pendingRetryInterval = time.Second * 5
+)
 
 // Reconciler reconciles a Helm object
 type Reconciler struct {
@@ -561,10 +568,22 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl.
 			return ctrl.Result{}, err
 		}
 
+	case statedNeedsRollback:
+		{
+			if err := r.doRollback(actionClient, &u, obj); err != nil {
+				return ctrl.Result{}, err
+			}
+		}
+
 	case stateUnchanged:
 		if err := r.doReconcile(actionClient, &u, rel, log); err != nil {
 			return ctrl.Result{}, err
 		}
+
+	case stateNeedsSkip:
+		// TODO(do-not-merge): set status custom resource status
+		return ctrl.Result{Requeue: true, RequeueAfter: pendingRetryInterval}, nil
+
 	default:
 		return ctrl.Result{}, fmt.Errorf("unexpected release state: %s", state)
 	}
@@ -614,10 +633,12 @@ func (r *Reconciler) getValues(ctx context.Context, obj *unstructured.Unstructur
 type helmReleaseState string
 
 const (
-	stateNeedsInstall helmReleaseState = "needs install"
-	stateNeedsUpgrade helmReleaseState = "needs upgrade"
-	stateUnchanged    helmReleaseState = "unchanged"
-	stateError        helmReleaseState = "error"
+	stateNeedsInstall   helmReleaseState = "needs install"
+	stateNeedsUpgrade   helmReleaseState = "needs upgrade"
+	stateUnchanged      helmReleaseState = "unchanged"
+	stateError          helmReleaseState = "error"
+	statedNeedsRollback helmReleaseState = "needs rollback"
+	stateNeedsSkip      helmReleaseState = "needs skip"
 )
 
 func (r *Reconciler) handleDeletion(ctx context.Context, actionClient helmclient.ActionInterface, obj *unstructured.Unstructured, log logr.Logger) error {
@@ -665,6 +686,19 @@ func (r *Reconciler) getReleaseState(client helmclient.ActionInterface, obj meta
 		return nil, stateNeedsInstall, nil
 	}
 
+	// If the release is pending it is likely that it happened because the helm install/upgrade/rollback action
+	// were cancelled unexpectedly.
+	//TODO(do-not-merge): as pending releases are now rolled back it is necessary to ensure that an in-progress helm release is not cancelled
+	// and this code is only executed to ensure aborted releases are rolled back.
+	if currentRelease.Info.Status.IsPending() {
+		t := time.Now().Sub(currentRelease.Info.LastDeployed.Time)
+		if t <= pendingReleaseTimeout {
+			r.log.Info("Release pending, skipped", "name", currentRelease.Name, "version", currentRelease.Version, "retry_in", pendingRetryInterval.String())
+			return nil, stateNeedsSkip, nil
+		}
+		return nil, statedNeedsRollback, nil
+	}
+
 	var opts []helmclient.UpgradeOption
 	for name, annot := range r.upgradeAnnotations {
 		if v, ok := obj.GetAnnotations()[name]; ok {
@@ -679,7 +713,7 @@ func (r *Reconciler) getReleaseState(client helmclient.ActionInterface, obj meta
 	if err != nil {
 		return currentRelease, stateError, err
 	}
-	if specRelease.Manifest != currentRelease.Manifest ||
+	if specRelease.Manifest != fmt.Sprintf("%d%s", rand.Int31(), currentRelease.Manifest) ||
 		currentRelease.Info.Status == release.StatusFailed ||
 		currentRelease.Info.Status == release.StatusSuperseded {
 		return currentRelease, stateNeedsUpgrade, nil
@@ -737,6 +771,22 @@ func (r *Reconciler) reportOverrideEvents(obj runtime.Object) {
 	}
 }
 
+func (r *Reconciler) doRollback(actionClient helmclient.ActionInterface, u *updater.Updater, obj *unstructured.Unstructured) error {
+	u.UpdateStatus(
+		//TODO(do-not-merge): Add better error message indicating that release was maybe aborted
+		updater.EnsureCondition(conditions.ReleasePending(corev1.ConditionFalse, "", "")),
+	)
+
+	if err := actionClient.Rollback(obj.GetName()); err != nil {
+		u.UpdateStatus(
+			//TODO(do-not-merge): Add better error message indicating that release was maybe aborted with manual steps
+			updater.EnsureCondition(conditions.RollbackFailed(corev1.ConditionFalse, "", err)),
+		)
+		return err
+	}
+	return nil
+}
+
 func (r *Reconciler) doReconcile(actionClient helmclient.ActionInterface, u *updater.Updater, rel *release.Release, log logr.Logger) error {
 	// If a change is made to the CR spec that causes a release failure, a
 	// ConditionReleaseFailed is added to the status conditions. If that change