support timeout in helm calls #4102

Author: drewwells

internal/cmd/helm-operator/run/cmd.go

@@ -145,9 +145,13 @@ func run(cmd *cobra.Command, f *flags.Flags) {
 	for _, w := range ws {
 		// Register the controller with the factory.
 		err := controller.Add(mgr, controller.WatchOptions{
-			Namespace:               namespace,
-			GVK:                     w.GroupVersionKind,
-			ManagerFactory:          release.NewManagerFactory(mgr, w.ChartDir),
+			Namespace: namespace,
+			GVK:       w.GroupVersionKind,
+			ManagerFactory: release.NewManagerFactory(release.ManagerFactoryOptions{
+				ChartDir:  w.ChartDir,
+				CRManager: mgr,
+				Timeout:   f.Timeout,
+			}),
 			ReconcilePeriod:         f.ReconcilePeriod,
 			WatchDependentResources: *w.WatchDependentResources,
 			OverrideValues:          w.OverrideValues,

internal/helm/flags/flag.go

@@ -15,6 +15,7 @@
 package flags
 
 import (
+	"flag"
 	"runtime"
 	"time"
 
@@ -24,6 +25,7 @@ import (
 // Flags - Options to be used by a helm operator
 type Flags struct {
 	ReconcilePeriod         time.Duration
+	Timeout                 time.Duration
 	WatchesFile             string
 	MetricsAddress          string
 	EnableLeaderElection    bool
@@ -34,6 +36,11 @@ type Flags struct {
 
 // AddTo - Add the helm operator flags to the the flagset
 func (f *Flags) AddTo(flagSet *pflag.FlagSet) {
+	flag.DurationVar(&f.Timeout,
+		"timeout",
+		time.Minute*5,
+		"time to wait for any individual Kubernetes operation (like Jobs for hooks)",
+	)
 	flagSet.DurationVar(&f.ReconcilePeriod,
 		"reconcile-period",
 		time.Minute,

internal/helm/release/manager.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 	"fmt"
 	"strings"
+	"time"
 
 	jsonpatch "gomodules.xyz/jsonpatch/v3"
 	"helm.sh/helm/v3/pkg/action"
@@ -66,6 +67,7 @@ type manager struct {
 	values map[string]interface{}
 	status *types.HelmAppStatus
 
+	timeout           time.Duration
 	isInstalled       bool
 	isUpgradeRequired bool
 	deployedRelease   *rpb.Release
@@ -152,13 +154,15 @@ func (m manager) getCandidateRelease(namespace, name string, chart *cpb.Chart,
 	values map[string]interface{}) (*rpb.Release, error) {
 	upgrade := action.NewUpgrade(m.actionConfig)
 	upgrade.Namespace = namespace
+	upgrade.Timeout = m.timeout
 	upgrade.DryRun = true
 	return upgrade.Run(name, chart, values)
 }
 
 // InstallRelease performs a Helm release install.
 func (m manager) InstallRelease(ctx context.Context, opts ...InstallOption) (*rpb.Release, error) {
 	install := action.NewInstall(m.actionConfig)
+	install.Timeout = m.timeout
 	install.ReleaseName = m.releaseName
 	install.Namespace = m.namespace
 	for _, o := range opts {
@@ -172,6 +176,7 @@ func (m manager) InstallRelease(ctx context.Context, opts ...InstallOption) (*rp
 		// Workaround for helm/helm#3338
 		if installedRelease != nil {
 			uninstall := action.NewUninstall(m.actionConfig)
+			uninstall.Timeout = m.timeout
 			_, uninstallErr := uninstall.Run(m.releaseName)
 
 			// In certain cases, InstallRelease will return a partial release in
@@ -202,6 +207,7 @@ func ForceUpgrade(force bool) UpgradeOption {
 func (m manager) UpgradeRelease(ctx context.Context, opts ...UpgradeOption) (*rpb.Release, *rpb.Release, error) {
 	upgrade := action.NewUpgrade(m.actionConfig)
 	upgrade.Namespace = m.namespace
+	upgrade.Timeout = m.timeout
 	for _, o := range opts {
 		if err := o(upgrade); err != nil {
 			return nil, nil, fmt.Errorf("failed to apply upgrade option: %w", err)
@@ -213,6 +219,7 @@ func (m manager) UpgradeRelease(ctx context.Context, opts ...UpgradeOption) (*rp
 		// Workaround for helm/helm#3338
 		if upgradedRelease != nil {
 			rollback := action.NewRollback(m.actionConfig)
+			rollback.Timeout = m.timeout
 			rollback.Force = true
 
 			// As of Helm 2.13, if UpgradeRelease returns a non-nil release, that
@@ -364,6 +371,7 @@ func (m manager) UninstallRelease(ctx context.Context, opts ...UninstallOption)
 	}
 
 	uninstall := action.NewUninstall(m.actionConfig)
+	uninstall.Timeout = m.timeout
 	for _, o := range opts {
 		if err := o(uninstall); err != nil {
 			return nil, fmt.Errorf("failed to apply uninstall option: %w", err)

internal/helm/release/manager_factory.go

@@ -16,6 +16,7 @@ package release
 
 import (
 	"fmt"
+	"time"
 
 	"helm.sh/helm/v3/pkg/action"
 	"helm.sh/helm/v3/pkg/chart/loader"
@@ -37,17 +38,32 @@ import (
 // improves decoupling between reconciliation logic and the Helm backend
 // components used to manage releases.
 type ManagerFactory interface {
-	NewManager(r *unstructured.Unstructured, overrideValues map[string]string) (Manager, error)
+	NewManager(*unstructured.Unstructured, map[string]string) (Manager, error)
 }
 
 type managerFactory struct {
 	mgr      crmanager.Manager
 	chartDir string
+	timeout  time.Duration
 }
 
 // NewManagerFactory returns a new Helm manager factory capable of installing and uninstalling releases.
-func NewManagerFactory(mgr crmanager.Manager, chartDir string) ManagerFactory {
-	return &managerFactory{mgr, chartDir}
+func NewManagerFactory(opts ManagerFactoryOptions) ManagerFactory {
+	return &managerFactory{
+		mgr:      opts.CRManager,
+		chartDir: opts.ChartDir,
+		timeout:  opts.Timeout,
+	}
+}
+
+// ManagerFactoryOptions enable configuration of Helm operators
+type ManagerFactoryOptions struct {
+	// Timeout specifies how long helm should attempt an action ie. install --timeout 5m
+	Timeout time.Duration
+	// CRManager is the controller-runtime manager
+	CRManager crmanager.Manager
+	// ChartDir is the directory containing directories of helm charts
+	ChartDir string
 }
 
 func (f managerFactory) NewManager(cr *unstructured.Unstructured, overrideValues map[string]string) (Manager, error) {
@@ -104,6 +120,7 @@ func (f managerFactory) NewManager(cr *unstructured.Unstructured, overrideValues
 		actionConfig:   actionConfig,
 		storageBackend: storageBackend,
 		kubeClient:     ownerRefClient,
+		timeout:        f.timeout,
 
 		releaseName: releaseName,
 		namespace:   cr.GetNamespace(),