WIP: flag plumbing

Notes:
 - Some of these parse into a map in their config and then call SetFrom
   Map -- need to retain ability to pass in via config
 - Should env enablement be part of "AddFlags" in these components?

```
$ for f in _output/bin/*; do X=$($f --help 2>&1 | grep gates); if [[ -n "$X" ]]; then echo; echo "$f"; echo "$X"; fi; done

_output/bin/apiextensions-apiserver
      --feature-gates mapStringBool                             4444 A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:
      --new-feature-gates mapStringBool                         A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:

_output/bin/e2e_node.test
      --feature-gates mapStringBool                                6666 A set of key=value pairs that describe feature gates for alpha/experimental features.
      --service-feature-gates mapStringBool                        7777 A set of key=value pairs that describe feature gates for alpha/experimental features for API service.

_output/bin/kube-aggregator
      --feature-gates mapStringBool                             4444 A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:
      --new-feature-gates mapStringBool                         A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:

_output/bin/kube-apiserver
      --feature-gates mapStringBool                        4444 A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:
      --new-feature-gates mapStringBool                    A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:

_output/bin/kube-controller-manager
      --feature-gates mapStringBool              4444 A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:
      --new-feature-gates mapStringBool          A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:

_output/bin/kubelet
      --feature-gates mapStringBool                              3333 A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:
      --new-feature-gates mapStringBool                          A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:

_output/bin/kube-proxy
      --feature-gates mapStringBool                  1111 A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:
      --new-feature-gates mapStringBool              A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:

_output/bin/kube-scheduler
      --feature-gates mapStringBool   4444 A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:
      --new-feature-gates mapStringBool   A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:
```

Author: thockin

cmd/kube-controller-manager/app/options/options.go

@@ -45,6 +45,7 @@ import (
 	kubectrlmgrconfigscheme "k8s.io/kubernetes/pkg/controller/apis/config/scheme"
 	"k8s.io/kubernetes/pkg/controller/garbagecollector"
 	garbagecollectorconfig "k8s.io/kubernetes/pkg/controller/garbagecollector/config"
+	"k8s.io/kubernetes/pkg/features"
 	netutils "k8s.io/utils/net"
 
 	// add the kubernetes feature gates
@@ -274,6 +275,8 @@ func (s *KubeControllerManagerOptions) Flags(allControllers []string, disabledBy
 	fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).")
 	fs.StringVar(&s.Generic.ClientConnection.Kubeconfig, "kubeconfig", s.Generic.ClientConnection.Kubeconfig, "Path to kubeconfig file with authorization and master location information (the master location can be overridden by the master flag).")
 	utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic"))
+	features.KubernetesGates().EnablePFlagControl("new-feature-gates", fss.FlagSet("generic"))
+	features.KubernetesGates().EnableEnvControl("KUBE_FEATURE_", func(err error) { panic(err.Error()) })
 
 	return fss
 }

cmd/kube-proxy/app/server.go

@@ -576,6 +576,10 @@ with the apiserver API to configure the proxy.`,
 	opts.AddFlags(fs)
 	fs.AddGoFlagSet(goflag.CommandLine) // for --boot-id-file and --machine-id-file
 
+	// Enable feature gates
+	features.KubernetesGates().EnablePFlagControl("new-feature-gates", fs)
+	features.KubernetesGates().EnableEnvControl("KUBE_FEATURE_", func(err error) { panic(err.Error()) })
+
 	_ = cmd.MarkFlagFilename("config", "yaml", "yml", "json")
 
 	return cmd

cmd/kube-scheduler/app/options/options.go

@@ -45,6 +45,7 @@ import (
 	"k8s.io/component-base/metrics"
 	"k8s.io/klog/v2"
 	schedulerappconfig "k8s.io/kubernetes/cmd/kube-scheduler/app/config"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/scheduler"
 	kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config"
 	"k8s.io/kubernetes/pkg/scheduler/apis/config/validation"
@@ -190,6 +191,8 @@ func (o *Options) initFlags() {
 	o.Deprecated.AddFlags(nfs.FlagSet("deprecated"))
 	options.BindLeaderElectionFlags(o.LeaderElection, nfs.FlagSet("leader election"))
 	utilfeature.DefaultMutableFeatureGate.AddFlag(nfs.FlagSet("feature gate"))
+	features.KubernetesGates().EnablePFlagControl("new-feature-gates", nfs.FlagSet("generic"))
+	features.KubernetesGates().EnableEnvControl("KUBE_FEATURE_", func(err error) { panic(err.Error()) })
 	o.Metrics.AddFlags(nfs.FlagSet("metrics"))
 	logsapi.AddFlags(o.Logs, nfs.FlagSet("logs"))
 

cmd/kubelet/app/server.go

@@ -289,6 +289,10 @@ is checked every 20 seconds (also configurable with a flag).`,
 	options.AddGlobalFlags(cleanFlagSet)
 	cleanFlagSet.BoolP("help", "h", false, fmt.Sprintf("help for %s", cmd.Name()))
 
+	// Enable feature gates
+	features.KubernetesGates().EnablePFlagControl("new-feature-gates", cleanFlagSet)
+	features.KubernetesGates().EnableEnvControl("KUBE_FEATURE_", func(err error) { panic(err.Error()) })
+
 	// ugly, but necessary, because Cobra's default UsageFunc and HelpFunc pollute the flagset with global flags
 	const usageFmt = "Usage:\n  %s\n\nFlags:\n%s"
 	cmd.SetUsageFunc(func(cmd *cobra.Command) error {
@@ -373,6 +377,7 @@ func kubeletConfigFlagPrecedence(kc *kubeletconfiginternal.KubeletConfiguration,
 	fs := newFakeFlagSet(newFlagSetWithGlobals())
 	// register throwaway KubeletFlags
 	options.NewKubeletFlags().AddFlags(fs)
+	features.KubernetesGates().EnablePFlagControl("new-feature-gates", fs)
 	// register new KubeletConfiguration
 	options.AddKubeletConfigFlags(fs, kc)
 	// Remember original feature gates, so we can merge with flag gates later

staging/src/k8s.io/cloud-provider/options/options.go

@@ -165,6 +165,7 @@ func (o *CloudControllerManagerOptions) Flags(allControllers []string, disabledB
 	fs.StringVar(&o.Generic.ClientConnection.Kubeconfig, "kubeconfig", o.Generic.ClientConnection.Kubeconfig, "Path to kubeconfig file with authorization and master location information (the master location can be overridden by the master flag).")
 	fs.DurationVar(&o.NodeStatusUpdateFrequency.Duration, "node-status-update-frequency", o.NodeStatusUpdateFrequency.Duration, "Specifies how often the controller updates nodes' status.")
 	utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic"))
+	//FIXME: what gateset(s) should this use?
 
 	return fss
 }

staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go

@@ -95,6 +95,7 @@ func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOpti
 	flags := cmd.Flags()
 	o.RecommendedOptions.AddFlags(flags)
 	utilfeature.DefaultMutableFeatureGate.AddFlag(flags)
+	//FIXME: what gateset(s) should this use?
 
 	return cmd
 }