[DCA] External Metrics Server Configuration

cmd/cluster-agent/custommetrics/server.go

@@ -15,11 +15,7 @@ import (
 	"github.com/kubernetes-incubator/custom-metrics-apiserver/pkg/apiserver"
 	basecmd "github.com/kubernetes-incubator/custom-metrics-apiserver/pkg/cmd"
 	"github.com/kubernetes-incubator/custom-metrics-apiserver/pkg/provider"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/apimachinery/pkg/runtime/serializer"
-	genericapiserver "k8s.io/apiserver/pkg/server"
+	"github.com/spf13/pflag"
 
 	"github.com/DataDog/datadog-agent/pkg/clusteragent/custommetrics"
 	"github.com/DataDog/datadog-agent/pkg/clusteragent/externalmetrics"
@@ -37,11 +33,25 @@ type DatadogMetricsAdapter struct {
 	basecmd.AdapterBase
 }
 
+const (
+	metricsServerConf string = "external_metrics_provider.config"
+)
+
 // RunServer creates and start a k8s custom metrics API server
 func RunServer(ctx context.Context) error {
 	defer clearServerResources()
 	cmd = &DatadogMetricsAdapter{}
-	cmd.Flags()
+	cmd.Name = "datadog-custom-metrics-adapter"
+	cmd.FlagSet = pflag.NewFlagSet(cmd.Name, pflag.ExitOnError)
+
+	var c []string
+	for k, v := range config.Datadog.GetStringMapString(metricsServerConf) {
+		c = append(c, fmt.Sprintf("--%s=%s", k, v))
+	}
+
+	if err := cmd.Flags().Parse(c); err != nil {
+		return err
+	}
 
 	provider, err := cmd.makeProviderOrDie(ctx)
 	if err != nil {
@@ -50,7 +60,6 @@ func RunServer(ctx context.Context) error {
 
 	// TODO when implementing the custom metrics provider, add cmd.WithCustomMetrics(provider) here
 	cmd.WithExternalMetrics(provider)
-	cmd.Name = "datadog-custom-metrics-adapter"
 
 	conf, err := cmd.Config()
 	if err != nil {
@@ -99,52 +108,21 @@ func (a *DatadogMetricsAdapter) makeProviderOrDie(ctx context.Context) (provider
 
 // Config creates the configuration containing the required parameters to communicate with the APIServer as an APIService
 func (a *DatadogMetricsAdapter) Config() (*apiserver.Config, error) {
-	a.SecureServing.ServerCert.CertDirectory = "/etc/datadog-agent/certificates"
-	a.SecureServing.BindPort = config.Datadog.GetInt("external_metrics_provider.port")
-
+	if a.FlagSet.Lookup("cert-dir").Changed == false {
+		// Ensure backward compatibility. Was hardcoded before.
+		// Config flag is now to be added to the map `external_metrics_provider.config` as, `cert-dir`.
+		a.SecureServing.ServerCert.CertDirectory = "/etc/datadog-agent/certificates"
+	}
+	if a.FlagSet.Lookup("secure-port").Changed == false {
+		// Ensure backward compatibility. 443 by default, but will error out if incorrectly set.
+		// refer to apiserver code in k8s.io/apiserver/pkg/server/option/serving.go
+		a.SecureServing.BindPort = config.Datadog.GetInt("external_metrics_provider.port")
+	}
 	if err := a.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost", nil, []net.IP{net.ParseIP("127.0.0.1")}); err != nil {
 		log.Errorf("Failed to create self signed AuthN/Z configuration %#v", err)
 		return nil, fmt.Errorf("error creating self-signed certificates: %v", err)
 	}
-
-	scheme := runtime.NewScheme()
-	codecs := serializer.NewCodecFactory(scheme)
-
-	// we need to add the options to empty v1
-	// TODO fix the server code to avoid this
-	metav1.AddToGroupVersion(scheme, schema.GroupVersion{Version: "v1"})
-
-	// TODO: keep the generic API server from wanting this
-	unversioned := schema.GroupVersion{Group: "", Version: "v1"}
-	scheme.AddUnversionedTypes(unversioned,
-		&metav1.Status{},
-		&metav1.APIVersions{},
-		&metav1.APIGroupList{},
-		&metav1.APIGroup{},
-		&metav1.APIResourceList{},
-	)
-	serverConfig := genericapiserver.NewConfig(codecs)
-
-	err := a.SecureServing.ApplyTo(&serverConfig.SecureServing, &serverConfig.LoopbackClientConfig)
-	if err != nil {
-		log.Errorf("Error while converting SecureServing type %v", err)
-		return nil, err
-	}
-
-	// Get the certificates from the extension-apiserver-authentication ConfigMap
-	if err := a.Authentication.ApplyTo(&serverConfig.Authentication, serverConfig.SecureServing, nil); err != nil {
-		log.Errorf("Could not create Authentication configuration: %v", err)
-		return nil, err
-	}
-
-	if err := a.Authorization.ApplyTo(&serverConfig.Authorization); err != nil {
-		log.Infof("Could not create Authorization configuration: %v", err)
-		return nil, err
-	}
-
-	return &apiserver.Config{
-		GenericConfig: serverConfig,
-	}, nil
+	return a.CustomMetricsAdapterServerOptions.Config()
 }
 
 // clearServerResources closes the connection and the server

go.mod

@@ -172,7 +172,7 @@ require (
 	gopkg.in/zorkian/go-datadog-api.v2 v2.29.0
 	k8s.io/api v0.17.4
 	k8s.io/apimachinery v0.17.4
-	k8s.io/apiserver v0.17.4
+	k8s.io/apiserver v0.17.4 // indirect
 	k8s.io/autoscaler/vertical-pod-autoscaler v0.0.0-20200123122250-fa95810cfc1e
 	k8s.io/client-go v12.0.0+incompatible
 	k8s.io/cri-api v0.0.0

pkg/config/config.go

@@ -581,6 +581,7 @@ func InitConfig(config Config) {
 	config.BindEnvAndSetDefault("external_metrics_provider.use_datadogmetric_crd", false) // Use DatadogMetric CRD with custom Datadog Queries instead of ConfigMap
 	config.BindEnvAndSetDefault("kubernetes_event_collection_timeout", 100)               // timeout between two successful event collections in milliseconds.
 	config.BindEnvAndSetDefault("kubernetes_informers_resync_period", 60*5)               // value in seconds. Default to 5 minutes
+	config.BindEnvAndSetDefault("external_metrics_provider.config", map[string]string{})  // list of options that can be used to configure the external metrics server
 	config.BindEnvAndSetDefault("external_metrics_provider.local_copy_refresh_rate", 30)  // value in seconds
 	// Cluster check Autodiscovery
 	config.BindEnvAndSetDefault("cluster_checks.enabled", false)

test/e2e/argo-workflows/cluster-agent.yaml

@@ -158,6 +158,8 @@ spec:
               auth_token: "c9e21a248434a400b1de021dbdd554d790983a1212a5eac0ba36e79346ec52fd"
             external_metrics_provider:
               enabled: true
+              config:
+                secure-port: 6443
 
     - name: cluster-agent-deployment
       value: |
@@ -240,7 +242,7 @@ spec:
           ports:
           - protocol: TCP
             port: 443
-            targetPort: 443
+            targetPort: 6443
 
     - name: nginx-hpa
       value: |