Merge pull request kubernetes#48859 from victorgp/master

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Support for custom tls cipher suites in api server and kubelet

**What this PR does / why we need it**:
This pull request aims to solve the problem of users not able to set custom cipher suites in the api server.
Several users have requested this given that some default ciphers are vulnerable.
There is a discussion in kubernetes#41038 of how to implement this. The options are:
- Setting a fixed list of ciphers, but users will have different requirements so a fixed list would be problematic.
- Letting the user set them by parameter, this requires adding a new parameter that could be pretty long with the list of all the ciphers.

I implemented the second option, if the ciphers are not passed by parameter, the Go default ones will be used (same behavior as now).

**Which issue this PR fixes** 
fixes kubernetes#41038 

**Special notes for your reviewer**:
The ciphers in Go tls config are constants and the ones passed by parameters are a comma-separated list. I needed to create the `type CipherSuitesFlag` to support that conversion/mapping, because i couldn't find any way to do this type of reflection in Go.
If you think there is another way to implement this, let me know.

If you want to test it out, this is a ciphers combination i tested without the weak ones:

```
TLS_RSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA
```

If this is merged i will implement the same for the Kubelet.

**Release note**:
```release-note
kube-apiserver and kubelet now support customizing TLS ciphers via a `--tls-cipher-suites` flag
```

Author: Kubernetes Submit Queue

cmd/kubelet/app/options/options.go

@@ -442,6 +442,10 @@ func AddKubeletConfigFlags(fs *pflag.FlagSet, c *kubeletconfig.KubeletConfigurat
 		"If --tls-cert-file and --tls-private-key-file are not provided, a self-signed certificate and key "+
 		"are generated for the public address and saved to the directory passed to --cert-dir.")
 	fs.StringVar(&c.TLSPrivateKeyFile, "tls-private-key-file", c.TLSPrivateKeyFile, "File containing x509 private key matching --tls-cert-file.")
+	fs.StringSliceVar(&c.TLSCipherSuites, "tls-cipher-suites", c.TLSCipherSuites,
+		"Comma-separated list of cipher suites for the server. "+
+			"Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants). "+
+			"If omitted, the default Go cipher suites will be used")
 
 	fs.Int32Var(&c.RegistryPullQPS, "registry-qps", c.RegistryPullQPS, "If > 0, limit registry pull QPS to this value.  If 0, unlimited.")
 	fs.Int32Var(&c.RegistryBurst, "registry-burst", c.RegistryBurst, "Maximum size of a bursty pulls, temporarily allows pulls to burst to this number, while still not exceeding registry-qps. Only used if --registry-qps > 0")

cmd/kubelet/app/server.go

@@ -567,12 +567,19 @@ func InitializeTLS(kf *options.KubeletFlags, kc *kubeletconfiginternal.KubeletCo
 			glog.V(4).Infof("Using self-signed cert (%s, %s)", kc.TLSCertFile, kc.TLSPrivateKeyFile)
 		}
 	}
+
+	tlsCipherSuites, err := flag.TLSCipherSuites(kc.TLSCipherSuites)
+	if err != nil {
+		return nil, err
+	}
+
 	tlsOptions := &server.TLSOptions{
 		Config: &tls.Config{
 			// Can't use SSLv3 because of POODLE and BEAST
 			// Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher
 			// Can't use TLSv1.1 because of RC4 cipher usage
-			MinVersion: tls.VersionTLS12,
+			MinVersion:   tls.VersionTLS12,
+			CipherSuites: tlsCipherSuites,
 		},
 		CertFile: kc.TLSCertFile,
 		KeyFile:  kc.TLSPrivateKeyFile,

pkg/kubelet/apis/kubeletconfig/helpers_test.go

@@ -175,6 +175,7 @@ var (
 		"HairpinMode",
 		"HealthzBindAddress",
 		"HealthzPort",
+		"TLSCipherSuites[*]",
 		"IPTablesDropBit",
 		"IPTablesMasqueradeBit",
 		"ImageGCHighThresholdPercent",

pkg/kubelet/apis/kubeletconfig/types.go

@@ -85,6 +85,9 @@ type KubeletConfiguration struct {
 	// tlsPrivateKeyFile is the ile containing x509 private key matching
 	// tlsCertFile.
 	TLSPrivateKeyFile string
+	// TLSCipherSuites is the list of allowed cipher suites for the server.
+	// Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants).
+	TLSCipherSuites []string
 	// authentication specifies how requests to the Kubelet's server are authenticated
 	Authentication KubeletAuthentication
 	// authorization specifies how requests to the Kubelet's server are authorized

pkg/kubelet/apis/kubeletconfig/v1alpha1/types.go

@@ -85,6 +85,9 @@ type KubeletConfiguration struct {
 	// tlsPrivateKeyFile is the ile containing x509 private key matching
 	// tlsCertFile.
 	TLSPrivateKeyFile string `json:"tlsPrivateKeyFile"`
+	// TLSCipherSuites is the list of allowed cipher suites for the server.
+	// Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants).
+	TLSCipherSuites []string `json:"tlsCipherSuites"`
 	// authentication specifies how requests to the Kubelet's server are authenticated
 	Authentication KubeletAuthentication `json:"authentication"`
 	// authorization specifies how requests to the Kubelet's server are authorized

pkg/kubelet/apis/kubeletconfig/v1alpha1/zz_generated.conversion.go

@@ -51,6 +51,9 @@ type SecureServingOptions struct {
 	ServerCert GeneratableKeyCert
 	// SNICertKeys are named CertKeys for serving secure traffic with SNI support.
 	SNICertKeys []utilflag.NamedCertKey
+	// CipherSuites is the list of allowed cipher suites for the server.
+	// Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants).
+	CipherSuites []string
 }
 
 type CertKey struct {
@@ -134,6 +137,11 @@ func (s *SecureServingOptions) AddFlags(fs *pflag.FlagSet) {
 		"Controllers. This must be a valid PEM-encoded CA bundle. Altneratively, the certificate authority "+
 		"can be appended to the certificate provided by --tls-cert-file.")
 
+	fs.StringSliceVar(&s.CipherSuites, "tls-cipher-suites", s.CipherSuites,
+		"Comma-separated list of cipher suites for the server. "+
+			"Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants). "+
+			"If omitted, the default Go cipher suites will be used")
+
 	fs.Var(utilflag.NewNamedCertKeyArray(&s.SNICertKeys), "tls-sni-cert-key", ""+
 		"A pair of x509 certificate and private key file paths, optionally suffixed with a list of "+
 		"domain patterns which are fully qualified domain names, possibly with prefixed wildcard "+
@@ -233,6 +241,14 @@ func (s *SecureServingOptions) applyServingInfoTo(c *server.Config) error {
 		}
 	}
 
+	if len(s.CipherSuites) != 0 {
+		cipherSuites, err := utilflag.TLSCipherSuites(s.CipherSuites)
+		if err != nil {
+			return err
+		}
+		secureServingInfo.CipherSuites = cipherSuites
+	}
+
 	// load SNI certs
 	namedTLSCerts := make([]server.NamedTLSCert, 0, len(s.SNICertKeys))
 	for _, nck := range s.SNICertKeys {

pkg/kubelet/apis/kubeletconfig/v1alpha1/zz_generated.deepcopy.go

@@ -9,6 +9,7 @@ load(
 go_test(
     name = "go_default_test",
     srcs = [
+        "ciphersuites_flag_test.go",
         "colon_separated_multimap_string_string_test.go",
         "langle_separated_map_string_string_test.go",
         "map_string_bool_test.go",
@@ -23,6 +24,7 @@ go_test(
 go_library(
     name = "go_default_library",
     srcs = [
+        "ciphersuites_flag.go",
         "colon_separated_multimap_string_string.go",
         "configuration_map.go",
         "flags.go",

pkg/kubelet/apis/kubeletconfig/zz_generated.deepcopy.go

@@ -0,0 +1,64 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package flag
+
+import (
+	"crypto/tls"
+	"fmt"
+)
+
+// ciphers maps strings into tls package cipher constants in
+// https://golang.org/pkg/crypto/tls/#pkg-constants
+var ciphers = map[string]uint16{
+	"TLS_RSA_WITH_RC4_128_SHA":                tls.TLS_RSA_WITH_RC4_128_SHA,
+	"TLS_RSA_WITH_3DES_EDE_CBC_SHA":           tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA,
+	"TLS_RSA_WITH_AES_128_CBC_SHA":            tls.TLS_RSA_WITH_AES_128_CBC_SHA,
+	"TLS_RSA_WITH_AES_256_CBC_SHA":            tls.TLS_RSA_WITH_AES_256_CBC_SHA,
+	"TLS_RSA_WITH_AES_128_CBC_SHA256":         tls.TLS_RSA_WITH_AES_128_CBC_SHA256,
+	"TLS_RSA_WITH_AES_128_GCM_SHA256":         tls.TLS_RSA_WITH_AES_128_GCM_SHA256,
+	"TLS_RSA_WITH_AES_256_GCM_SHA384":         tls.TLS_RSA_WITH_AES_256_GCM_SHA384,
+	"TLS_ECDHE_ECDSA_WITH_RC4_128_SHA":        tls.TLS_ECDHE_ECDSA_WITH_RC4_128_SHA,
+	"TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA":    tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
+	"TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA":    tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
+	"TLS_ECDHE_RSA_WITH_RC4_128_SHA":          tls.TLS_ECDHE_RSA_WITH_RC4_128_SHA,
+	"TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA":     tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,
+	"TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA":      tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
+	"TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA":      tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
+	"TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256": tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,
+	"TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256":   tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256,
+	"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256":   tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+	"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256": tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+	"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384":   tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
+	"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384": tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
+	"TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305":    tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
+	"TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305":  tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
+}
+
+func TLSCipherSuites(cipherNames []string) ([]uint16, error) {
+	if len(cipherNames) == 0 {
+		return nil, nil
+	}
+	ciphersIntSlice := make([]uint16, 0)
+	for _, cipher := range cipherNames {
+		intValue, ok := ciphers[cipher]
+		if !ok {
+			return nil, fmt.Errorf("Cipher suite %s not supported or doesn't exist", cipher)
+		}
+		ciphersIntSlice = append(ciphersIntSlice, intValue)
+	}
+	return ciphersIntSlice, nil
+}

staging/src/k8s.io/apiserver/pkg/server/options/serving.go

@@ -0,0 +1,100 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package flag
+
+import (
+	"crypto/tls"
+	"fmt"
+	"go/importer"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestStrToUInt16(t *testing.T) {
+	tests := []struct {
+		flag           []string
+		expected       []uint16
+		expected_error bool
+	}{
+		{
+			// Happy case
+			flag:           []string{"TLS_RSA_WITH_RC4_128_SHA", "TLS_RSA_WITH_AES_128_CBC_SHA", "TLS_ECDHE_RSA_WITH_RC4_128_SHA", "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA"},
+			expected:       []uint16{tls.TLS_RSA_WITH_RC4_128_SHA, tls.TLS_RSA_WITH_AES_128_CBC_SHA, tls.TLS_ECDHE_RSA_WITH_RC4_128_SHA, tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA},
+			expected_error: false,
+		},
+		{
+			// One flag only
+			flag:           []string{"TLS_RSA_WITH_RC4_128_SHA"},
+			expected:       []uint16{tls.TLS_RSA_WITH_RC4_128_SHA},
+			expected_error: false,
+		},
+		{
+			// Empty flag
+			flag:           []string{},
+			expected:       nil,
+			expected_error: false,
+		},
+		{
+			// Duplicated flag
+			flag:           []string{"TLS_RSA_WITH_RC4_128_SHA", "TLS_RSA_WITH_AES_128_CBC_SHA", "TLS_ECDHE_RSA_WITH_RC4_128_SHA", "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA", "TLS_RSA_WITH_RC4_128_SHA"},
+			expected:       []uint16{tls.TLS_RSA_WITH_RC4_128_SHA, tls.TLS_RSA_WITH_AES_128_CBC_SHA, tls.TLS_ECDHE_RSA_WITH_RC4_128_SHA, tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, tls.TLS_RSA_WITH_RC4_128_SHA},
+			expected_error: false,
+		},
+		{
+			// Invalid flag
+			flag:           []string{"foo"},
+			expected:       nil,
+			expected_error: true,
+		},
+	}
+
+	for i, test := range tests {
+		uIntFlags, err := TLSCipherSuites(test.flag)
+		if reflect.DeepEqual(uIntFlags, test.expected) == false {
+			t.Errorf("%d: expected %+v, got %+v", i, test.expected, uIntFlags)
+		}
+		if test.expected_error && err == nil {
+			t.Errorf("%d: expecting error, got %+v", i, err)
+		}
+	}
+}
+
+func TestConstantMaps(t *testing.T) {
+	pkg, err := importer.Default().Import("crypto/tls")
+	if err != nil {
+		fmt.Printf("error: %s\n", err.Error())
+		return
+	}
+	discoveredCiphers := map[string]bool{}
+	for _, declName := range pkg.Scope().Names() {
+		if strings.HasPrefix(declName, "TLS_RSA_") || strings.HasPrefix(declName, "TLS_ECDHE_") {
+			discoveredCiphers[declName] = true
+		}
+	}
+
+	for k := range discoveredCiphers {
+		if _, ok := ciphers[k]; !ok {
+			t.Errorf("discovered cipher tls.%s not in ciphers map", k)
+		}
+	}
+	for k := range ciphers {
+		if _, ok := discoveredCiphers[k]; !ok {
+			t.Errorf("ciphers map has %s not in tls package", k)
+		}
+	}
+}

staging/src/k8s.io/apiserver/pkg/util/flag/BUILD

@@ -58,6 +58,7 @@ filegroup(
         "//test/integration/secrets:all-srcs",
         "//test/integration/serviceaccount:all-srcs",
         "//test/integration/storageclasses:all-srcs",
+        "//test/integration/tls:all-srcs",
         "//test/integration/ttlcontroller:all-srcs",
         "//test/integration/volume:all-srcs",
     ],

staging/src/k8s.io/apiserver/pkg/util/flag/ciphersuites_flag.go

@@ -0,0 +1,35 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
+
+go_test(
+    name = "go_default_test",
+    size = "large",
+    srcs = [
+        "ciphers_test.go",
+        "main_test.go",
+    ],
+    importpath = "k8s.io/kubernetes/test/integration/tls",
+    tags = ["integration"],
+    deps = [
+        "//cmd/kube-apiserver/app:go_default_library",
+        "//cmd/kube-apiserver/app/options:go_default_library",
+        "//test/integration/framework:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/server:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+        "//vendor/k8s.io/client-go/rest:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)