Merge pull request #95981 from caesarxuchao/http2-healthcheck

Enables HTTP/2 health check

Kubernetes-commit: afeac926fa79179bfd0c5e718459086845a2a80c

pkg/util/net/http.go

@@ -33,6 +33,7 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"time"
 	"unicode"
 	"unicode/utf8"
 
@@ -132,13 +133,61 @@ func SetTransportDefaults(t *http.Transport) *http.Transport {
 	if s := os.Getenv("DISABLE_HTTP2"); len(s) > 0 {
 		klog.Infof("HTTP2 has been explicitly disabled")
 	} else if allowsHTTP2(t) {
-		if err := http2.ConfigureTransport(t); err != nil {
+		if err := configureHTTP2Transport(t); err != nil {
 			klog.Warningf("Transport failed http2 configuration: %v", err)
 		}
 	}
 	return t
 }
 
+func readIdleTimeoutSeconds() int {
+	ret := 30
+	// User can set the readIdleTimeout to 0 to disable the HTTP/2
+	// connection health check.
+	if s := os.Getenv("HTTP2_READ_IDLE_TIMEOUT_SECONDS"); len(s) > 0 {
+		i, err := strconv.Atoi(s)
+		if err != nil {
+			klog.Warningf("Illegal HTTP2_READ_IDLE_TIMEOUT_SECONDS(%q): %v."+
+				" Default value %d is used", s, err, ret)
+			return ret
+		}
+		ret = i
+	}
+	return ret
+}
+
+func pingTimeoutSeconds() int {
+	ret := 15
+	if s := os.Getenv("HTTP2_PING_TIMEOUT_SECONDS"); len(s) > 0 {
+		i, err := strconv.Atoi(s)
+		if err != nil {
+			klog.Warningf("Illegal HTTP2_PING_TIMEOUT_SECONDS(%q): %v."+
+				" Default value %d is used", s, err, ret)
+			return ret
+		}
+		ret = i
+	}
+	return ret
+}
+
+func configureHTTP2Transport(t *http.Transport) error {
+	t2, err := http2.ConfigureTransports(t)
+	if err != nil {
+		return err
+	}
+	// The following enables the HTTP/2 connection health check added in
+	// https://github.com/golang/net/pull/55. The health check detects and
+	// closes broken transport layer connections. Without the health check,
+	// a broken connection can linger too long, e.g., a broken TCP
+	// connection will be closed by the Linux kernel after 13 to 30 minutes
+	// by default, which caused
+	// https://github.com/kubernetes/client-go/issues/374 and
+	// https://github.com/kubernetes/kubernetes/issues/87615.
+	t2.ReadIdleTimeout = time.Duration(readIdleTimeoutSeconds()) * time.Second
+	t2.PingTimeout = time.Duration(pingTimeoutSeconds()) * time.Second
+	return nil
+}
+
 func allowsHTTP2(t *http.Transport) bool {
 	if t.TLSClientConfig == nil || len(t.TLSClientConfig.NextProtos) == 0 {
 		// the transport expressed no NextProto preference, allow

pkg/util/net/http_test.go

@@ -1071,3 +1071,39 @@ func TestIsProbableEOF(t *testing.T) {
 		})
 	}
 }
+
+func setEnv(key, value string) func() {
+	originalValue := os.Getenv(key)
+	os.Setenv(key, value)
+	return func() {
+		os.Setenv(key, originalValue)
+	}
+}
+
+func TestReadIdleTimeoutSeconds(t *testing.T) {
+	reset := setEnv("HTTP2_READ_IDLE_TIMEOUT_SECONDS", "60")
+	if e, a := 60, readIdleTimeoutSeconds(); e != a {
+		t.Errorf("expected %d, got %d", e, a)
+	}
+	reset()
+
+	reset = setEnv("HTTP2_READ_IDLE_TIMEOUT_SECONDS", "illegal value")
+	if e, a := 30, readIdleTimeoutSeconds(); e != a {
+		t.Errorf("expected %d, got %d", e, a)
+	}
+	reset()
+}
+
+func TestPingTimeoutSeconds(t *testing.T) {
+	reset := setEnv("HTTP2_PING_TIMEOUT_SECONDS", "60")
+	if e, a := 60, pingTimeoutSeconds(); e != a {
+		t.Errorf("expected %d, got %d", e, a)
+	}
+	reset()
+
+	reset = setEnv("HTTP2_PING_TIMEOUT_SECONDS", "illegal value")
+	if e, a := 15, pingTimeoutSeconds(); e != a {
+		t.Errorf("expected %d, got %d", e, a)
+	}
+	reset()
+}