http: add /debug/gogc to set GOGC on the fly

metrics/grafana/tidb_runtime.json

@@ -142,15 +142,15 @@
               "refId": "A"
             },
             {
-              "expr": "go_memstats_next_gc_bytes{instance=~\"$instance\"} / 2",
+              "expr": "go_memstats_next_gc_bytes{instance=~\"$instance\"} / (1 + tidb_server_gogc{instance=~\"$instance\"} / 100)",
               "format": "time_series",
               "hide": false,
               "intervalFactor": 1,
               "legendFormat": "estimate-inuse",
               "refId": "H"
             },
             {
-              "expr": "go_memstats_heap_alloc_bytes{instance=~\"$instance\"} - go_memstats_next_gc_bytes{instance=~\"$instance\"} / 2",
+              "expr": "go_memstats_heap_alloc_bytes{instance=~\"$instance\"} - go_memstats_next_gc_bytes{instance=~\"$instance\"} / (1 + tidb_server_gogc{instance=~\"$instance\"} / 100)",
               "format": "time_series",
               "hide": false,
               "intervalFactor": 1,

metrics/metrics.go

@@ -167,4 +167,5 @@ func RegisterMetrics() {
 	prometheus.MustRegister(TiKVNoAvailableConnectionCounter)
 	prometheus.MustRegister(TiKVAsyncCommitTxnCounter)
 	prometheus.MustRegister(MaxProcs)
+	prometheus.MustRegister(GOGC)
 }

metrics/server.go

@@ -178,6 +178,14 @@ var (
 			Name:      "maxprocs",
 			Help:      "The value of GOMAXPROCS.",
 		})
+
+	GOGC = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Namespace: "tidb",
+			Subsystem: "server",
+			Name:      "gogc",
+			Help:      "The value of GOGC",
+		})
 )
 
 // ExecuteErrorToLabel converts an execute error to label.

server/http_handler.go

@@ -23,6 +23,7 @@ import (
 	"math"
 	"net/http"
 	"net/url"
+	"runtime"
 	"strconv"
 	"strings"
 	"sync/atomic"
@@ -1606,7 +1607,9 @@ func (h *mvccTxnHandler) handleMvccGetByTxn(params map[string]string) (interface
 
 // serverInfo is used to report the servers info when do http request.
 type serverInfo struct {
-	IsOwner bool `json:"is_owner"`
+	IsOwner  bool `json:"is_owner"`
+	MaxProcs int  `json:"max_procs"`
+	GOGC     int  `json:"gogc"`
 	*infosync.ServerInfo
 }
 
@@ -1626,6 +1629,8 @@ func (h serverInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 		return
 	}
 	info.IsOwner = do.DDL().OwnerManager().IsOwner()
+	info.MaxProcs = runtime.GOMAXPROCS(0)
+	info.GOGC = util.GetGOGC()
 	writeData(w, info)
 }
 

server/http_status.go

@@ -21,6 +21,7 @@ import (
 	"crypto/x509"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"net"
 	"net/http"
 	"net/http/pprof"
@@ -182,6 +183,30 @@ func (s *Server) startHTTPServer() {
 	serverMux.HandleFunc("/debug/pprof/profile", pprof.Profile)
 	serverMux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
 	serverMux.HandleFunc("/debug/pprof/trace", pprof.Trace)
+	serverMux.HandleFunc("/debug/gogc", func(w http.ResponseWriter, r *http.Request) {
+		switch r.Method {
+		case http.MethodGet:
+			_, err := w.Write([]byte(strconv.Itoa(util.GetGOGC())))
+			terror.Log(err)
+		case http.MethodPost:
+			body, err := ioutil.ReadAll(r.Body)
+			if err != nil {
+				terror.Log(err)
+				return
+			}
+
+			val, err := strconv.Atoi(string(body))
+			if err != nil {
+				w.WriteHeader(http.StatusBadRequest)
+				if _, err := w.Write([]byte(err.Error())); err != nil {
+					terror.Log(err)
+				}
+				return
+			}
+
+			util.SetGOGC(val)
+		}
+	})
 
 	serverMux.HandleFunc("/debug/zip", func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="tidb_debug"`+time.Now().Format("20060102150405")+".zip"))

util/gogc.go

@@ -0,0 +1,45 @@
+// Copyright 2020 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package util
+
+import (
+	"os"
+	"runtime/debug"
+	"strconv"
+	"sync/atomic"
+
+	"github.com/pingcap/tidb/metrics"
+)
+
+var gogcValue int64
+
+func init() {
+	gogcValue = 100
+	if val, err := strconv.Atoi(os.Getenv("GOGC")); err == nil {
+		gogcValue = int64(val)
+	}
+	metrics.GOGC.Set(float64(gogcValue))
+}
+
+// SetGOGC update GOGC and related metrics.
+func SetGOGC(val int) {
+	debug.SetGCPercent(val)
+	metrics.GOGC.Set(float64(val))
+	atomic.StoreInt64(&gogcValue, int64(val))
+}
+
+// GetGOGC returns the current value of GOGC.
+func GetGOGC() int {
+	return int(atomic.LoadInt64(&gogcValue))
+}