util: set memory limit tuner to valid value after trigger tuning (#39343

)

ref #37816

util/gctuner/memory_limit_tuner.go

@@ -38,6 +38,9 @@ type memoryLimitTuner struct {
 	nextGCTriggeredByMemoryLimit atomicutil.Bool
 }
 
+// fallbackPercentage indicates the fallback memory limit percentage when turning.
+const fallbackPercentage float64 = 1.1
+
 // tuning check the memory nextGC and judge whether this GC is trigger by memory limit.
 // Go runtime ensure that it will be called serially.
 func (t *memoryLimitTuner) tuning() {
@@ -61,15 +64,15 @@ func (t *memoryLimitTuner) tuning() {
 			go func() {
 				memory.MemoryLimitGCLast.Store(time.Now())
 				memory.MemoryLimitGCTotal.Add(1)
-				debug.SetMemoryLimit(math.MaxInt64)
+				debug.SetMemoryLimit(t.calcMemoryLimit(fallbackPercentage))
 				resetInterval := 1 * time.Minute // Wait 1 minute and set back, to avoid frequent GC
 				failpoint.Inject("testMemoryLimitTuner", func(val failpoint.Value) {
 					if val, ok := val.(bool); val && ok {
 						resetInterval = 1 * time.Second
 					}
 				})
 				time.Sleep(resetInterval)
-				debug.SetMemoryLimit(t.calcMemoryLimit())
+				debug.SetMemoryLimit(t.calcMemoryLimit(t.GetPercentage()))
 				for !t.waitingReset.CompareAndSwap(true, false) {
 					continue
 				}
@@ -106,23 +109,27 @@ func (t *memoryLimitTuner) GetPercentage() float64 {
 // UpdateMemoryLimit updates the memory limit.
 // This function should be called when `tidb_server_memory_limit` or `tidb_server_memory_limit_gc_trigger` is modified.
 func (t *memoryLimitTuner) UpdateMemoryLimit() {
-	var memoryLimit = t.calcMemoryLimit()
+	var memoryLimit = t.calcMemoryLimit(t.GetPercentage())
 	if memoryLimit == math.MaxInt64 {
 		t.isTuning.Store(false)
+		memoryLimit = initGOMemoryLimitValue
 	} else {
 		t.isTuning.Store(true)
 	}
 	debug.SetMemoryLimit(memoryLimit)
 }
 
-func (t *memoryLimitTuner) calcMemoryLimit() int64 {
-	memoryLimit := int64(float64(memory.ServerMemoryLimit.Load()) * t.percentage.Load()) // `tidb_server_memory_limit` * `tidb_server_memory_limit_gc_trigger`
+func (*memoryLimitTuner) calcMemoryLimit(percentage float64) int64 {
+	memoryLimit := int64(float64(memory.ServerMemoryLimit.Load()) * percentage) // `tidb_server_memory_limit` * `tidb_server_memory_limit_gc_trigger`
 	if memoryLimit == 0 {
 		memoryLimit = math.MaxInt64
 	}
 	return memoryLimit
 }
 
+var initGOMemoryLimitValue int64
+
 func init() {
+	initGOMemoryLimitValue = debug.SetMemoryLimit(-1)
 	GlobalMemoryLimitTuner.Start()
 }

util/gctuner/memory_limit_tuner_test.go

@@ -15,7 +15,6 @@
 package gctuner
 
 import (
-	"math"
 	"runtime"
 	"runtime/debug"
 	"testing"
@@ -76,10 +75,9 @@ func TestGlobalMemoryTuner(t *testing.T) {
 	checkNextGCEqualMemoryLimit := func() {
 		runtime.ReadMemStats(r)
 		nextGC := r.NextGC
-		memoryLimit := GlobalMemoryLimitTuner.calcMemoryLimit()
-		// In golang source, nextGC = memoryLimit - three parts memory. So check 90%~100% here.
+		memoryLimit := GlobalMemoryLimitTuner.calcMemoryLimit(GlobalMemoryLimitTuner.GetPercentage())
+		// In golang source, nextGC = memoryLimit - three parts memory.
 		require.True(t, nextGC < uint64(memoryLimit))
-		require.True(t, nextGC > uint64(memoryLimit)/10*9)
 	}
 
 	memory600mb := allocator.alloc(600 << 20)
@@ -91,7 +89,7 @@ func TestGlobalMemoryTuner(t *testing.T) {
 	require.True(t, gcNum < getNowGCNum())
 	// Test waiting for reset
 	time.Sleep(500 * time.Millisecond)
-	require.Equal(t, int64(math.MaxInt64), debug.SetMemoryLimit(-1))
+	require.Equal(t, GlobalMemoryLimitTuner.calcMemoryLimit(fallbackPercentage), debug.SetMemoryLimit(-1))
 	gcNum = getNowGCNum()
 	memory100mb := allocator.alloc(100 << 20)
 	time.Sleep(100 * time.Millisecond)
@@ -102,7 +100,7 @@ func TestGlobalMemoryTuner(t *testing.T) {
 	runtime.GC()
 	// Trigger GC in 80% again
 	time.Sleep(500 * time.Millisecond)
-	require.Equal(t, GlobalMemoryLimitTuner.calcMemoryLimit(), debug.SetMemoryLimit(-1))
+	require.Equal(t, GlobalMemoryLimitTuner.calcMemoryLimit(GlobalMemoryLimitTuner.GetPercentage()), debug.SetMemoryLimit(-1))
 	time.Sleep(100 * time.Millisecond)
 	gcNum = getNowGCNum()
 	checkNextGCEqualMemoryLimit()