This is an automated cherry-pick of pingcap#54822

Signed-off-by: ti-chi-bot <ti-community-prow-bot@tidb.io>

pkg/executor/test/analyzetest/BUILD.bazel

@@ -26,7 +26,10 @@ go_test(
         "//pkg/sessionctx",
         "//pkg/sessionctx/variable",
         "//pkg/statistics",
+<<<<<<< HEAD
         "//pkg/statistics/handle/autoanalyze",
+=======
+>>>>>>> 7e73ddc91b5 (statistics: add metrics for unneeded analyze table (#54822))
         "//pkg/testkit",
         "//pkg/util/dbterror/exeerrors",
         "@com_github_pingcap_errors//:errors",

pkg/executor/test/analyzetest/analyze_test.go

@@ -39,7 +39,10 @@ import (
 	"github.com/pingcap/tidb/pkg/sessionctx"
 	"github.com/pingcap/tidb/pkg/sessionctx/variable"
 	"github.com/pingcap/tidb/pkg/statistics"
+<<<<<<< HEAD
 	"github.com/pingcap/tidb/pkg/statistics/handle/autoanalyze"
+=======
+>>>>>>> 7e73ddc91b5 (statistics: add metrics for unneeded analyze table (#54822))
 	"github.com/pingcap/tidb/pkg/testkit"
 	"github.com/pingcap/tidb/pkg/util/dbterror/exeerrors"
 	"github.com/stretchr/testify/require"
@@ -743,11 +746,19 @@ func TestSavedAnalyzeOptions(t *testing.T) {
 		tk.MustExec(fmt.Sprintf("set global tidb_auto_analyze_ratio = %v", originalVal2))
 	}()
 	tk.MustExec("set global tidb_auto_analyze_ratio = 0.01")
+<<<<<<< HEAD
 	originalVal3 := autoanalyze.AutoAnalyzeMinCnt
 	defer func() {
 		autoanalyze.AutoAnalyzeMinCnt = originalVal3
 	}()
 	autoanalyze.AutoAnalyzeMinCnt = 0
+=======
+	originalVal3 := statistics.AutoAnalyzeMinCnt
+	defer func() {
+		statistics.AutoAnalyzeMinCnt = originalVal3
+	}()
+	statistics.AutoAnalyzeMinCnt = 0
+>>>>>>> 7e73ddc91b5 (statistics: add metrics for unneeded analyze table (#54822))
 
 	tk.MustExec("use test")
 	tk.MustExec("set @@session.tidb_analyze_version = 2")
@@ -1085,11 +1096,19 @@ func TestSavedAnalyzeColumnOptions(t *testing.T) {
 		tk.MustExec(fmt.Sprintf("set global tidb_auto_analyze_ratio = %v", originalVal2))
 	}()
 	tk.MustExec("set global tidb_auto_analyze_ratio = 0.01")
+<<<<<<< HEAD
 	originalVal3 := autoanalyze.AutoAnalyzeMinCnt
 	defer func() {
 		autoanalyze.AutoAnalyzeMinCnt = originalVal3
 	}()
 	autoanalyze.AutoAnalyzeMinCnt = 0
+=======
+	originalVal3 := statistics.AutoAnalyzeMinCnt
+	defer func() {
+		statistics.AutoAnalyzeMinCnt = originalVal3
+	}()
+	statistics.AutoAnalyzeMinCnt = 0
+>>>>>>> 7e73ddc91b5 (statistics: add metrics for unneeded analyze table (#54822))
 	originalVal4 := tk.MustQuery("select @@tidb_enable_column_tracking").Rows()[0][0].(string)
 	defer func() {
 		tk.MustExec(fmt.Sprintf("set global tidb_enable_column_tracking = %v", originalVal4))
@@ -1966,9 +1985,15 @@ func testKillAutoAnalyze(t *testing.T, ver int) {
 	tk := testkit.NewTestKit(t, store)
 	oriStart := tk.MustQuery("select @@tidb_auto_analyze_start_time").Rows()[0][0].(string)
 	oriEnd := tk.MustQuery("select @@tidb_auto_analyze_end_time").Rows()[0][0].(string)
+<<<<<<< HEAD
 	autoanalyze.AutoAnalyzeMinCnt = 0
 	defer func() {
 		autoanalyze.AutoAnalyzeMinCnt = 1000
+=======
+	statistics.AutoAnalyzeMinCnt = 0
+	defer func() {
+		statistics.AutoAnalyzeMinCnt = 1000
+>>>>>>> 7e73ddc91b5 (statistics: add metrics for unneeded analyze table (#54822))
 		tk.MustExec(fmt.Sprintf("set global tidb_auto_analyze_start_time='%v'", oriStart))
 		tk.MustExec(fmt.Sprintf("set global tidb_auto_analyze_end_time='%v'", oriEnd))
 	}()
@@ -2050,9 +2075,15 @@ func TestKillAutoAnalyzeIndex(t *testing.T) {
 	tk := testkit.NewTestKit(t, store)
 	oriStart := tk.MustQuery("select @@tidb_auto_analyze_start_time").Rows()[0][0].(string)
 	oriEnd := tk.MustQuery("select @@tidb_auto_analyze_end_time").Rows()[0][0].(string)
+<<<<<<< HEAD
 	autoanalyze.AutoAnalyzeMinCnt = 0
 	defer func() {
 		autoanalyze.AutoAnalyzeMinCnt = 1000
+=======
+	statistics.AutoAnalyzeMinCnt = 0
+	defer func() {
+		statistics.AutoAnalyzeMinCnt = 1000
+>>>>>>> 7e73ddc91b5 (statistics: add metrics for unneeded analyze table (#54822))
 		tk.MustExec(fmt.Sprintf("set global tidb_auto_analyze_start_time='%v'", oriStart))
 		tk.MustExec(fmt.Sprintf("set global tidb_auto_analyze_end_time='%v'", oriEnd))
 	}()
@@ -2805,12 +2836,21 @@ func TestAutoAnalyzeAwareGlobalVariableChange(t *testing.T) {
 		"3 0",
 	))
 
+<<<<<<< HEAD
 	originalVal1 := autoanalyze.AutoAnalyzeMinCnt
 	originalVal2 := tk.MustQuery("select @@global.tidb_auto_analyze_ratio").Rows()[0][0].(string)
 	autoanalyze.AutoAnalyzeMinCnt = 0
 	tk.MustExec("set global tidb_auto_analyze_ratio = 0.001")
 	defer func() {
 		autoanalyze.AutoAnalyzeMinCnt = originalVal1
+=======
+	originalVal1 := statistics.AutoAnalyzeMinCnt
+	originalVal2 := tk.MustQuery("select @@global.tidb_auto_analyze_ratio").Rows()[0][0].(string)
+	statistics.AutoAnalyzeMinCnt = 0
+	tk.MustExec("set global tidb_auto_analyze_ratio = 0.001")
+	defer func() {
+		statistics.AutoAnalyzeMinCnt = originalVal1
+>>>>>>> 7e73ddc91b5 (statistics: add metrics for unneeded analyze table (#54822))
 		tk.MustExec(fmt.Sprintf("set global tidb_auto_analyze_ratio = %v", originalVal2))
 	}()
 

pkg/executor/test/analyzetest/memorycontrol/BUILD.bazel

@@ -13,7 +13,11 @@ go_test(
         "//pkg/config",
         "//pkg/executor",
         "//pkg/sessionctx/variable",
+<<<<<<< HEAD
         "//pkg/statistics/handle/autoanalyze",
+=======
+        "//pkg/statistics",
+>>>>>>> 7e73ddc91b5 (statistics: add metrics for unneeded analyze table (#54822))
         "//pkg/testkit",
         "//pkg/util",
         "//pkg/util/memory",

pkg/executor/test/analyzetest/memorycontrol/memory_control_test.go

@@ -22,7 +22,11 @@ import (
 
 	"github.com/pingcap/failpoint"
 	"github.com/pingcap/tidb/pkg/executor"
+<<<<<<< HEAD
 	"github.com/pingcap/tidb/pkg/statistics/handle/autoanalyze"
+=======
+	"github.com/pingcap/tidb/pkg/statistics"
+>>>>>>> 7e73ddc91b5 (statistics: add metrics for unneeded analyze table (#54822))
 	"github.com/pingcap/tidb/pkg/testkit"
 	"github.com/pingcap/tidb/pkg/util"
 	"github.com/pingcap/tidb/pkg/util/memory"
@@ -144,12 +148,21 @@ func TestGlobalMemoryControlForAutoAnalyze(t *testing.T) {
 	require.Len(t, rs0.Rows(), 0)
 
 	h := dom.StatsHandle()
+<<<<<<< HEAD
 	originalVal4 := autoanalyze.AutoAnalyzeMinCnt
 	originalVal5 := tk.MustQuery("select @@global.tidb_auto_analyze_ratio").Rows()[0][0].(string)
 	autoanalyze.AutoAnalyzeMinCnt = 0
 	tk.MustExec("set global tidb_auto_analyze_ratio = 0.001")
 	defer func() {
 		autoanalyze.AutoAnalyzeMinCnt = originalVal4
+=======
+	originalVal4 := statistics.AutoAnalyzeMinCnt
+	originalVal5 := tk.MustQuery("select @@global.tidb_auto_analyze_ratio").Rows()[0][0].(string)
+	statistics.AutoAnalyzeMinCnt = 0
+	tk.MustExec("set global tidb_auto_analyze_ratio = 0.001")
+	defer func() {
+		statistics.AutoAnalyzeMinCnt = originalVal4
+>>>>>>> 7e73ddc91b5 (statistics: add metrics for unneeded analyze table (#54822))
 		tk.MustExec(fmt.Sprintf("set global tidb_auto_analyze_ratio = %v", originalVal5))
 	}()
 

pkg/statistics/BUILD.bazel

@@ -86,7 +86,10 @@ go_test(
         "//pkg/parser/mysql",
         "//pkg/sessionctx",
         "//pkg/sessionctx/stmtctx",
+<<<<<<< HEAD
         "//pkg/statistics/handle/autoanalyze",
+=======
+>>>>>>> 7e73ddc91b5 (statistics: add metrics for unneeded analyze table (#54822))
         "//pkg/testkit",
         "//pkg/testkit/testdata",
         "//pkg/testkit/testmain",

pkg/statistics/handle/autoanalyze/BUILD.bazel

@@ -40,6 +40,11 @@ go_test(
         "//pkg/sessionctx",
         "//pkg/sessionctx/variable",
         "//pkg/statistics",
+<<<<<<< HEAD
+=======
+        "//pkg/statistics/handle/util",
+        "//pkg/statistics/handle/util/test",
+>>>>>>> 7e73ddc91b5 (statistics: add metrics for unneeded analyze table (#54822))
         "//pkg/testkit",
         "@com_github_stretchr_testify//require",
         "@com_github_tikv_client_go_v2//oracle",

pkg/statistics/handle/autoanalyze/autoanalyze.go

@@ -277,11 +277,37 @@ func RandomPickOneTableAndTryAutoAnalyze(
 // AutoAnalyzeMinCnt means if the count of table is less than this value, we needn't do auto analyze.
 var AutoAnalyzeMinCnt int64 = 1000
 
+<<<<<<< HEAD
 func autoAnalyzeTable(sctx sessionctx.Context,
 	statsHandle statsutil.StatsHandle,
 	tblInfo *model.TableInfo, statsTbl *statistics.Table,
 	ratio float64, sql string, params ...interface{}) bool {
 	if statsTbl.Pseudo || statsTbl.RealtimeCount < AutoAnalyzeMinCnt {
+=======
+	for _, def := range defs {
+		partitionStats[def.ID] = statsHandle.GetPartitionStatsForAutoAnalyze(tblInfo, def.ID)
+	}
+
+	return partitionStats
+}
+
+// Determine whether the table and index require analysis.
+func tryAutoAnalyzeTable(
+	sctx sessionctx.Context,
+	statsHandle statstypes.StatsHandle,
+	sysProcTracker sysproctrack.Tracker,
+	tblInfo *model.TableInfo,
+	statsTbl *statistics.Table,
+	ratio float64,
+	sql string,
+	params ...any,
+) bool {
+	// 1. If the statistics are either not loaded or are classified as pseudo, there is no need for analyze
+	//    Pseudo statistics can be created by the optimizer, so we need to double check it.
+	// 2. If the table is too small, we don't want to waste time to analyze it.
+	//    Leave the opportunity to other bigger tables.
+	if statsTbl == nil || statsTbl.Pseudo || statsTbl.RealtimeCount < statistics.AutoAnalyzeMinCnt {
+>>>>>>> 7e73ddc91b5 (statistics: add metrics for unneeded analyze table (#54822))
 		return false
 	}
 	if needAnalyze, reason := NeedAnalyzeTable(statsTbl, 20*statsHandle.Lease(), ratio); needAnalyze {
@@ -374,8 +400,17 @@ func autoAnalyzePartitionTableInDynamicMode(sctx sessionctx.Context,
 	analyzePartitionBatchSize := int(variable.AutoAnalyzePartitionBatchSize.Load())
 	partitionNames := make([]interface{}, 0, len(partitionDefs))
 	for _, def := range partitionDefs {
+<<<<<<< HEAD
 		partitionStatsTbl := statsHandle.GetPartitionStats(tblInfo, def.ID)
 		if partitionStatsTbl.Pseudo || partitionStatsTbl.RealtimeCount < AutoAnalyzeMinCnt {
+=======
+		partitionStats := partitionStats[def.ID]
+		// 1. If the statistics are either not loaded or are classified as pseudo, there is no need for analyze.
+		//	  Pseudo statistics can be created by the optimizer, so we need to double check it.
+		// 2. If the table is too small, we don't want to waste time to analyze it.
+		//    Leave the opportunity to other bigger tables.
+		if partitionStats == nil || partitionStats.Pseudo || partitionStats.RealtimeCount < statistics.AutoAnalyzeMinCnt {
+>>>>>>> 7e73ddc91b5 (statistics: add metrics for unneeded analyze table (#54822))
 			continue
 		}
 		if needAnalyze, reason := NeedAnalyzeTable(partitionStatsTbl, 20*statsHandle.Lease(), ratio); needAnalyze {