Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

*: GC-aware memory track for analyze (#35248) #36380

Open
wants to merge 1 commit into
base: release-6.1
Choose a base branch
from
+1,720 −14
Open

*: GC-aware memory track for analyze (#35248) #36380

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
440 changes: 440 additions & 0 deletions executor/analyze_col.go
View file
Open in desktop

Large diffs are not rendered by default.

798 changes: 798 additions & 0 deletions executor/analyze_col_v2.go
View file
Open in desktop

Large diffs are not rendered by default.

10 changes: 9 additions & 1 deletion metrics/grafana/tidb.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1960,10 +1960,18 @@
},
{
"expr": "tidb_server_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}",
"hide": true,
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{type}}-{{instance}}",
"legendFormat": "{{module}}-{{type}}-{{instance}}",
"refId": "H"
},
{
"expr": "sum(tidb_server_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}) by (module, instance)",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{module}}-{{instance}}",
"refId": "I"
}
],
"thresholds": [],
Expand Down
2 changes: 1 addition & 1 deletion metrics/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ var (
Subsystem: "server",
Name: "memory_usage",
Help: "Memory Usage",
}, []string{LblType})
}, []string{LblModule, LblType})
)

// metrics labels.
Expand Down
10 changes: 10 additions & 0 deletions metrics/session.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,4 +165,14 @@ const (
LblVersion = "version"
LblHash = "hash"
LblCTEType = "cte_type"
<<<<<<< HEAD
=======
LblIdle = "idle"
LblRunning = "executing_sql"
LblLockWaiting = "waiting_for_lock"
LblCommitting = "committing"
LblRollingBack = "rolling_back"
LblHasLock = "has_lock"
LblModule = "module"
>>>>>>> fd6f5dd79... *: GC-aware memory track for analyze (#35248)
)
34 changes: 34 additions & 0 deletions sessionctx/variable/sysvar.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,11 @@ import (
"github.com/pingcap/tidb/util/collate"
"github.com/pingcap/tidb/util/logutil"
"github.com/pingcap/tidb/util/mathutil"
<<<<<<< HEAD
=======
"github.com/pingcap/tidb/util/memory"
"github.com/pingcap/tidb/util/paging"
>>>>>>> fd6f5dd79... *: GC-aware memory track for analyze (#35248)
"github.com/pingcap/tidb/util/stmtsummary"
"github.com/pingcap/tidb/util/tikvutil"
"github.com/pingcap/tidb/util/tls"
Expand Down Expand Up @@ -797,6 +802,35 @@ var defaultSysVars = []*SysVar{
return err
},
},
<<<<<<< HEAD
=======
{Scope: ScopeGlobal, Name: TiDBEnableConcurrentDDL, Value: BoolToOnOff(DefTiDBEnableConcurrentDDL), Type: TypeBool, SetGlobal: func(s *SessionVars, val string) error {
EnableConcurrentDDL.Store(TiDBOptOn(val))
return nil
}, GetGlobal: func(s *SessionVars) (string, error) {
return BoolToOnOff(EnableConcurrentDDL.Load()), nil
}},
{Scope: ScopeGlobal, Name: TiDBEnableNoopVariables, Value: BoolToOnOff(DefTiDBEnableNoopVariables), Type: TypeEnum, PossibleValues: []string{Off, On, Warn}, SetGlobal: func(s *SessionVars, val string) error {
EnableNoopVariables.Store(TiDBOptOn(val))
return nil
}, GetGlobal: func(s *SessionVars) (string, error) {
return BoolToOnOff(EnableNoopVariables.Load()), nil
}},
{Scope: ScopeGlobal, Name: TiDBAuthSigningCert, Value: "", Type: TypeStr, SetGlobal: func(s *SessionVars, val string) error {
sessionstates.SetCertPath(val)
return nil
}},
{Scope: ScopeGlobal, Name: TiDBAuthSigningKey, Value: "", Type: TypeStr, SetGlobal: func(s *SessionVars, val string) error {
sessionstates.SetKeyPath(val)
return nil
}},
{Scope: ScopeGlobal, Name: TiDBEnableGCAwareMemoryTrack, Value: BoolToOnOff(DefEnableTiDBGCAwareMemoryTrack), Type: TypeBool, SetGlobal: func(s *SessionVars, val string) error {
memory.EnableGCAwareMemoryTrack.Store(TiDBOptOn(val))
return nil
}, GetGlobal: func(s *SessionVars) (string, error) {
return BoolToOnOff(memory.EnableGCAwareMemoryTrack.Load()), nil
}},
>>>>>>> fd6f5dd79... *: GC-aware memory track for analyze (#35248)

/* The system variables below have GLOBAL and SESSION scope */
{Scope: ScopeGlobal | ScopeSession, Name: SQLSelectLimit, Value: "18446744073709551615", Type: TypeUnsigned, MinValue: 0, MaxValue: math.MaxUint64, SetSession: func(s *SessionVars, val string) error {
Expand Down
214 changes: 214 additions & 0 deletions sessionctx/variable/tidb_vars.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -704,6 +704,19 @@ const (
// TiDBMaxAutoAnalyzeTime is the max time that auto analyze can run. If auto analyze runs longer than the value, it
// will be killed. 0 indicates that there is no time limit.
TiDBMaxAutoAnalyzeTime = "tidb_max_auto_analyze_time"
<<<<<<< HEAD
=======
// TiDBEnableConcurrentDDL indicates whether to enable the new DDL framework.
TiDBEnableConcurrentDDL = "tidb_enable_concurrent_ddl"
// TiDBAuthSigningCert indicates the path of the signing certificate to do token-based authentication.
TiDBAuthSigningCert = "tidb_auth_signing_cert"
// TiDBAuthSigningKey indicates the path of the signing key to do token-based authentication.
TiDBAuthSigningKey = "tidb_auth_signing_key"
// TiDBGenerateBinaryPlan indicates whether binary plan should be generated in slow log and statements summary.
TiDBGenerateBinaryPlan = "tidb_generate_binary_plan"
// TiDBEnableGCAwareMemoryTrack indicates whether to turn-on GC-aware memory track.
TiDBEnableGCAwareMemoryTrack = "tidb_enable_gc_aware_memory_track"
>>>>>>> fd6f5dd79... *: GC-aware memory track for analyze (#35248)
)

// TiDB intentional limits
Expand All @@ -717,6 +730,7 @@ const (

// Default TiDB system variable values.
const (
<<<<<<< HEAD
DefHostname = "localhost"
DefIndexLookupConcurrency = ConcurrencyUnset
DefIndexLookupJoinConcurrency = ConcurrencyUnset
Expand Down Expand Up @@ -892,6 +906,206 @@ const (
DefTiDBEnablePrepPlanCache = true
DefTiDBPrepPlanCacheSize = 100
DefTiDBPrepPlanCacheMemoryGuardRatio = 0.1
=======
DefHostname = "localhost"
DefIndexLookupConcurrency = ConcurrencyUnset
DefIndexLookupJoinConcurrency = ConcurrencyUnset
DefIndexSerialScanConcurrency = 1
DefIndexJoinBatchSize = 25000
DefIndexLookupSize = 20000
DefDistSQLScanConcurrency = 15
DefBuildStatsConcurrency = 4
DefAutoAnalyzeRatio = 0.5
DefAutoAnalyzeStartTime = "00:00 +0000"
DefAutoAnalyzeEndTime = "23:59 +0000"
DefAutoIncrementIncrement = 1
DefAutoIncrementOffset = 1
DefChecksumTableConcurrency = 4
DefSkipUTF8Check = false
DefSkipASCIICheck = false
DefOptAggPushDown = false
DefOptCartesianBCJ = 1
DefOptMPPOuterJoinFixedBuildSide = false
DefOptWriteRowID = false
DefOptEnableCorrelationAdjustment = true
DefOptLimitPushDownThreshold = 100
DefOptCorrelationThreshold = 0.9
DefOptCorrelationExpFactor = 1
DefOptCPUFactor = 3.0
DefOptCopCPUFactor = 3.0
DefOptTiFlashConcurrencyFactor = 24.0
DefOptNetworkFactor = 1.0
DefOptScanFactor = 1.5
DefOptDescScanFactor = 3.0
DefOptSeekFactor = 20.0
DefOptMemoryFactor = 0.001
DefOptDiskFactor = 1.5
DefOptConcurrencyFactor = 3.0
DefOptCPUFactorV2 = 30.0
DefOptCopCPUFactorV2 = 30.0
DefOptTiFlashCPUFactorV2 = 2.0
DefOptNetworkFactorV2 = 4.0
DefOptScanFactorV2 = 100.0
DefOptDescScanFactorV2 = 150.0
DefOptTiFlashScanFactorV2 = 15.0
DefOptSeekFactorV2 = 9500000.0
DefOptMemoryFactorV2 = 0.001
DefOptDiskFactorV2 = 1.5
DefOptConcurrencyFactorV2 = 3.0
DefOptInSubqToJoinAndAgg = true
DefOptPreferRangeScan = false
DefBatchInsert = false
DefBatchDelete = false
DefBatchCommit = false
DefCurretTS = 0
DefInitChunkSize = 32
DefMinPagingSize = int(paging.MinPagingSize)
DefMaxChunkSize = 1024
DefDMLBatchSize = 0
DefMaxPreparedStmtCount = -1
DefWaitTimeout = 28800
DefTiDBMemQuotaApplyCache = 32 << 20 // 32MB.
DefTiDBMemQuotaBindingCache = 64 << 20 // 64MB.
DefTiDBGeneralLog = false
DefTiDBPProfSQLCPU = 0
DefTiDBRetryLimit = 10
DefTiDBDisableTxnAutoRetry = true
DefTiDBConstraintCheckInPlace = false
DefTiDBHashJoinConcurrency = ConcurrencyUnset
DefTiDBProjectionConcurrency = ConcurrencyUnset
DefBroadcastJoinThresholdSize = 100 * 1024 * 1024
DefBroadcastJoinThresholdCount = 10 * 1024
DefTiDBOptimizerSelectivityLevel = 0
DefTiDBOptimizerEnableNewOFGB = false
DefTiDBEnableOuterJoinReorder = true
DefTiDBAllowBatchCop = 1
DefTiDBAllowMPPExecution = true
DefTiDBHashExchangeWithNewCollation = true
DefTiDBEnforceMPPExecution = false
DefTiFlashMaxThreads = -1
DefTiDBMPPStoreFailTTL = "60s"
DefTiDBTxnMode = ""
DefTiDBRowFormatV1 = 1
DefTiDBRowFormatV2 = 2
DefTiDBDDLReorgWorkerCount = 4
DefTiDBDDLReorgBatchSize = 256
DefTiDBDDLErrorCountLimit = 512
DefTiDBMaxDeltaSchemaCount = 1024
DefTiDBPointGetCache = false
DefTiDBPlacementMode = PlacementModeStrict
DefTiDBEnableAutoIncrementInGenerated = false
DefTiDBHashAggPartialConcurrency = ConcurrencyUnset
DefTiDBHashAggFinalConcurrency = ConcurrencyUnset
DefTiDBWindowConcurrency = ConcurrencyUnset
DefTiDBMergeJoinConcurrency = 1 // disable optimization by default
DefTiDBStreamAggConcurrency = 1
DefTiDBForcePriority = mysql.NoPriority
DefEnableWindowFunction = true
DefEnablePipelinedWindowFunction = true
DefEnableStrictDoubleTypeCheck = true
DefEnableVectorizedExpression = true
DefTiDBOptJoinReorderThreshold = 0
DefTiDBDDLSlowOprThreshold = 300
DefTiDBUseFastAnalyze = false
DefTiDBSkipIsolationLevelCheck = false
DefTiDBExpensiveQueryTimeThreshold = 60 // 60s
DefTiDBScatterRegion = false
DefTiDBWaitSplitRegionFinish = true
DefWaitSplitRegionTimeout = 300 // 300s
DefTiDBEnableNoopFuncs = Off
DefTiDBEnableNoopVariables = true
DefTiDBAllowRemoveAutoInc = false
DefTiDBUsePlanBaselines = true
DefTiDBEvolvePlanBaselines = false
DefTiDBEvolvePlanTaskMaxTime = 600 // 600s
DefTiDBEvolvePlanTaskStartTime = "00:00 +0000"
DefTiDBEvolvePlanTaskEndTime = "23:59 +0000"
DefInnodbLockWaitTimeout = 50 // 50s
DefTiDBStoreLimit = 0
DefTiDBMetricSchemaStep = 60 // 60s
DefTiDBMetricSchemaRangeDuration = 60 // 60s
DefTiDBFoundInPlanCache = false
DefTiDBFoundInBinding = false
DefTiDBEnableCollectExecutionInfo = true
DefTiDBAllowAutoRandExplicitInsert = false
DefTiDBEnableClusteredIndex = ClusteredIndexDefModeIntOnly
DefTiDBRedactLog = false
DefTiDBRestrictedReadOnly = false
DefTiDBSuperReadOnly = false
DefTiDBShardAllocateStep = math.MaxInt64
DefTiDBEnableTelemetry = true
DefTiDBEnableParallelApply = false
DefTiDBEnableAmendPessimisticTxn = false
DefTiDBPartitionPruneMode = "static"
DefTiDBEnableRateLimitAction = true
DefTiDBEnableAsyncCommit = false
DefTiDBEnable1PC = false
DefTiDBGuaranteeLinearizability = true
DefTiDBAnalyzeVersion = 2
DefTiDBEnableIndexMergeJoin = false
DefTiDBTrackAggregateMemoryUsage = true
DefTiDBEnableExchangePartition = false
DefCTEMaxRecursionDepth = 1000
DefTiDBTmpTableMaxSize = 64 << 20 // 64MB.
DefTiDBEnableLocalTxn = false
DefTiDBTSOClientBatchMaxWaitTime = 0.0 // 0ms
DefTiDBEnableTSOFollowerProxy = false
DefTiDBEnableOrderedResultMode = false
DefTiDBEnablePseudoForOutdatedStats = true
DefTiDBRegardNULLAsPoint = true
DefEnablePlacementCheck = true
DefTimestamp = "0"
DefTiDBEnableStmtSummary = true
DefTiDBStmtSummaryInternalQuery = false
DefTiDBStmtSummaryRefreshInterval = 1800
DefTiDBStmtSummaryHistorySize = 24
DefTiDBStmtSummaryMaxStmtCount = 3000
DefTiDBStmtSummaryMaxSQLLength = 4096
DefTiDBCapturePlanBaseline = Off
DefTiDBEnableIndexMerge = true
DefEnableLegacyInstanceScope = true
DefTiDBTableCacheLease = 3 // 3s
DefTiDBPersistAnalyzeOptions = true
DefTiDBEnableColumnTracking = false
DefTiDBStatsLoadSyncWait = 0
DefTiDBStatsLoadPseudoTimeout = false
DefSysdateIsNow = false
DefTiDBEnableMutationChecker = false
DefTiDBTxnAssertionLevel = AssertionOffStr
DefTiDBIgnorePreparedCacheCloseStmt = false
DefTiDBBatchPendingTiFlashCount = 4000
DefRCReadCheckTS = false
DefTiDBRemoveOrderbyInSubquery = false
DefTiDBSkewDistinctAgg = false
DefTiDBReadStaleness = 0
DefTiDBGCMaxWaitTime = 24 * 60 * 60
DefMaxAllowedPacket uint64 = 67108864
DefTiDBEnableBatchDML = false
DefTiDBMemQuotaQuery = 1073741824 // 1GB
DefTiDBStatsCacheMemQuota = 0
MaxTiDBStatsCacheMemQuota = 1024 * 1024 * 1024 * 1024 // 1TB
DefTiDBQueryLogMaxLen = 4096
DefRequireSecureTransport = false
DefTiDBCommitterConcurrency = 128
DefTiDBBatchDMLIgnoreError = false
DefTiDBMemQuotaAnalyze = -1
DefTiDBEnableAutoAnalyze = true
DefTiDBMemOOMAction = "CANCEL"
DefTiDBMaxAutoAnalyzeTime = 12 * 60 * 60
DefTiDBEnablePrepPlanCache = true
DefTiDBPrepPlanCacheSize = 100
DefTiDBPrepPlanCacheMemoryGuardRatio = 0.1
DefTiDBEnableConcurrentDDL = true
DefTiDBSimplifiedMetrics = false
DefTiDBEnablePaging = true
DefTiFlashFineGrainedShuffleStreamCount = -1
DefStreamCountWhenMaxThreadsNotSet = 8
DefTiFlashFineGrainedShuffleBatchSize = 8192
DefAdaptiveClosestReadThreshold = 4096
DefTiDBEnableAnalyzeSnapshot = false
DefTiDBGenerateBinaryPlan = true
DefEnableTiDBGCAwareMemoryTrack = true
>>>>>>> fd6f5dd79... *: GC-aware memory track for analyze (#35248)
)

// Process global variables.
Expand Down
Loading