-
Notifications
You must be signed in to change notification settings - Fork 283
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
checker(dm): port some precheck from lightning #7617
Changes from 9 commits
5f98a30
06dbbaa
ce9fd63
1061df5
fdc4793
6230a88
eb6a184
a609857
0d952fa
15dc4d4
0b765db
7db68c7
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -25,12 +25,14 @@ import ( | |
"time" | ||
|
||
_ "github.com/go-sql-driver/mysql" // for mysql | ||
"github.com/pingcap/tidb/br/pkg/lightning/restore" | ||
"github.com/pingcap/tidb/dumpling/export" | ||
"github.com/pingcap/tidb/parser/mysql" | ||
"github.com/pingcap/tidb/util/dbutil" | ||
"github.com/pingcap/tidb/util/filter" | ||
regexprrouter "github.com/pingcap/tidb/util/regexpr-router" | ||
"github.com/pingcap/tiflow/dm/config" | ||
"github.com/pingcap/tiflow/dm/loader" | ||
"github.com/pingcap/tiflow/dm/pb" | ||
"github.com/pingcap/tiflow/dm/pkg/binlog" | ||
"github.com/pingcap/tiflow/dm/pkg/checker" | ||
|
@@ -336,6 +338,45 @@ func (c *Checker) Init(ctx context.Context) (err error) { | |
} | ||
} | ||
|
||
if instance.cfg.Mode != config.ModeIncrement && instance.cfg.LoaderConfig.ImportMode == config.LoadModePhysical { | ||
lCfg, err := loader.GetLightningConfig(loader.MakeGlobalConfig(instance.cfg), instance.cfg) | ||
if err != nil { | ||
return err | ||
} | ||
// bypass an Adjust error | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. what error? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. updated
|
||
lCfg.Mydumper.SourceDir = "noop://" | ||
err = lCfg.Adjust(ctx) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
builder, err := restore.NewPrecheckItemBuilderFromConfig(c.tctx.Context(), lCfg) | ||
if err != nil { | ||
return err | ||
} | ||
if _, ok := c.checkingItems[config.LightningEmptyRegionChecking]; ok { | ||
lChecker, err := builder.BuildPrecheckItem(restore.CheckTargetClusterEmptyRegion) | ||
if err != nil { | ||
return err | ||
} | ||
c.checkList = append(c.checkList, checker.NewLightningEmptyRegionChecker(lChecker)) | ||
} | ||
if _, ok := c.checkingItems[config.LightningRegionDistributionChecking]; ok { | ||
lChecker, err := builder.BuildPrecheckItem(restore.CheckTargetClusterRegionDist) | ||
if err != nil { | ||
return err | ||
} | ||
c.checkList = append(c.checkList, checker.NewLightningRegionDistributionChecker(lChecker)) | ||
} | ||
if _, ok := c.checkingItems[config.LightningDownstreamVersionChecking]; ok { | ||
lChecker, err := builder.BuildPrecheckItem(restore.CheckTargetClusterVersion) | ||
if err != nil { | ||
return err | ||
} | ||
c.checkList = append(c.checkList, checker.NewLightningClusterVersionChecker(lChecker)) | ||
} | ||
} | ||
|
||
c.tctx.Logger.Info(c.displayCheckingItems()) | ||
return nil | ||
} | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -37,6 +37,10 @@ const ( | |
BinlogDBChecking = "binlog_db" | ||
ConnNumberChecking = "conn_number" | ||
TargetDBPrivilegeChecking = "target_privilege" | ||
// lighting prechecks. | ||
LightningEmptyRegionChecking = "lightning_empty_region" | ||
LightningRegionDistributionChecking = "lightning_region_distribution" | ||
LightningDownstreamVersionChecking = "lightning_downstream_version" | ||
) | ||
|
||
// AllCheckingItems contains all checking items. | ||
|
@@ -56,6 +60,10 @@ var AllCheckingItems = map[string]string{ | |
BinlogDBChecking: "binlog db checking item", | ||
ConnNumberChecking: "connection number checking item", | ||
TargetDBPrivilegeChecking: "privileges of target DB checking item", | ||
// lightning prechecks | ||
LightningEmptyRegionChecking: "lightning empty region checking item", | ||
LightningRegionDistributionChecking: "lightning region distribution checking item", | ||
LightningDownstreamVersionChecking: "lightning downstream version checking item", | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I remember we need to check the empty space for local backend. Is it necessary here? |
||
} | ||
|
||
// MaxSourceIDLength is the max length for dm-worker source id. | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,117 @@ | ||
// Copyright 2022 PingCAP, Inc. | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package checker | ||
|
||
import ( | ||
"context" | ||
|
||
"github.com/pingcap/tidb/br/pkg/lightning/restore" | ||
) | ||
|
||
func convertLightningPrecheck( | ||
ctx context.Context, | ||
dmResult *Result, | ||
lightningPrechecker restore.PrecheckItem, | ||
failLevel State, | ||
) { | ||
lightningResult, err := lightningPrechecker.Check(ctx) | ||
if err != nil { | ||
markCheckError(dmResult, err) | ||
return | ||
} | ||
if !lightningResult.Passed { | ||
dmResult.State = failLevel | ||
dmResult.Errors = append(dmResult.Errors, &Error{Severity: failLevel, ShortErr: lightningResult.Message}) | ||
return | ||
} | ||
dmResult.State = StateSuccess | ||
} | ||
|
||
// LightningEmptyRegionChecker checks whether there are too many empty regions in the cluster. | ||
type LightningEmptyRegionChecker struct { | ||
inner restore.PrecheckItem | ||
} | ||
|
||
// NewLightningEmptyRegionChecker creates a new LightningEmptyRegionChecker. | ||
func NewLightningEmptyRegionChecker(lightningChecker restore.PrecheckItem) RealChecker { | ||
return &LightningEmptyRegionChecker{inner: lightningChecker} | ||
} | ||
|
||
// Name implements the RealChecker interface. | ||
func (c *LightningEmptyRegionChecker) Name() string { | ||
return "lightning_empty_region" | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. why not use There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. other checkers do not have same name with checking items. I'll ask PM |
||
} | ||
|
||
// Check implements the RealChecker interface. | ||
func (c *LightningEmptyRegionChecker) Check(ctx context.Context) *Result { | ||
result := &Result{ | ||
Name: c.Name(), | ||
Desc: "check downstream cluster has too many empty regions", | ||
State: StateFailure, | ||
} | ||
convertLightningPrecheck(ctx, result, c.inner, StateWarning) | ||
return result | ||
} | ||
|
||
// LightningRegionDistributionChecker checks whether the region distribution is balanced. | ||
type LightningRegionDistributionChecker struct { | ||
inner restore.PrecheckItem | ||
} | ||
|
||
// NewLightningRegionDistributionChecker creates a new LightningRegionDistributionChecker. | ||
func NewLightningRegionDistributionChecker(lightningChecker restore.PrecheckItem) RealChecker { | ||
return &LightningRegionDistributionChecker{inner: lightningChecker} | ||
} | ||
|
||
// Name implements the RealChecker interface. | ||
func (c *LightningRegionDistributionChecker) Name() string { | ||
return "lightning_region_distribution" | ||
} | ||
|
||
// Check implements the RealChecker interface. | ||
func (c *LightningRegionDistributionChecker) Check(ctx context.Context) *Result { | ||
result := &Result{ | ||
Name: c.Name(), | ||
Desc: "check downstream cluster region distribution", | ||
State: StateFailure, | ||
} | ||
convertLightningPrecheck(ctx, result, c.inner, StateWarning) | ||
return result | ||
} | ||
|
||
// LightningClusterVersionChecker checks whether the cluster version is compatible with Lightning. | ||
type LightningClusterVersionChecker struct { | ||
inner restore.PrecheckItem | ||
} | ||
|
||
// NewLightningClusterVersionChecker creates a new LightningClusterVersionChecker. | ||
func NewLightningClusterVersionChecker(lightningChecker restore.PrecheckItem) RealChecker { | ||
return &LightningClusterVersionChecker{inner: lightningChecker} | ||
} | ||
|
||
// Name implements the RealChecker interface. | ||
func (c *LightningClusterVersionChecker) Name() string { | ||
return "lightning_cluster_version" | ||
} | ||
|
||
// Check implements the RealChecker interface. | ||
func (c *LightningClusterVersionChecker) Check(ctx context.Context) *Result { | ||
result := &Result{ | ||
Name: c.Name(), | ||
Desc: "check downstream cluster version", | ||
State: StateFailure, | ||
} | ||
convertLightningPrecheck(ctx, result, c.inner, StateFailure) | ||
return result | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Do we need some checks for tidb backend?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
the checks in this PR is only for local backend because they are related to storage layer. I'll add more later
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Does DM support local backend now? 🤔
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
#7448