Skip to content
This repository has been archived by the owner on Jul 24, 2024. It is now read-only.

restore: split & scatter regions concurrently #1363

Closed
wants to merge 27 commits into from
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
92eeef1
restore: concurrently split
YuJuncen Jul 19, 2021
49d5d0e
backup: revert some unrelated changes
YuJuncen Jul 19, 2021
3130688
restore: fix CI
YuJuncen Jul 19, 2021
0bfe38a
testing: fix CI
YuJuncen Jul 19, 2021
7a2b9fe
Merge branch 'master' into conc-split
YuJuncen Jul 19, 2021
e3a8b9e
restore: return eraly in failure
YuJuncen Jul 19, 2021
cf99e08
Merge branch 'master' of https://github.com/pingcap/br into conc-split
YuJuncen Jul 19, 2021
ec78919
Merge branch 'conc-split' of https://github.com/yujuncen/br into conc…
YuJuncen Jul 19, 2021
dcad51c
Merge branch 'master' into conc-split
YuJuncen Jul 19, 2021
53436d2
restore: FnOnce called twice...
YuJuncen Jul 21, 2021
aa5ecef
Merge branch 'conc-split' of https://github.com/yujuncen/br into conc…
YuJuncen Jul 21, 2021
18b4010
Merge branch 'master' into conc-split
YuJuncen Jul 21, 2021
3f3fed0
Merge branch 'master' into conc-split
YuJuncen Jul 22, 2021
be155aa
*: revert unrelated changes
YuJuncen Jul 23, 2021
7d3bc0c
restore: revert unrelative changes
YuJuncen Jul 23, 2021
82a04b5
restore: revert unrelative change
YuJuncen Jul 23, 2021
39a3050
Merge branch 'master' into conc-split
YuJuncen Jul 26, 2021
a16168f
restore: add some comments
YuJuncen Jul 26, 2021
4b722e4
Merge branch 'conc-split' of https://github.com/yujuncen/br into conc…
YuJuncen Jul 26, 2021
58c1bc5
Merge branch 'master' into conc-split
YuJuncen Jul 28, 2021
812ad75
restore: address comments
YuJuncen Aug 5, 2021
e4ef413
Update pipeline_items.go
YuJuncen Aug 5, 2021
d14a2a3
Update util.go
YuJuncen Aug 5, 2021
42d77b2
Update client.go
YuJuncen Aug 5, 2021
2e8882f
Merge branch 'master' into conc-split
YuJuncen Aug 5, 2021
fa28dfb
Merge branch 'master' into conc-split
YuJuncen Aug 6, 2021
ac6fe36
Merge branch 'master' into conc-split
YuJuncen Aug 6, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
restore: fix CI
  • Loading branch information
YuJuncen committed Jul 19, 2021
commit 3130688fc59729e0d49e870b1b474429ee29a8b0
5 changes: 3 additions & 2 deletions pkg/logutil/rate.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import (
"go.uber.org/zap"
)

// Rater is a trivial rate tracer.
// TrivialRater is a trivial rate tracer.
// It doesn't record any time sequence, and always
// return the average speed over all the time.
// TODO: replace it with Prometheus.
Expand All @@ -20,6 +20,7 @@ type TrivialRater struct {
current uint64
}

// NewTrivialRater make a trivial rater.
func NewTrivialRater() TrivialRater {
return TrivialRater{
start: time.Now(),
Expand All @@ -37,7 +38,7 @@ func (r *TrivialRater) Rate(unit time.Duration) float64 {
return float64(atomic.LoadUint64(&r.current)) / float64(time.Since(r.start)/unit)
}

// Log log the current rate(in ops per second) to the info level.
// L make a logger with the current speed.
func (r *TrivialRater) L() *zap.Logger {
return log.With(zap.String("speed", fmt.Sprintf("%.2f ops/s", r.Rate(time.Second))))
}
1 change: 1 addition & 0 deletions pkg/restore/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -378,6 +378,7 @@ func SplitRangesAndThen(
}))
}

// SplitRanges is the synchronous version of SplitRangesAndThen.
func SplitRanges(
ctx context.Context,
client *Client,
Expand Down
5 changes: 3 additions & 2 deletions pkg/rtree/logging.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"go.uber.org/zap/zapcore"

"github.com/docker/go-units"

"github.com/pingcap/br/pkg/logutil"
"github.com/pingcap/br/pkg/redact"
)
Expand Down Expand Up @@ -56,7 +57,7 @@ func (rs rangesMarshaler) MarshalLogObject(encoder zapcore.ObjectEncoder) error

encoder.AddInt("file-count", totalFile)
encoder.AddUint64("kv-paris-count", totalKV)
encoder.AddString("after-compress-size", units.HumanSize(float64(totalBytes)))
encoder.AddString("data-size", units.HumanSize(float64(totalSize)))
encoder.AddString("data-size", units.HumanSize(float64(totalBytes)))
encoder.AddString("after-compress-size", units.HumanSize(float64(totalSize)))
return nil
}
17 changes: 8 additions & 9 deletions pkg/rtree/logging_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,14 @@ func (s *testLoggingSuite) TestLogRanges(c *C) {
count int
expect string
}{
{0, `{"ranges": {"total": 0, "ranges": [], "totalFiles": 0, "totalKVs": 0, "totalBytes": 0, "totalSize": 0}}`},
{1, `{"ranges": {"total": 1, "ranges": ["[30, 31)"], "totalFiles": 1, "totalKVs": 0, "totalBytes": 0, "totalSize": 0}}`},
{2, `{"ranges": {"total": 2, "ranges": ["[30, 31)", "[31, 32)"], "totalFiles": 2, "totalKVs": 1, "totalBytes": 1, "totalSize": 1}}`},
{3, `{"ranges": {"total": 3, "ranges": ["[30, 31)", "[31, 32)", "[32, 33)"], "totalFiles": 3, "totalKVs": 3, "totalBytes": 3, "totalSize": 3}}`},
{4, `{"ranges": {"total": 4, "ranges": ["[30, 31)", "[31, 32)", "[32, 33)", "[33, 34)"], "totalFiles": 4, "totalKVs": 6, "totalBytes": 6, "totalSize": 6}}`},
{5, `{"ranges": {"total": 5, "ranges": ["[30, 31)", "(skip 3)", "[34, 35)"], "totalFiles": 5, "totalKVs": 10, "totalBytes": 10, "totalSize": 10}}`},
{6, `{"ranges": {"total": 6, "ranges": ["[30, 31)", "(skip 4)", "[35, 36)"], "totalFiles": 6, "totalKVs": 15, "totalBytes": 15, "totalSize": 15}}`},
{1024, `{"ranges": {"total": 1024, "ranges": ["[30, 31)", "(skip 1022)", "[31303233, 31303234)"], "totalFiles": 1024, "totalKVs": 523776, "totalBytes": 523776, "totalSize": 523776}}`},
}
{0, `{"ranges": {"total": 0, "ranges": [], "file-count": 0, "kv-paris-count": 0, "data-size": "0B", "after-compress-size": "0B"}}`},
{1, `{"ranges": {"total": 1, "ranges": ["[30, 31)"], "file-count": 1, "kv-paris-count": 0, "data-size": "0B", "after-compress-size": "0B"}}`},
{2, `{"ranges": {"total": 2, "ranges": ["[30, 31)", "[31, 32)"], "file-count": 2, "kv-paris-count": 1, "data-size": "1B", "after-compress-size": "0B"}}`},
{3, `{"ranges": {"total": 3, "ranges": ["[30, 31)", "[31, 32)", "[32, 33)"], "file-count": 3, "kv-paris-count": 3, "data-size": "3B", "after-compress-size": "0B"}}`},
{4, `{"ranges": {"total": 4, "ranges": ["[30, 31)", "[31, 32)", "[32, 33)", "[33, 34)"], "file-count": 4, "kv-paris-count": 6, "data-size": "6B", "after-compress-size": "0B"}}`},
{5, `{"ranges": {"total": 5, "ranges": ["[30, 31)", "(skip 3)", "[34, 35)"], "file-count": 5, "kv-paris-count": 10, "data-size": "10B", "after-compress-size": "0B"}}`},
{6, `{"ranges": {"total": 6, "ranges": ["[30, 31)", "(skip 4)", "[35, 36)"], "file-count": 6, "kv-paris-count": 15, "data-size": "15B", "after-compress-size": "0B"}}`},
{1024, `{"ranges": {"total": 1024, "ranges": ["[30, 31)", "(skip 1022)", "[31303233, 31303234)"], "file-count": 1024, "kv-paris-count": 523776, "data-size": "523.8kB", "after-compress-size": "0B"}}`}}

encoder := zapcore.NewConsoleEncoder(zapcore.EncoderConfig{})
for _, cs := range cases {
Expand Down
4 changes: 2 additions & 2 deletions pkg/task/restore.go
Original file line number Diff line number Diff line change
Expand Up @@ -149,11 +149,11 @@ func (cfg *RestoreConfig) ParseFromFlags(flags *pflag.FlagSet) error {
}
cfg.PDConcurrency, err = flags.GetUint(FlagPDConcurrency)
if err != nil {
return errors.Wrapf(err, "failed to get flag %s", FlagPDConcurrency)
return errors.Annotatef(err, "failed to get flag %s", FlagPDConcurrency)
}
cfg.BatchFlushInterval, err = flags.GetDuration(FlagBatchFlushInterval)
if err != nil {
return errors.Wrapf(err, "failed to get flag %s", FlagBatchFlushInterval)
return errors.Annotatef(err, "failed to get flag %s", FlagBatchFlushInterval)
}
return nil
}
Expand Down