*: replace storage.Create by storage.New, update BR dependencies (#286)…

… (#287)

Makefile

@@ -112,7 +112,6 @@ lint: tools
 tidy:
 	@echo "go mod tidy"
 	GO111MODULE=on go mod tidy
-	cd tools && GO111MODULE=on go mod tidy
 	git diff --exit-code go.mod go.sum tools/go.mod tools/go.sum
 
 bins:

go.mod

@@ -3,34 +3,47 @@ module github.com/pingcap/dumpling
 go 1.13
 
 require (
-	github.com/DATA-DOG/go-sqlmock v1.4.1
+	github.com/DATA-DOG/go-sqlmock v1.5.0
+	github.com/HdrHistogram/hdrhistogram-go v0.9.0 // indirect
+	github.com/carlmjohnson/flagext v0.21.0 // indirect
+	github.com/codahale/hdrhistogram v0.9.0 // indirect
 	github.com/coreos/go-semver v0.3.0
 	github.com/docker/go-units v0.4.0
+	github.com/fatih/color v1.10.0 // indirect
 	github.com/fsouza/fake-gcs-server v1.19.0 // indirect
 	github.com/go-sql-driver/mysql v1.5.0
+	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/mock v1.4.4 // indirect
-	github.com/mattn/go-colorable v0.1.7 // indirect
-	github.com/mattn/go-runewidth v0.0.9 // indirect
-	github.com/onsi/ginkgo v1.11.0 // indirect
-	github.com/onsi/gomega v1.8.1 // indirect
-	github.com/pingcap/br v4.0.0-beta.2.0.20210203034957-7bc483ab69d5+incompatible
+	github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf // indirect
+	github.com/google/go-cmp v0.5.2 // indirect
+	github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/mattn/go-runewidth v0.0.10 // indirect
+	github.com/pingcap/br v4.0.14-0.20210715051733-8afeee8f2859+incompatible
 	github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712
 	github.com/pingcap/errors v0.11.5-0.20201126102027-b0a155152ca3
-	github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce
-	github.com/pingcap/log v0.0.0-20201112100606-8f1e84a3abc8
-	github.com/pingcap/tidb v1.1.0-beta.0.20210129045644-ea6ccf82e934 // indirect
+	github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd
+	github.com/pingcap/log v0.0.0-20210317133921-96f4fcab92a4
+	github.com/pingcap/tidb v1.1.0-beta.0.20210715025933-fb96fe79e72a // indirect
 	github.com/pingcap/tidb-tools v4.0.9-0.20201127090955-2707c97b3853+incompatible
 	github.com/prometheus/client_golang v1.5.1
 	github.com/prometheus/client_model v0.2.0
+	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/soheilhy/cmux v0.1.4
 	github.com/spf13/pflag v1.0.5
-	github.com/syndtr/goleveldb v1.0.1-0.20190625010220-02440ea7a285 // indirect
-	github.com/tikv/pd v1.1.0-beta.0.20201125070607-d4b90eee0c70
-	go.etcd.io/bbolt v1.3.5 // indirect
+	github.com/tikv/pd v0.0.0-20210105112549-e5be7fd38659
+	github.com/uber/jaeger-lib v2.4.0+incompatible // indirect
+	github.com/xitongsys/parquet-go v1.6.0 // indirect
 	go.etcd.io/etcd v0.5.0-alpha.5.0.20200824191128-ae9734ed278b
 	go.uber.org/zap v1.16.0
+	golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a // indirect
+	golang.org/x/mod v0.4.2 // indirect
+	golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4 // indirect
 	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
-	golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8 // indirect
-	golang.org/x/tools v0.0.0-20200823205832-c024452afbcd // indirect
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	golang.org/x/sys v0.0.0-20210324051608-47abb6519492 // indirect
+	golang.org/x/text v0.3.6 // indirect
+	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	honnef.co/go/tools v0.1.4 // indirect
+	modernc.org/mathutil v1.2.2 // indirect
 )

tools/go.sum

@@ -115,7 +115,6 @@ github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:x
 github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
 github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
-github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 h1:23T5iq8rbUYlhpt5DB4XJkc6BU31uODLD1o1gKvZmD0=
 github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4=
@@ -605,7 +604,6 @@ google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
 google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
 google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
-google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

tools/go_mod_guard.go

@@ -7,7 +7,7 @@ import (
 	_ "github.com/golangci/golangci-lint/pkg/commands"
 
 	// revive is a file-based linter
-	_ "github.com/mgechev/revive/lint"
+	_ "github.com/mgechev/revive"
 
 	// govet checks for code correctness
 	_ "github.com/dnephin/govet"

v4/export/config.go

@@ -6,7 +6,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"net/http"
 	"regexp"
 	"strconv"
 	"strings"
@@ -526,18 +525,10 @@ func (conf *Config) createExternalStorage(ctx context.Context) (storage.External
 	if err != nil {
 		return nil, errors.Trace(err)
 	}
-	httpClient := http.DefaultClient
-	httpClient.Timeout = 30 * time.Second
-	maxIdleConnsPerHost := http.DefaultMaxIdleConnsPerHost
-	if conf.Threads > maxIdleConnsPerHost {
-		maxIdleConnsPerHost = conf.Threads
-	}
-	transport := http.DefaultTransport.(*http.Transport).Clone()
-	transport.MaxIdleConnsPerHost = maxIdleConnsPerHost
-	httpClient.Transport = transport
 
 	return storage.New(ctx, b, &storage.ExternalStorageOptions{
-		HTTPClient: httpClient,
+		SkipCheckPath:   true,
+		SendCredentials: false,
 	})
 }
 

v4/export/dump.go

@@ -963,11 +963,7 @@ func initLogger(d *Dumper) error {
 // createExternalStore is an initialization step of Dumper.
 func createExternalStore(d *Dumper) error {
 	tctx, conf := d.tctx, d.conf
-	b, err := storage.ParseBackend(conf.OutputDirPath, &conf.BackendOptions)
-	if err != nil {
-		return errors.Trace(err)
-	}
-	extStore, err := storage.Create(tctx, b, false)
+	extStore, err := conf.createExternalStorage(tctx)
 	if err != nil {
 		return errors.Trace(err)
 	}

v4/export/writer_test.go

@@ -12,7 +12,6 @@ import (
 	tcontext "github.com/pingcap/dumpling/v4/context"
 
 	"github.com/DATA-DOG/go-sqlmock"
-	"github.com/pingcap/br/pkg/storage"
 	. "github.com/pingcap/check"
 )
 
@@ -27,9 +26,7 @@ func defaultConfigForTest(c *C) *Config {
 }
 
 func (s *testWriterSuite) newWriter(conf *Config, c *C) *Writer {
-	b, err := storage.ParseBackend(conf.OutputDirPath, &conf.BackendOptions)
-	c.Assert(err, IsNil)
-	extStore, err := storage.Create(context.Background(), b, false)
+	extStore, err := conf.createExternalStorage(context.Background())
 	c.Assert(err, IsNil)
 	db, _, err := sqlmock.New()
 	c.Assert(err, IsNil)