Skip to content

Commit

Permalink
resolver: replace standard http client to go-retryablehttp
Browse files Browse the repository at this point in the history 
Previously, our snapshot will download the image through the go standard http
client, if the http download fails and does not do a retry. Now we retry
for defaultRetryMax = 4 times.

Fixes: #54

Signed-off-by: zhaoshang <zhaoshangsjtu@linux.alibaba.com>
  • Loading branch information
@power-more
power-more committed Jun 23, 2022
1 parent 0de2dec commit 9dcce02
Show file tree
Hide file tree
Showing 4 changed files with 20 additions and 8 deletions.
4 changes: 3 additions & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,9 @@ require (
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da
github.com/google/go-containerregistry v0.5.1
github.com/google/uuid v1.2.0
github.com/hashicorp/go-retryablehttp v0.7.1
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.0.2-0.20211117181255-693428a734f5
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.11.0
github.com/prometheus/client_model v0.2.0
Expand Down Expand Up @@ -42,10 +44,10 @@ require (
github.com/docker/libcontainer v2.2.1+incompatible // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
github.com/klauspost/compress v1.15.1 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/moby/sys/mountinfo v0.5.0 // indirect
github.com/opencontainers/image-spec v1.0.2-0.20211117181255-693428a734f5 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/russross/blackfriday/v2 v2.0.1 // indirect
Expand Down
5 changes: 5 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -460,12 +460,17 @@ github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyN
github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM=
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
github.com/hashicorp/go-retryablehttp v0.7.1 h1:sUiuQAnLlbvmExtFQs72iFW/HXeUn8Z1aJLQ4LJJbTQ=
github.com/hashicorp/go-retryablehttp v0.7.1/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
Expand Down
2 changes: 1 addition & 1 deletion pkg/filesystem/nydus/fs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ func (fs *filesystem) PrepareBlobLayer(ctx context.Context, snapshot storage.Sna
start := time.Now()
defer func() {
duration := time.Since(start)
log.G(ctx).Infof("total nydus prepare data layer duration %d", duration.Milliseconds())
log.G(ctx).Infof("total nydus prepare data layer duration %d ms", duration.Milliseconds())
}()

if fs.imageMode == OnDemand {
Expand Down
17 changes: 11 additions & 6 deletions pkg/filesystem/nydus/resolver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"github.com/containerd/nydus-snapshotter/pkg/auth"
"github.com/containerd/nydus-snapshotter/pkg/utils/registry"
"github.com/google/go-containerregistry/pkg/name"
retryablehttp "github.com/hashicorp/go-retryablehttp"
"github.com/pkg/errors"
)

Expand Down Expand Up @@ -61,22 +62,26 @@ func (r *Resolver) Resolve(ref, digest string, labels map[string]string) (io.Rea
nref.Context().RepositoryStr(),
digest)

req, err := http.NewRequest("GET", url, nil)
req, err := retryablehttp.NewRequest("GET", url, nil)
if err != nil {
return nil, errors.Wrapf(err, "faild to new http get %s", url)
}

client := &http.Client{
Transport: tr,
Timeout: time.Second * 30,
}
client := newRetryHTTPClient(tr)
res, err := client.Do(req)
if err != nil {
return nil, errors.Wrapf(err, "faild to http get %s", url)
}

if res.StatusCode != http.StatusOK {
return nil, fmt.Errorf("failed to GET request with code %d", res.StatusCode)
return nil, fmt.Errorf("failed to GET request with code %d", res.StatusCode)
}
return res.Body, nil
}

func newRetryHTTPClient(tr http.RoundTripper) *retryablehttp.Client {
retryClient := retryablehttp.NewClient()
retryClient.HTTPClient.Transport = tr
retryClient.HTTPClient.Timeout = time.Second * 30
return retryClient
}

0 comments on commit 9dcce02

Please sign in to comment.