Add support download with retry (#42)

* Add support download with retry

* Fix the jcli download cmd

* Add comments for export functions

.github/workflows/pull-request.yaml

@@ -30,6 +30,10 @@ jobs:
         with:
           version: latest
           args: release --skip-publish --rm-dist
+      - name: Test against the cmd
+        run: |
+          sudo ./release/http-downloader_linux_amd64/hd install -t 8 jenkins-zh/jenkins-cli/jcli
+          jcli version
       - name: Upload Artifact for darwin
         uses: actions/upload-artifact@v2
         with:

cmd/get.go

@@ -31,6 +31,10 @@ func NewGetCmd() (cmd *cobra.Command) {
 	flags.StringVarP(&opt.Output, "output", "o", "", "Write output to <file> instead of stdout.")
 	flags.BoolVarP(&opt.Fetch, "fetch", "", true,
 		"If fetch the latest config from https://github.com/LinuxSuRen/hd-home")
+	flags.IntVarP(&opt.Timeout, "time", "", 10,
+		`The default timeout in seconds with the HTTP request`)
+	flags.IntVarP(&opt.MaxAttempts, "max-attempts", "", 10,
+		`Max times to attempt to download, zero means there's no retry action'`)
 	flags.BoolVarP(&opt.ShowProgress, "show-progress", "", true, "If show the progress of download")
 	flags.Int64VarP(&opt.ContinueAt, "continue-at", "", -1, "ContinueAt")
 	flags.IntVarP(&opt.Thread, "thread", "t", 0,
@@ -48,6 +52,8 @@ type downloadOption struct {
 	Output       string
 	ShowProgress bool
 	Fetch        bool
+	Timeout      int
+	MaxAttempts  int
 
 	ContinueAt int64
 
@@ -292,7 +298,7 @@ func (o *downloadOption) preRunE(cmd *cobra.Command, args []string) (err error)
 
 func (o *downloadOption) runE(cmd *cobra.Command, args []string) (err error) {
 	if o.Thread <= 1 {
-		err = pkg.DownloadWithContinue(o.URL, o.Output, o.ContinueAt, 0, o.ShowProgress)
+		err = pkg.DownloadWithContinue(o.URL, o.Output, o.ContinueAt, -1, 0, o.ShowProgress)
 	} else {
 		err = pkg.DownloadFileWithMultipleThreadKeepParts(o.URL, o.Output, o.Thread, o.KeepPart, o.ShowProgress)
 	}

go.mod

@@ -13,4 +13,5 @@ require (
 	github.com/onsi/ginkgo v1.15.0
 	github.com/onsi/gomega v1.10.4
 	github.com/spf13/cobra v1.1.3
+	github.com/stretchr/testify v1.7.0
 )

pkg/http.go

@@ -12,6 +12,7 @@ import (
 	"path"
 	"strconv"
 	"sync"
+	"time"
 )
 
 const (
@@ -39,7 +40,10 @@ type HTTPDownloader struct {
 	// PreStart returns false will don't continue
 	PreStart func(*http.Response) bool
 
-	Thread int
+	Thread      int
+	Title       string
+	Timeout     int
+	MaxAttempts int
 
 	Debug        bool
 	RoundTripper http.RoundTripper
@@ -121,7 +125,13 @@ func (h *HTTPDownloader) DownloadFile() error {
 		}
 		tr = trp
 	}
-	client := &http.Client{Transport: tr}
+	client := &RetryClient{
+		Client: http.Client{
+			Transport: tr,
+			Timeout:   time.Duration(h.Timeout) * time.Second,
+		},
+		MaxAttempts: h.MaxAttempts,
+	}
 	var resp *http.Response
 
 	if resp, err = client.Do(req); err != nil {
@@ -140,8 +150,11 @@ func (h *HTTPDownloader) DownloadFile() error {
 		return nil
 	}
 
+	if h.Title == "" {
+		h.Title = "Downloading"
+	}
 	writer := &ProgressIndicator{
-		Title: "Downloading",
+		Title: h.Title,
 	}
 	if showProgress {
 		if total, ok := resp.Header["Content-Length"]; ok && len(total) > 0 {
@@ -206,7 +219,7 @@ func DownloadFileWithMultipleThreadKeepParts(targetURL, targetFilePath string, t
 				}
 				start := unit * int64(index)
 
-				if downloadErr := DownloadWithContinue(targetURL, fmt.Sprintf("%s-%d", targetFilePath, index), start, end, showProgress); downloadErr != nil {
+				if downloadErr := DownloadWithContinue(targetURL, fmt.Sprintf("%s-%d", targetFilePath, index), int64(index), start, end, showProgress); downloadErr != nil {
 					fmt.Println(downloadErr)
 				}
 			}(i, &wg)
@@ -238,18 +251,21 @@ func DownloadFileWithMultipleThreadKeepParts(targetURL, targetFilePath string, t
 		}
 	} else {
 		fmt.Println("cannot download it using multiple threads, failed to one")
-		err = DownloadWithContinue(targetURL, targetFilePath, 0, 0, true)
+		err = DownloadWithContinue(targetURL, targetFilePath, -1, 0, 0, true)
 	}
 	return
 }
 
 // DownloadWithContinue downloads the files continuously
-func DownloadWithContinue(targetURL, output string, continueAt, end int64, showProgress bool) (err error) {
+func DownloadWithContinue(targetURL, output string, index, continueAt, end int64, showProgress bool) (err error) {
 	downloader := HTTPDownloader{
 		TargetFilePath: output,
 		URL:            targetURL,
 		ShowProgress:   showProgress,
 	}
+	if index >= 0 {
+		downloader.Title = fmt.Sprintf("Downloading part %d", index)
+	}
 
 	if continueAt >= 0 {
 		downloader.Header = make(map[string]string, 1)

pkg/http_test.go

@@ -65,7 +65,7 @@ var _ = Describe("http test", func() {
 				Body:       ioutil.NopCloser(bytes.NewBufferString(responseBody)),
 			}
 			roundTripper.EXPECT().
-				RoundTrip((request)).Return(response, nil)
+				RoundTrip(request).Return(response, nil)
 			err := downloader.DownloadFile()
 			Expect(err).To(BeNil())
 

pkg/retry_client.go

@@ -0,0 +1,25 @@
+package pkg
+
+import (
+	"net/http"
+	"net/url"
+)
+
+// RetryClient is the wrap of http.Client
+type RetryClient struct {
+	http.Client
+	MaxAttempts     int
+	currentAttempts int
+}
+
+// Do is the wrap of http.Client.Do
+func (c *RetryClient) Do(req *http.Request) (rsp *http.Response, err error) {
+	rsp, err = c.Client.Do(req)
+	if _, ok := err.(*url.Error); ok {
+		if c.currentAttempts < c.MaxAttempts {
+			c.currentAttempts++
+			return c.Do(req)
+		}
+	}
+	return
+}

pkg/retry_client_test.go

@@ -0,0 +1,44 @@
+package pkg
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/golang/mock/gomock"
+	"github.com/linuxsuren/http-downloader/mock/mhttp"
+	"github.com/stretchr/testify/assert"
+	"io/ioutil"
+	"net/http"
+	"reflect"
+	"testing"
+)
+
+const fakeURL = "http://fake"
+
+func TestRetry(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	roundTripper := mhttp.NewMockRoundTripper(ctrl)
+
+	client := &RetryClient{
+		Client: http.Client{
+			Transport: roundTripper,
+		},
+		MaxAttempts: 3,
+	}
+
+	mockRequest, _ := http.NewRequest(http.MethodGet, fakeURL, nil)
+	mockResponse := &http.Response{
+		StatusCode: http.StatusOK,
+		Proto:      "HTTP/1.1",
+		Request:    mockRequest,
+		Body:       ioutil.NopCloser(bytes.NewBufferString("responseBody")),
+	}
+	roundTripper.EXPECT().
+		RoundTrip(mockRequest).Return(mockResponse, nil)
+
+	request, _ := http.NewRequest(http.MethodGet, fakeURL, nil)
+	response, err := client.Do(request)
+	fmt.Println(reflect.TypeOf(err))
+	assert.Nil(t, err)
+	assert.NotNil(t, response)
+	assert.Equal(t, http.StatusOK, response.StatusCode)
+}