init resp

README.md

@@ -0,0 +1,12 @@
+## 简介
+
+> 最近挺喜欢听IU的blueming，所以命名为blueming
+> 主要用于获取网站备份文件
+
+## TODO
+
+> 基本满足以下要求即可
+
+- [x] 通过URL自动生成文件名
+- [x] 根据后缀名将URL定义为对应的文件格式，如zip、tar.gz等
+- [x] 自动下载备份文件，并进行重命名

cmd/blueming/main.go

@@ -0,0 +1,51 @@
+package main
+
+import (
+	"flag"
+	"github.com/bufsnake/blueming/internal/core"
+	"github.com/bufsnake/blueming/pkg/log"
+	"io/ioutil"
+	"os"
+	"strings"
+)
+
+func main() {
+	err := os.Remove("output/download_error")
+	if err != nil {
+		log.Warn(err)
+	}
+	thread := flag.Int("t", 10, "set blueming thread")
+	timeout := flag.Int("s", 10, "set blueming timeout")
+	url := flag.String("u", "", "set blueming url")
+	urlfile := flag.String("f", "", "set blueming url file")
+	loglevel := flag.String("l", log.DEBUG, "set blueming log level(trace debug info warn fatal)")
+	flag.Parse()
+	log.SetLevel(*loglevel)
+	urls := []string{}
+	if *url != "" {
+		urls = append(urls, *url)
+	} else if *urlfile != "" {
+		file, err := ioutil.ReadFile(*urlfile)
+		if err != nil {
+			log.Warn(err)
+		}
+		split := strings.Split(string(file), "\n")
+		for i := 0; i < len(split); i++ {
+			split[i] = strings.Trim(split[i], "\r")
+			urls = append(urls, split[i])
+		}
+	} else {
+		flag.Usage()
+		return
+	}
+	log.Info(len(urls), "个URL,", *thread, "线程,", *timeout, "超时")
+	create, err := os.Create("output/download_error")
+	if err != nil {
+		log.Warn(err)
+	}
+	err = create.Close()
+	if err != nil {
+		log.Warn(err)
+	}
+	core.NewCore(urls, *thread, *timeout)
+}

internal/core/core.go

@@ -0,0 +1,86 @@
+package core
+
+import (
+	file_download "github.com/bufsnake/blueming/pkg/file-download"
+	general_file_name "github.com/bufsnake/blueming/pkg/general-file-name"
+	http_request "github.com/bufsnake/blueming/pkg/http-request"
+	"github.com/bufsnake/blueming/pkg/log"
+	"io/ioutil"
+	"sync"
+)
+
+func NewCore(url []string, thread, timeout int) {
+	requestlist := make([][]string, 0)
+	for i := 0; i < len(url); i++ {
+		genURL, err := general_file_name.NewGenURL(url[i])
+		if err != nil {
+			log.Warn(err)
+			continue
+		}
+		getURL := genURL.GetURL()
+		requestlist = append(requestlist, *getURL)
+	}
+	httpr := sync.WaitGroup{}
+	httpc := make(chan string, thread)
+	httpd := make(chan string, thread)
+	for i := 0; i < thread; i++ {
+		httpr.Add(1)
+		go httprequest(&httpr, httpc, httpd, timeout)
+	}
+	down := sync.WaitGroup{}
+	for i := 0; i < thread; i++ {
+		down.Add(1)
+		go httpdownload(&down, httpd)
+	}
+	i := 0
+	for {
+		flag := 0
+		for j := 0; j < len(requestlist); j++ {
+			if i >= len(requestlist[j]) {
+				flag++
+				continue
+			}
+			httpc <- requestlist[j][i]
+		}
+		i++
+		if flag == len(requestlist) {
+			break
+		}
+	}
+
+	close(httpc)
+	httpr.Wait()
+	close(httpd)
+	down.Wait()
+}
+
+func httprequest(wait *sync.WaitGroup, httpc, httpd chan string, timeout int) {
+	defer wait.Done()
+	for url := range httpc {
+		log.Trace(url)
+		status, size, err := http_request.HTTPRequest(url, timeout)
+		if err != nil {
+			log.Warn(err)
+			continue
+		}
+		if size != "0B" && size != "0.0B" && status == 200 {
+			log.Info(size, url)
+			httpd <- url
+		}
+	}
+}
+
+func httpdownload(wait *sync.WaitGroup, httpd chan string) {
+	defer wait.Done()
+	for url := range httpd {
+		err := file_download.DownloadFile(url)
+		if err != nil {
+			log.Info("file download error", err)
+			// 将URL保存到文件
+			err := ioutil.WriteFile("output/download_error", []byte(url+"\n"), 644)
+			if err != nil {
+				log.Warn(err)
+			}
+		}
+	}
+}

pkg/file-download/main.go

@@ -0,0 +1,55 @@
+package file_download
+
+import (
+	"crypto/tls"
+	"github.com/bufsnake/blueming/pkg/useragent"
+	"io"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+)
+
+func DownloadFile(url string) error {
+	client := &http.Client{
+		Timeout: 1 * time.Hour,
+		Transport: &http.Transport{
+			DisableKeepAlives: true,
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: true,
+			},
+		},
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+	}
+	req, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		return err
+	}
+	req.Header.Add("Accept", "*/*")
+	req.Header.Add("Referer", "http://www.baidu.com")
+	req.Header.Add("Connection", "close")
+	req.Header.Add("Cache-Control", "no-cache")
+	req.Header.Add("User-Agent", useragent.RandomUserAgent())
+	do, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer do.Body.Close()
+	temp_file := strings.ReplaceAll(url, ":", ".")
+	temp_file = strings.ReplaceAll(temp_file, "/", ".")
+	temp_file = strings.ReplaceAll(temp_file, "..", ".")
+	temp_file = strings.ReplaceAll(temp_file, "..", ".")
+	temp_file = strings.ReplaceAll(temp_file, "..", ".")
+	out, err := os.Create("output/" + temp_file)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+	_, err = io.Copy(out, do.Body)
+	if err != nil {
+		return err
+	}
+	return nil
+}

pkg/general-file-name/main.go

@@ -0,0 +1,49 @@
+package general_file_name
+
+import (
+	"net/url"
+	"regexp"
+	"strings"
+)
+
+type general_file_name struct {
+	url string
+}
+
+func NewGenURL(url string) (*general_file_name, error) {
+	return &general_file_name{url: strings.TrimRight(url, "/")}, nil
+}
+
+func (g *general_file_name) GetURL() *[]string {
+	ret := make([]string, 0)
+	prefix := []string{"www", "admin", "wwwroot", "web", "data", "backup", "db", "database", "code", "test", "user", "sql"}
+	suffix := []string{".zip", ".rar", ".tar.gz", ".tgz", ".tar.bz2", ".tar", ".jar", ".war", ".7z", ".bak", ".sql"}
+	parse, err := url.Parse(g.url)
+	if err == nil {
+		if strings.Contains(parse.Host, ":") {
+			parse.Host = strings.Split(parse.Host, ":")[0]
+		}
+		parse.Host = strings.TrimLeft(parse.Host, "www.")
+		prefix = append(prefix, parse.Host)
+		if isdomain(parse.Host) {
+			split := strings.Split(parse.Host, ".")
+			if len(split) > 2 {
+				prefix = append(prefix, split[len(split)-2]+"."+split[len(split)-1])
+				prefix = append(prefix, split[len(split)-2])
+			}
+		}
+	}
+	for i := 0; i < len(prefix); i++ {
+		for j := 0; j < len(suffix); j++ {
+			ret = append(ret, g.url+"/"+prefix[i]+suffix[j])
+		}
+	}
+	return &ret
+}
+
+func isdomain(str string) bool {
+	if matched, _ := regexp.MatchString("\\d{0,3}\\.\\d{0,3}\\.\\d{0,3}\\.\\d{0,3}", str); matched {
+		return false
+	}
+	return true
+}

pkg/http-request/main.go

@@ -0,0 +1,65 @@
+package http_request
+
+import (
+	"crypto/tls"
+	"fmt"
+	"github.com/bufsnake/blueming/pkg/useragent"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"time"
+)
+
+func HTTPRequest(url string, timeout int) (status int, size string, err error) {
+	client := &http.Client{
+		Timeout: time.Duration(timeout) * time.Second,
+		Transport: &http.Transport{
+			DisableKeepAlives: true,
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: true,
+			},
+		},
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+	}
+	req, err := http.NewRequest(http.MethodHead, url, nil)
+	if err != nil {
+		return 0, "0B", err
+	}
+	req.Header.Add("Accept", "*/*")
+	req.Header.Add("Referer", "http://www.baidu.com")
+	req.Header.Add("Connection", "close")
+	req.Header.Add("Cache-Control", "no-cache")
+	req.Header.Add("User-Agent", useragent.RandomUserAgent())
+	do, err := client.Do(req)
+	if err != nil {
+		return 0, "0B", err
+	}
+	defer do.Body.Close()
+	_, err = io.Copy(ioutil.Discard, do.Body)
+	if err != nil {
+		return 0, "0B", err
+	}
+	if !strings.Contains(do.Header.Get("Content-Type"), "application/octet-stream") {
+		return 0, "0B", err
+	}
+	temp := float64(do.ContentLength)
+	SIZE := []string{"B", "K", "M", "G", "T"}
+	i := 0
+	for {
+		if temp < 1024 {
+			break
+		}
+		temp = temp / 1024.0
+		i++
+	}
+	length := ""
+	if i > len(SIZE) {
+		length = fmt.Sprintf("%0.2fX", temp)
+	} else {
+		length = fmt.Sprintf("%0.1f%s", temp, SIZE[i])
+	}
+	return do.StatusCode, length, nil
+}