feat: support use server time for file creation

.github/workflows/build.yml

@@ -53,6 +53,7 @@ jobs:
 
           go build -tags nosqlite -ldflags="-w -s -X github.com/GopeedLab/gopeed/pkg/base.Version=$VERSION" -buildmode=c-shared -o ui/flutter/windows/libgopeed.dll github.com/GopeedLab/gopeed/bind/desktop
           cd ui/flutter
+          $TAG = "v$env:VERSION"
           flutter build windows
           $mingw = "C:\Program Files\Git\mingw64\bin"
           $system = "C:\Windows\System32"
@@ -65,7 +66,7 @@ jobs:
           cp $system\vcruntime140_1.dll $release
 
           New-Item -Path build\windows\Output -ItemType Directory
-          Compress-Archive -Path "$release*" -DestinationPath "build\windows\Output\Gopeed-$env:TAG-windows-amd64-portable.zip"
+          Compress-Archive -Path "$release*" -DestinationPath "build\windows\Output\Gopeed-$TAG-windows-amd64-portable.zip"
 
           cd build/windows
           echo @"
@@ -124,7 +125,6 @@ jobs:
           Filename: "{app}\{#MyAppExeName}"; Description: "{cm:LaunchProgram,{#StringChange(MyAppName, '&', '&&')}}"; Flags: nowait postinstall skipifsilent
           "@ > setup.iss
           iscc.exe setup.iss
-          $TAG = "v$env:VERSION"
           mv "Output\gopeed.exe" "Output\Gopeed-$TAG-windows-amd64.exe"
           Compress-Archive -Path "Output\Gopeed-$TAG-windows-amd64.exe" -DestinationPath "Output\Gopeed-$TAG-windows-amd64.zip"
           Remove-Item -Path "Output\Gopeed-$TAG-windows-amd64.exe"

go.mod

@@ -96,6 +96,7 @@ require (
 	github.com/stretchr/testify v1.8.4 // indirect
 	github.com/tidwall/btree v1.7.0 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
+	github.com/xiaoqidun/setft v0.0.0-20220310121541-be86327699ad // indirect
 	go.opentelemetry.io/otel v1.22.0 // indirect
 	go.opentelemetry.io/otel/metric v1.22.0 // indirect
 	go.opentelemetry.io/otel/trace v1.22.0 // indirect

go.sum

@@ -425,6 +425,8 @@ github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPyS
 github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
 github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
 github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
+github.com/xiaoqidun/setft v0.0.0-20220310121541-be86327699ad h1:QtuWk6dsrNXc/ugwCBEN0jG52q/4tXRzMvZm4fH6T9g=
+github.com/xiaoqidun/setft v0.0.0-20220310121541-be86327699ad/go.mod h1:Jj8p9bgKGTPQ+M8CdUMS9p7Mmdoxa3OAcAjJQBu0CcI=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=

internal/protocol/http/config.go

@@ -1,6 +1,7 @@
 package http
 
 type config struct {
-	UserAgent   string `json:"userAgent"`
-	Connections int    `json:"connections"`
+	UserAgent      string `json:"userAgent"`
+	Connections    int    `json:"connections"`
+	UseServerCtime bool   `json:"useServerCtime"`
 }

internal/protocol/http/fetcher.go

@@ -9,6 +9,7 @@ import (
 	"github.com/GopeedLab/gopeed/internal/fetcher"
 	"github.com/GopeedLab/gopeed/pkg/base"
 	fhttp "github.com/GopeedLab/gopeed/pkg/protocol/http"
+	"github.com/xiaoqidun/setft"
 	"golang.org/x/sync/errgroup"
 	"io"
 	"mime"
@@ -131,8 +132,17 @@ func (f *Fetcher) Resolve(req *base.Request) error {
 	} else {
 		return NewRequestError(httpResp.StatusCode, httpResp.Status)
 	}
+	// Parse last modified time
+	var lastModifiedTime *time.Time
+	lastModified := httpResp.Header.Get(base.HttpHeaderLastModified)
+	if lastModified != "" {
+		// ignore parse error
+		t, _ := time.Parse(time.RFC1123, lastModified)
+		lastModifiedTime = &t
+	}
 	file := &base.FileInfo{
-		Size: res.Size,
+		Size:  res.Size,
+		Ctime: lastModifiedTime,
 	}
 	contentDisposition := httpResp.Header.Get(base.HttpHeaderContentDisposition)
 	if contentDisposition != "" {
@@ -258,6 +268,10 @@ func (f *Fetcher) fetch() {
 			return
 		}
 		f.file.Close()
+		// Update file last modified time
+		if f.config.UseServerCtime && f.meta.Res.Files[0].Ctime != nil {
+			setft.SetFileTime(f.file.Name(), time.Now(), *f.meta.Res.Files[0].Ctime, *f.meta.Res.Files[0].Ctime)
+		}
 		f.doneCh <- err
 	}()
 }
@@ -276,7 +290,7 @@ func (f *Fetcher) fetchChunk(index int, ctx context.Context) (err error) {
 		maxRetries = 3
 	)
 	// retry until all remain chunks failed
-	for true {
+	for {
 		// if chunk is completed, return
 		if f.meta.Res.Range && chunk.Downloaded >= chunk.End-chunk.Begin+1 {
 			return

internal/protocol/http/fetcher_test.go

@@ -151,10 +151,34 @@ func TestFetcher_DownloadWithProxy(t *testing.T) {
 	downloadWithProxy(httpListener, proxyListener, t)
 }
 
-func TestFetcher_Config(t *testing.T) {
+func TestFetcher_ConfigConnections(t *testing.T) {
 	listener := test.StartTestFileServer()
 	defer listener.Close()
-	fetcher := doDownloadReady(buildConfigFetcher(), listener, 0, t)
+	fetcher := doDownloadReady(buildConfigFetcher(config{
+		Connections: 16,
+	}), listener, 0, t)
+	err := fetcher.Start()
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = fetcher.Wait()
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := test.FileMd5(test.BuildFile)
+	got := test.FileMd5(test.DownloadFile)
+	if want != got {
+		t.Errorf("Download() got = %v, want %v", got, want)
+	}
+}
+
+func TestFetcher_ConfigUseServerCtime(t *testing.T) {
+	listener := test.StartTestFileServer()
+	defer listener.Close()
+	fetcher := doDownloadReady(buildConfigFetcher(config{
+		Connections:    16,
+		UseServerCtime: true,
+	}), listener, 0, t)
 	err := fetcher.Start()
 	if err != nil {
 		t.Fatal(err)
@@ -338,14 +362,11 @@ func buildFetcher() *Fetcher {
 	return fetcher.(*Fetcher)
 }
 
-func buildConfigFetcher() fetcher.Fetcher {
+func buildConfigFetcher(cfg config) fetcher.Fetcher {
 	fetcher := new(FetcherBuilder).Build()
 	newController := controller.NewController()
-	mockCfg := config{
-		Connections: 16,
-	}
 	newController.GetConfig = func(v any) bool {
-		if err := json.Unmarshal([]byte(test.ToJson(mockCfg)), v); err != nil {
+		if err := json.Unmarshal([]byte(test.ToJson(cfg)), v); err != nil {
 			return false
 		}
 		return true

pkg/base/constants.go

@@ -20,7 +20,8 @@ const (
 	HttpHeaderContentLength      = "Content-Length"
 	HttpHeaderContentRange       = "Content-Range"
 	HttpHeaderContentDisposition = "Content-Disposition"
-	HttpHeaderUserAgent          = "Usr-Agent"
+	HttpHeaderUserAgent          = "User-Agent"
+	HttpHeaderLastModified       = "Last-Modified"
 
 	HttpHeaderBytes       = "bytes"
 	HttpHeaderRangeFormat = "bytes=%d-%d"

pkg/base/model.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"github.com/GopeedLab/gopeed/pkg/util"
 	"golang.org/x/exp/slices"
+	"time"
 )
 
 // Request download request
@@ -59,9 +60,10 @@ func (r *Resource) CalcSize(selectFiles []int) {
 }
 
 type FileInfo struct {
-	Name string `json:"name"`
-	Path string `json:"path"`
-	Size int64  `json:"size"`
+	Name  string     `json:"name"`
+	Path  string     `json:"path"`
+	Size  int64      `json:"size"`
+	Ctime *time.Time `json:"ctime"`
 
 	Req *Request `json:"req"`
 }

pkg/download/downloader.go

@@ -124,6 +124,11 @@ func (d *Downloader) Setup() error {
 	}
 	// init default config
 	d.cfg.DownloaderStoreConfig.Init()
+	for _, fb := range d.fetcherBuilders {
+		f := fb.Build()
+		d.setupFetcher(f)
+
+	}
 	// load tasks from storage
 	var tasks []*Task
 	if err = d.storage.List(bucketTask, &tasks); err != nil {

ui/flutter/lib/api/model/downloader_config.dart

@@ -4,13 +4,16 @@ part 'downloader_config.g.dart';
 
 @JsonSerializable(explicitToJson: true)
 class DownloaderConfig {
-  String downloadDir = '';
-  int maxRunning = 0;
+  String downloadDir;
+  int maxRunning;
   ProtocolConfig protocolConfig = ProtocolConfig();
   ExtraConfig extra = ExtraConfig();
   ProxyConfig proxy = ProxyConfig();
 
-  DownloaderConfig();
+  DownloaderConfig({
+    this.downloadDir = '',
+    this.maxRunning = 0,
+  });
 
   factory DownloaderConfig.fromJson(Map<String, dynamic> json) =>
       _$DownloaderConfigFromJson(json);
@@ -31,11 +34,16 @@ class ProtocolConfig {
 
 @JsonSerializable()
 class HttpConfig {
-  String userAgent =
-      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36';
-  int connections = 0;
+  String userAgent;
+  int connections;
+  bool useServerCtime;
 
-  HttpConfig();
+  HttpConfig({
+    this.userAgent =
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36',
+    this.connections = 16,
+    this.useServerCtime = false,
+  });
 
   factory HttpConfig.fromJson(Map<String, dynamic> json) =>
       _$HttpConfigFromJson(json);
@@ -69,13 +77,19 @@ class ExtraConfig {
 
 @JsonSerializable()
 class ProxyConfig {
-  bool enable = false;
-  String scheme = '';
-  String host = '';
-  String usr = '';
-  String pwd = '';
-
-  ProxyConfig();
+  bool enable;
+  String scheme;
+  String host;
+  String usr;
+  String pwd;
+
+  ProxyConfig({
+    this.enable = false,
+    this.scheme = '',
+    this.host = '',
+    this.usr = '',
+    this.pwd = '',
+  });
 
   factory ProxyConfig.fromJson(Map<String, dynamic> json) =>
       _$ProxyConfigFromJson(json);
@@ -85,13 +99,20 @@ class ProxyConfig {
 
 @JsonSerializable()
 class ExtraConfigBt {
-  List<String> trackerSubscribeUrls = [];
-  List<String> subscribeTrackers = [];
+  List<String> trackerSubscribeUrls;
+  List<String> subscribeTrackers;
+  bool autoUpdateTrackers;
   DateTime? lastTrackerUpdateTime;
 
-  List<String> customTrackers = [];
+  List<String> customTrackers;
 
-  ExtraConfigBt();
+  ExtraConfigBt({
+    this.trackerSubscribeUrls = const [],
+    this.subscribeTrackers = const [],
+    this.autoUpdateTrackers = true,
+    this.lastTrackerUpdateTime,
+    this.customTrackers = const [],
+  });
 
   factory ExtraConfigBt.fromJson(Map<String, dynamic> json) =>
       _$ExtraConfigBtFromJson(json);