Merge branch 'master' into feat/nv21

.github/workflows/baisc_checks.yml

@@ -13,7 +13,7 @@ jobs:
   check:
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up Go
         uses: actions/setup-go@v4

.github/workflows/common_build_upload.yml

@@ -29,7 +29,7 @@ jobs:
   build-ubuntu:
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: 'true'
           fetch-depth: '0'
@@ -220,7 +220,7 @@ jobs:
   build-macos:
     runs-on: macos-11
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: 'true'
           fetch-depth: '0'

.github/workflows/common_docker_image.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-20.04
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Create vars
       id: vars
       run: |

.github/workflows/common_go.yml

@@ -17,7 +17,7 @@ jobs:
   check:
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up Go
         uses: actions/setup-go@v4

.github/workflows/test.yml

@@ -16,7 +16,7 @@ jobs:
   test:
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up Go
         uses: actions/setup-go@v4

CHANGELOG.md

@@ -1,5 +1,47 @@
 # venus changelog
 
+## v1.13.0
+## v1.13.0-rc1
+
+### New Features
+
+* Feat/add sign type to wallet types by @LinZexiao in https://github.com/filecoin-project/venus/pull/6036
+* feat: upgrade builtin actors to v1.23.1 by @0x5459 in https://github.com/filecoin-project/venus/pull/6039
+* feat: Increase the environment variable to skip checking whether mine… by @simlecode in https://github.com/filecoin-project/venus/pull/6055
+* feat: modify query params for message and deal by @LinZexiao in https://github.com/filecoin-project/venus/pull/6066
+* feat: add bootstrap by @simlecode in https://github.com/filecoin-project/venus/pull/6084
+* feat: add proxy interface to gateway by @LinZexiao in https://github.com/filecoin-project/venus/pull/6089
+* feat(market): filter deals by sector lifetime by @0x5459 in https://github.com/filecoin-project/venus/pull/6093
+* feat: set tipset to the given epoch by default by @LinZexiao in https://github.com/filecoin-project/venus/pull/6099
+* feat: remove MinPeerThreshold in bootstrap config because it is not used /删除MinPeerThreshold字段从bootstrap配置，没有用到 by @elvin-du in https://github.com/filecoin-project/venus/pull/6063
+* feat: make execution trace configurable via env variable venus / 通过VENUS_EXEC_TRACE_CACHE环境变量谁知trace缓存大小 by @elvin-du in https://github.com/filecoin-project/venus/pull/6100
+
+### Optimizations
+
+* opt: Adjust size flag to string type by @simlecode in https://github.com/filecoin-project/venus/pull/6102
+
+### Bug Fixes
+
+* fix: Unsubscribe required on exit by @simlecode in https://github.com/filecoin-project/venus/pull/6103
+* fix: ethtypes: handle length overflow case / 处理rlp长度越界问题 by @elvin-du in https://github.com/filecoin-project/venus/pull/6101
+
+### Documentation and Chores
+
+* doc: add config desc / 添加config注释 by @elvin-du in https://github.com/filecoin-project/venus/pull/6062
+* doc: add design doc of sync/添加同步设计文档 by @elvin-du in https://github.com/filecoin-project/venus/pull/5989
+
+* chore: correct god eye url and prioritize ftp by @LinZexiao in https://github.com/filecoin-project/venus/pull/6031
+* chore(deps): bump actions/setup-go from 3 to 4 by @dependabot in https://github.com/filecoin-project/venus/pull/6029
+* chore(deps): bump TheDoctor0/zip-release from 0.6.0 to 0.7.1 by @dependabot in https://github.com/filecoin-project/venus/pull/6028
+* chore: bump github.com/gin-gonic/gin from v1.9.0 to v1.9.1 by @simlecode in https://github.com/filecoin-project/venus/pull/6037
+* chore: update minimum Go version to 1.19 by @0x5459 in https://github.com/filecoin-project/venus/pull/6038
+* chore: Update issue template enhancement.yml by @Fatman13 in https://github.com/filecoin-project/venus/pull/6046
+* chore: fix bug issue template by @Fatman13 in https://github.com/filecoin-project/venus/pull/6047
+* chore: pick 10971,10955,10934 from lotus by @LinZexiao in https://github.com/filecoin-project/venus/pull/6078
+* chore(deps): bump github.com/libp2p/go-libp2p from 0.27.5 to 0.27.8 in /venus-devtool by @dependabot in https://github.com/filecoin-project/venus/pull/6091
+* chore(deps): bump github.com/libp2p/go-libp2p from 0.27.5 to 0.27.8 by @dependabot in https://github.com/filecoin-project/venus/pull/6090
+* chore: transport code by @simlecode in https://github.com/filecoin-project/venus/pull/6097
+
 ## v1.12.0
 
 * fix: compatible with older versions by @simlecode in [[#6024](https://github.com/filecoin-project/venus/pull/6024)]

app/submodule/mpool/mpool_submodule.go

@@ -146,8 +146,6 @@ func (mp *MessagePoolSubmodule) Validate(ctx context.Context, pid peer.ID, msg *
 			fallthrough
 		case errors.Is(err, messagepool.ErrNotEnoughFunds):
 			fallthrough
-		case errors.Is(err, messagepool.ErrGasFeeCapTooLow):
-			fallthrough
 		case errors.Is(err, messagepool.ErrExistingNonce):
 			return pubsub.ValidationIgnore
 

app/submodule/network/network_submodule.go

@@ -81,6 +81,7 @@ type NetworkSubmodule struct { //nolint
 
 	PeerMgr        peermgr.IPeerMgr
 	ExchangeClient filexchange.Client
+	exchangeServer filexchange.Server
 	// data transfer
 	DataTransfer     datatransfer.Manager
 	DataTransferHost dtnet.DataTransferNetwork
@@ -210,6 +211,7 @@ func NewNetworkSubmodule(ctx context.Context,
 	// build network
 	network := net.New(peerHost, rawHost, net.NewRouter(router), bandwidthTracker)
 	exchangeClient := filexchange.NewClient(peerHost, peerMgr)
+	exchangeServer := filexchange.NewServer(chainStore, messageStore, peerHost)
 	helloHandler := helloprotocol.NewHelloProtocolHandler(peerHost, peerMgr, exchangeClient, chainStore, messageStore, config.GenesisCid(), time.Duration(config.Repo().Config().NetworkParams.BlockDelay)*time.Second)
 	// build the network submdule
 	return &NetworkSubmodule{
@@ -221,6 +223,7 @@ func NewNetworkSubmodule(ctx context.Context,
 		Bitswap:          bswap,
 		GraphExchange:    gsync,
 		ExchangeClient:   exchangeClient,
+		exchangeServer:   exchangeServer,
 		Network:          network,
 		DataTransfer:     dt,
 		DataTransferHost: dtNet,
@@ -236,6 +239,9 @@ func (networkSubmodule *NetworkSubmodule) Start(ctx context.Context) error {
 	if !networkSubmodule.cfg.OfflineMode() {
 		go networkSubmodule.PeerMgr.Run(ctx)
 	}
+
+	networkSubmodule.exchangeServer.Register()
+
 	return nil
 }
 

cmd/import.go

@@ -5,10 +5,11 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"net/http"
 	"os"
 	"strings"
 
+	"github.com/filecoin-project/venus/pkg/httpreader"
+
 	"github.com/DataDog/zstd"
 	"github.com/filecoin-project/venus/pkg/chain"
 	"github.com/filecoin-project/venus/pkg/repo"
@@ -30,18 +31,13 @@ func importChain(ctx context.Context, r repo.Repo, fname string) error {
 	var rd io.Reader
 	var l int64
 	if strings.HasPrefix(fname, "http://") || strings.HasPrefix(fname, "https://") {
-		resp, err := http.Get(fname) //nolint:gosec
+		rrd, err := httpreader.NewResumableReader(ctx, fname)
 		if err != nil {
-			return err
-		}
-		defer resp.Body.Close() //nolint:errcheck
-
-		if resp.StatusCode != http.StatusOK {
-			return fmt.Errorf("non-200 response: %d", resp.StatusCode)
+			return fmt.Errorf("fetching chain CAR failed: setting up resumable reader: %w", err)
 		}
 
-		rd = resp.Body
-		l = resp.ContentLength
+		rd = rrd
+		l = rrd.ContentLength()
 	} else {
 		fname, err := homedir.Expand(fname)
 		if err != nil {

documentation/en/venus-market-design-roadmap.md

@@ -6,13 +6,13 @@ Sep 2021
 
 ## Background
 
-As the rebranding of filecoin terminology spearheaded by [FIP0018](https://github.com/filecoin-project/FIPs/blob/master/_fips/fip-0018.md) settled, consensus has been reached across communities (developers, providers, ecosystem partners and etc) to push for taking on more storage deals to improve the public perception on the fact that most of the network storages are still commited capacities (CCs). Given the above sentiment, design and implementation of droplet module has been put into the spot light. A clear long-term roadmap is due for Venus community to dissus and iterate on, also as a means for better communications with filecoin eocsystem in general.
+As the rebranding of filecoin terminology spearheaded by [FIP0018](https://github.com/filecoin-project/FIPs/blob/master/_fips/fip-0018.md) settled, consensus has been reached across communities (developers, providers, ecosystem partners etc) to push for taking on more storage deals to improve the public perception on the fact that most of the network storages are still commited capacities (CCs). Given the above sentiment, design and implementation of droplet module have been put into the spotlight. A clear long-term roadmap is due for Venus community to discuss and iterate on, also as a means for better communications with filecoin ecosystem in general.
 
-While Lotus is leading the way of implementing a full-fledged market module according to the [spec](https://spec.filecoin.io/#section-systems.filecoin_markets), Venus has been making much efforts to catch up and closing the gap in regard to [markets](https://github.com/filecoin-project/venus/discussions/4532). Right now, Lotus supports a 1-to-1 (client-to-provider) storage and retrieval model where burdens of discoveries are mostly on storage clients and match making services like Estuary. Negotiation is a fairly mannual process and does not support much flexibility. As venus-team is picking up the reminiscences of the [Filecoin Component Architecture](https://docs.google.com/document/d/1ukPD8j6plLEbbzUjxfo7eCauIrOeC_tqxqYK_ls9xbc/edit#), emergent ways of how market could facilitate the dynamics between storage providers and storage clients are constatntly being intergrated into the long-term vision of Venus filecoin. 
+While Lotus is leading the way of implementing a full-fledged market module according to the [spec](https://spec.filecoin.io/#section-systems.filecoin_markets), Venus has been making much efforts to catch up and closing the gap in regard to [markets](https://github.com/filecoin-project/venus/discussions/4532). Right now, Lotus supports a 1-to-1 (client-to-provider) storage and retrieval model where burdens of discoveries are mostly on storage clients and match making services like Estuary. Negotiation is a fairly manual process and does not support much flexibility. As venus-team is picking up the reminiscences of the [Filecoin Component Architecture](https://docs.google.com/document/d/1ukPD8j6plLEbbzUjxfo7eCauIrOeC_tqxqYK_ls9xbc/edit#), emergent ways of how market could facilitate the dynamics between storage providers and storage clients are constantly being integrated into the long-term vision of Venus filecoin. 
 
 ## Goals
 
-Current roadmap for droplet are loosely broken into the following phases.
+Current roadmap for droplet is loosely broken into the following phases.
 
 ### Phase 1: peer-to-peer model (short-term)
 
@@ -22,11 +22,11 @@ For phase 1, droplet will deliver a complete deal making experience as what lotu
 
 - Implementation of the one-to-one model of lotus market like module and fully interoperable with lotus implementation, which means compatibility with lotus client and more
 - droplet deployed as independent module, like sophon-cluster and venus-wallet
-- Implementation of a reliable market module that runs a seperate process from the main storage process
+- Implementation of a reliable market module that runs a separate process from the main storage process
 - A clear module boundary that allows interoperability and user customizations
 - Flexibilities of market module to interact with existing venus infrastructures using RPC APIs
 - Supports for mainnet, calibration and Nerpa
-- Lightweight client: compatibility with Lotus and support for droplet unique features including client running seperately as a process and remove dependencies for node; great for bootstraping tests on deal making process
+- Lightweight client: compatibility with Lotus and support for droplet unique features including client running separately as a process and remove dependencies for node; great for bootstrapping tests on deal making process
 
 ### Phase 2: platform model
 
@@ -39,13 +39,13 @@ For phase 2, droplet is taking the following approach.
 ![image-20210910160837732](https://i.loli.net/2021/09/10/sRY5u6Bw9aj713H.png)
 
 - Taking advantages of Venus' distributed architectural nature, a gateway service backend built on top of current infrastructure
-- Compact API: seperation of node and droplet data enabling local storage of some of the deal related meta data
+- Compact API: separation of node and droplet data enabling local storage of some of the deal related meta data
 - Data transfer support for different protocols in addition to `Graphsync` [*](https://docs.google.com/document/d/1XWcTp2MEOVtKLpcpiFeeDvc_gTwQ0Bc6yABCTzDmeP0/edit#heading=h.1oxn84bcd1n1)
 - Meta data stored locally in HA database like mySQL by droplet
 - droplet as deal gateway for storage providers using venus chain services (venus shared modules)
 - Deal match making: multiple copies for store and faster retrieval
 
-### Phase 3: Decentrialized market (Dp2p) model (long-term vision)
+### Phase 3: Decentralized market (Dp2p) model (long-term vision)
 
 For phase 3, droplet will look into ways to automate deal flow between client and provider using a peer-to-peer approach, giving up its role as a gateway in phase 2. Additionally, venus pool can be positioned as a retrieval node which is fully aware of deal meta that chain services helped to record.
 
@@ -64,7 +64,7 @@ Design draws inspirations from the original [filecoin component document](https:
 ### Terminology
 
 - module and component: "A **module** means a code library that may be compiled and linked against. A **component** means a distinct process presenting or consuming RPC API." In this document, the distinction is not as clear. Will need revamping of the Venus documentation to redefine all terms.
-- **GraphSync**: "The default underlying transfer protocol used by the Scheduler. The full graphsync specification can be found at [here](https://github.com/ipld/specs/blob/master/block-layer/graphsync/graphsync.md)."
+- **GraphSync**: "The default underlying transfer protocol used by the Scheduler. The full graphsync specification can be found [here](https://github.com/ipld/specs/blob/master/block-layer/graphsync/graphsync.md)."
 
 ### Modules and processes
 
@@ -74,31 +74,31 @@ Similar to what is described for storage component above, droplet will be depend
 
 ### Deal flow
 
-In phase 1, louts market deal flow will be mirrored in Venus. Maintainace of the market and evolution with the network.
+In phase 1, louts market deal flow will be mirrored in Venus. Maintenance of the market and evolution with the network.
 
 In phase 2, proposing and accepting a deal will work as following.
 
 - [Provider] Add storage ask bidding policy along with other deal acceptance policy
 - [Client] Query asks from droplet with filters like geo locations, redundancy, deal lifespan and etc
 - [droplet] Aggregate requirements from both providers and clients, matchmaking on demand
-- [droplet] Provider(s) and client go through rounds of real-time biding to match-make
+- [droplet] Provider(s) and client go through rounds of real-time bidding to match-make
 - Once matched, provider proceeds to store data as in the one-to-one model
 
 ### Meta data
 
-Platform model implementation of droplet may store metadata on the deals it distributes to providers under its wings. Like Airbnb, it may include metrics that a repututation system of both client and provider can be built upon. Metrics like storage success rate, retrieval success rate, fast retrieval enabled? and etc.
+Platform model implementation of droplet may store metadata on the deals it distributes to providers under its wings. Like Airbnb, it may include metrics that a reputation system of both client and provider can be built upon. Metrics like storage success rate, retrieval success rate, fast retrieval enabled? and etc.
 
 ### Dependencies
 
 1. `venus` module to provide node services
 2. `sophon-messager` module to provide data services
 3. `sophon-gateway` to provide signature services
 4. `sophon-cluster` to provide sealing and data lookup services
-5. `go-fil-market` compatible with lotus one-to-one model (For compatbility with lotus only)
+5. `go-fil-market` compatible with lotus one-to-one model (For compatibility with lotus only)
 6. piece data from external deals
 7. datastore using HA databases for deal meta data
 
-*Note that `go-fil-market` included as dependencies is sololy for the use of compatibility with lotus. droplet will be bundling other unique features along with compatibility with lotus.*
+*Note that `go-fil-market` included as dependencies is solely for the use of compatibility with lotus. droplet will be bundling other unique features along with compatibility with lotus.*
 
 ![模块图](https://i.loli.net/2021/09/08/7UxfVujcNPmszyR.jpg)
 

fixtures/networks/mainnet.go

@@ -16,9 +16,6 @@ func Mainnet() *NetworkConf {
 				"/dns4/bootstrap-0.mainnet.filops.net/tcp/1347/p2p/12D3KooWCVe8MmsEMes2FzgTpt9fXtmCY7wrq91GRiaC8PHSCCBj",
 				"/dns4/bootstrap-1.mainnet.filops.net/tcp/1347/p2p/12D3KooWCwevHg1yLCvktf2nvLu7L9894mcrJR4MsBCcm4syShVc",
 				"/dns4/bootstrap-2.mainnet.filops.net/tcp/1347/p2p/12D3KooWEWVwHGn2yR36gKLozmb4YjDJGerotAPGxmdWZx2nxMC4",
-				"/dns4/bootstrap-3.mainnet.filops.net/tcp/1347/p2p/12D3KooWKhgq8c7NQ9iGjbyK7v7phXvG6492HQfiDaGHLHLQjk7R",
-				"/dns4/bootstrap-4.mainnet.filops.net/tcp/1347/p2p/12D3KooWL6PsFNPhYftrJzGgF5U18hFoaVhfGk7xwzD8yVrHJ3Uc",
-				"/dns4/bootstrap-5.mainnet.filops.net/tcp/1347/p2p/12D3KooWLFynvDQiUpXoHroV1YxKHhPJgysQGH2k3ZGwtWzR4dFH",
 				"/dns4/bootstrap-6.mainnet.filops.net/tcp/1347/p2p/12D3KooWP5MwCiqdMETF9ub1P3MbCvQCcfconnYHbWg6sUJcDRQQ",
 				"/dns4/bootstrap-7.mainnet.filops.net/tcp/1347/p2p/12D3KooWRs3aY1p3juFjPy8gPN95PEQChm2QKGUCAdcDCC4EBMKf",
 				"/dns4/bootstrap-8.mainnet.filops.net/tcp/1347/p2p/12D3KooWScFR7385LTyR4zU1bYdzSiiAb5rnNABfVahPvVSzyTkR",
@@ -29,6 +26,7 @@ func Mainnet() *NetworkConf {
 				"/dns4/bootstrap-0.ipfsmain.cn/tcp/34721/p2p/12D3KooWQnwEGNqcM2nAcPtRR9rAX8Hrg4k9kJLCHoTR5chJfz6d",
 				"/dns4/bootstrap-1.ipfsmain.cn/tcp/34723/p2p/12D3KooWMKxMkD5DMpSWsW7dBddKxKT7L2GgbNuckz9otxvkvByP",
 				"/dns4/bootstarp-0.1475.io/tcp/61256/p2p/12D3KooWRzCVDwHUkgdK7eRgnoXbjDAELhxPErjHzbRLguSV1aRt",
+				"/dns4/bootstrap-venus.mainnet.filincubator.com/tcp/8888/p2p/QmQu8C6deXwKvJP2D8B6QGyhngc3ZiDnFzEHBDx8yeBXST",
 			},
 			Period: "30s",
 		},