refactor： refractor ci and test/build (#4831)

* refractor ci and test/build

* rename job

* apt update before install

* fix imports

* fix gen

* fix goimports

* fix lint

* fix doc gen

* fix inline-gen

.codecov.yml

@@ -26,7 +26,3 @@ codecov:
     # yes: will delay sending notifications until all ci is finished
     # no: will send notifications without checking ci status and wait till "after_n_builds" are uploaded
     require_ci_to_pass: false
-    # number of expected builds to receive before sending notifications
-    # we can set this to prevent notifications of partial results due to CI parallelism
-    # set this with respect to the sum of test parallelism in Circle CI configuration.
-    after_n_builds: 6

.github/workflows/baisc_checks.yml

@@ -1,41 +1,59 @@
-name: basic checks
+name: basic-check
 
-on: ["push", "pull_request"]
+on:
+  push:
+    branches:
+      - master
+      - main
+  pull_request:
+    branches:
+      - '**'
 
 jobs:
+
   check:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-18.04
     steps:
-      - name: dependencies
-        run: sudo apt update & sudo apt upgrade & sudo apt install --reinstall mesa-opencl-icd ocl-icd-opencl-dev gcc git bzr jq pkg-config curl clang build-essential hwloc libhwloc-dev wget -y
-
       - uses: actions/checkout@v2
-        with:
-          submodules: 'true'
 
-      - name: setup go
+      - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: '1.16'
+          go-version: 1.17
+
+      - name: install deps
+        run: |
+          sudo apt-get update
+          sudo apt-get -o Acquire::Retries=3 install make ncftp mesa-opencl-icd ocl-icd-opencl-dev gcc git bzr jq pkg-config curl clang build-essential hwloc libhwloc-dev wget -y && sudo apt upgrade -y
 
-      - name: deps
-        run: make deps
+      - name: dep
+        env:
+          GOPROXY: "https://proxy.golang.org,direct"
+          GO111MODULE: "on"
+        run: |
+          make deps
 
-      - name: build
-        run: make all
+      - name: Build
+        env:
+          GOPROXY: "https://proxy.golang.org,direct"
+          GO111MODULE: "on"
+        run: |
+          make
 
-      - name: tests-shared
-        run: make test-venus-shared
+      - name: Lint
+        run: |
+          curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.44.2
+          golangci-lint run --timeout 10m
 
-      - name: compatible all
+      - name: Compatible all
         run: |
           make compatible-all
 
-      - name: gen all
+      - name: Gen all
         run: |
           make gen-all
 
-      - name: detect changes
+      - name: Detect changes
         run: |
           git status --porcelain
-          test -z "$(git status --porcelain)"
+          test -z "$(git status --porcelain)"

.github/workflows/tag-workflow.yml

@@ -87,15 +87,11 @@ jobs:
         uses: actions/setup-go@v2
         with:
           go-version: 1.17
-      - name: Install GCC
-        uses: egor-tensin/setup-gcc@v1
-        with:
-          version: latest
-          platform: x64
-      - name: Setup Rust
-        run: (sudo apt-get update && sudo apt-get install -y clang libssl-dev && which cargo && which rustc) || (curl https://sh.rustup.rs -sSf | sh -s -- -y)
       - name: install deps
-        run: sudo apt-get install make ncftp mesa-opencl-icd ocl-icd-opencl-dev gcc git bzr jq pkg-config curl clang build-essential hwloc libhwloc-dev wget -y && sudo apt upgrade -y
+        run: |
+          sudo apt-get update
+          sudo apt-get -o Acquire::Retries=3 install make ncftp mesa-opencl-icd ocl-icd-opencl-dev gcc git bzr jq pkg-config curl clang build-essential hwloc libhwloc-dev wget -y && sudo apt upgrade -y
+
       - name: Build
         run: |
           go clean --modcache && make deps && make

.github/workflows/test.yml

@@ -0,0 +1,76 @@
+name: Test
+
+on:
+  push:
+    branches:
+      - master
+      - main
+  pull_request:
+    branches:
+      - '**'
+
+jobs:
+
+  test:
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.17
+
+      - name: install deps
+        run: |
+          sudo apt-get update
+          sudo apt-get -o Acquire::Retries=3 install make ncftp mesa-opencl-icd ocl-icd-opencl-dev gcc git bzr jq pkg-config curl clang build-essential hwloc libhwloc-dev wget -y && sudo apt upgrade -y
+
+      - name: install statediff globally
+        run: |
+          ## statediff is optional; we succeed even if compilation fails.
+          mkdir -p /tmp/statediff
+          git clone https://github.com/filecoin-project/statediff.git /tmp/statediff
+          cd /tmp/statediff
+          go install ./cmd/statediff || exit 0
+
+      - name: dep
+        env:
+          GOPROXY: "https://proxy.golang.org,direct"
+          GO111MODULE: "on"
+        run: |
+          make deps
+
+      - name: Build
+        env:
+          GOPROXY: "https://proxy.golang.org,direct"
+          GO111MODULE: "on"
+        run: |
+          make
+
+      - name: Download vk file
+        run: |
+          ./venus fetch --size=0
+
+      - name: GenTool
+        run: |
+          go build -o genesis-file-server ./tools/genesis-file-server
+          go build -o gengen ./tools/gengen
+          ./gengen --keypath ./fixtures/live --out-car ./fixtures/live/genesis.car --out-json  ./fixtures/live/gen.json --config ./fixtures/setup.json
+          ./gengen --keypath ./fixtures/test --out-car ./fixtures/test/genesis.car --out-json  ./fixtures/test/gen.json --config ./fixtures/setup.json
+
+      - name: Unit Test
+        run: go test -coverpkg=./... -coverprofile=coverage_unit.txt -covermode=atomic -timeout=30m -parallel=4 -v ./... -integration=false -unit=true
+
+      - name: Integration Test
+        run: go test -coverpkg=./... -coverprofile=coverage_integration.txt -covermode=atomic -timeout=30m -parallel=4  -v ./... -integration=true -unit=false
+
+      - name: Upload
+        uses: codecov/codecov-action@v2
+        with:
+          token:
+          files: ./coverage_unit.txt,./coverage_integration.txt
+          flags: unittests,integration
+          name: venus
+          fail_ci_if_error: true
+          verbose: true

.gitignore

@@ -42,5 +42,8 @@ debug.test
 
 // HomeBrew
 Brewfile.lock.json
-
+build-dep
+coverage.*
 receipt.json
+genesis-file-server
+gengen

.golangci.yml

@@ -5,7 +5,7 @@ linters:
     - govet
     - misspell
     - goconst
-    - golint
+    - revive
     - errcheck
     - unconvert
     - staticcheck
@@ -15,6 +15,7 @@ linters:
     - unused
     - stylecheck
     - gosimple
+    - goimports
 
 issues:
   exclude:

Makefile

@@ -1,28 +1,46 @@
 export CGO_CFLAGS_ALLOW=-D__BLST_PORTABLE__
 export CGO_CFLAGS=-D__BLST_PORTABLE__
 
-all:
-	go run ./build/*.go build
+all: build
+.PHONY: all
 
-deps:
-	git submodule update --init
-	go run ./build/*.go smartdeps
+## variables
 
-lint:
-	go run ./build/*.go lint
+# git modules that need to be loaded
+MODULES:=
 
-test: test-venus-shared
-	go run ./build/*.go test -timeout=30m
+ldflags=-X=github.com/filecoin-project/venus/pkg/constants.CurrentCommit=+git.$(subst -,.,$(shell git describe --always --match=NeVeRmAtCh --dirty 2>/dev/null || git rev-parse --short HEAD 2>/dev/null))
+ifneq ($(strip $(LDFLAGS)),)
+	    ldflags+=-extldflags=$(LDFLAGS)
+	endif
 
-# WARNING THIS BUILDS A GO PLUGIN AND PLUGINS *DO NOT* WORK ON WINDOWS SYSTEMS
-iptb:
-	make -C tools/iptb-plugins all
+GOFLAGS+=-ldflags="$(ldflags)"
 
-clean:
-	rm ./venus
+## FFI
 
-	rm -rf ./extern/filecoin-ffi
-	rm -rf ./extern/test-vectors
+FFI_PATH:=extern/filecoin-ffi/
+FFI_DEPS:=.install-filcrypto
+FFI_DEPS:=$(addprefix $(FFI_PATH),$(FFI_DEPS))
+
+$(FFI_DEPS): build-dep/.filecoin-install ;
+
+build-dep/.filecoin-install: $(FFI_PATH)
+	$(MAKE) -C $(FFI_PATH) $(FFI_DEPS:$(FFI_PATH)%=%)
+	@touch $@
+
+MODULES+=$(FFI_PATH)
+BUILD_DEPS+=build-dep/.filecoin-install
+CLEAN+=build-dep/.filecoin-install
+
+## modules
+build-dep:
+	mkdir $@
+
+$(MODULES): build-dep/.update-modules;
+# dummy file that marks the last time modules were updated
+build-dep/.update-modules: build-dep;
+	git submodule update --init --recursive
+	touch $@
 
 gen-all: cborgen gogen inline-gen api-gen
 
@@ -75,3 +93,23 @@ actor-render:
 
 actor-replica:
 	cd venus-devtool && go run ./compatible/actors/*.go replica --dst ../venus-shared/actors/
+
+test:
+	go build -o genesis-file-server ./tools/genesis-file-server
+	go build -o gengen ./tools/gengen
+	./gengen --keypath ./fixtures/live --out-car ./fixtures/live/genesis.car --out-json  ./fixtures/live/gen.json --config ./fixtures/setup.json
+	./gengen --keypath ./fixtures/test --out-car ./fixtures/test/genesis.car --out-json  ./fixtures/test/gen.json --config ./fixtures/setup.json
+	 go test  -v ./... -integration=true -unit=false
+
+lint: $(BUILD_DEPS)
+	staticcheck ./...
+
+deps: $(BUILD_DEPS)
+
+dist-clean:
+	git clean -xdff
+	git submodule deinit --all -f
+
+build: $(BUILD_DEPS)
+	rm -f venus
+	go build -o ./venus $(GOFLAGS) .

app/node/builder_getter.go

@@ -1,13 +1,14 @@
 package node
 
 import (
+	"time"
+
 	"github.com/filecoin-project/venus/pkg/clock"
 	"github.com/filecoin-project/venus/pkg/journal"
 	"github.com/filecoin-project/venus/pkg/repo"
 	"github.com/filecoin-project/venus/pkg/util/ffiwrapper"
 	"github.com/ipfs/go-cid"
 	"github.com/libp2p/go-libp2p"
-	"time"
 )
 
 // Builder private method accessors for impl's

app/node/inspector_api.go

@@ -1,13 +1,13 @@
 package node
 
 import (
-	"fmt"
-	"github.com/filecoin-project/venus/build/flags"
+	"os"
+	"runtime"
+
 	"github.com/filecoin-project/venus/pkg/config"
+	"github.com/filecoin-project/venus/pkg/constants"
 	"github.com/filecoin-project/venus/pkg/repo"
 	sysi "github.com/whyrusleeping/go-sysinfo"
-	"os"
-	"runtime"
 )
 
 type IInspector interface {
@@ -146,5 +146,5 @@ func (g *inspector) Config() *config.Config {
 
 // FilecoinVersion returns the version of venus.
 func (g *inspector) FilecoinVersion() string {
-	return fmt.Sprintf("%s %s", flags.GitTag, flags.GitCommit)
+	return constants.UserVersion()
 }

app/node/test/api.go

@@ -4,11 +4,12 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/filecoin-project/venus/cmd"
 	"os"
 	"strings"
 	"testing"
 
+	"github.com/filecoin-project/venus/cmd"
+
 	"github.com/stretchr/testify/require"
 
 	"github.com/filecoin-project/venus/app/node"

app/node/test/builder.go

@@ -2,9 +2,10 @@ package test
 
 import (
 	"context"
-	"github.com/filecoin-project/venus/pkg/jwtauth"
 	"testing"
 
+	"github.com/filecoin-project/venus/pkg/jwtauth"
+
 	"github.com/filecoin-project/venus/pkg/wallet"
 
 	"github.com/stretchr/testify/require"

app/node/test/setup.go

@@ -7,11 +7,12 @@ import (
 	"testing"
 	"time"
 
+	th "github.com/filecoin-project/venus/pkg/testhelpers"
+
 	"github.com/filecoin-project/go-address"
 	"github.com/stretchr/testify/require"
 
 	"github.com/filecoin-project/venus/app/node"
-	"github.com/filecoin-project/venus/build/project"
 	"github.com/filecoin-project/venus/pkg/clock"
 	"github.com/filecoin-project/venus/pkg/constants"
 	gengen "github.com/filecoin-project/venus/tools/gengen/util"
@@ -26,7 +27,7 @@ func CreateBootstrapSetup(t *testing.T) (*ChainSeed, *gengen.GenesisCfg, clock.C
 	fakeClock := clock.NewFake(time.Unix(genTime, 0))
 
 	// Load genesis config fixture.
-	genCfgPath := project.Root("fixtures/setup.json")
+	genCfgPath := th.Root("fixtures/setup.json")
 	genCfg := loadGenesisConfig(t, genCfgPath)
 	genCfg.Miners = append(genCfg.Miners, &gengen.CreateStorageMinerConfig{
 		Owner:         5,