Speed up tests on CI (filecoin-project#409)

* feat: speed up tests on CI

* add configs for batch precommit and commit

* dtypes for lotus ; mockProofs for dummydeal

* remove lotus-artefacts

* enable market v1 deals; split itest for market v1 in two circleci jobs

* split itest for market v1

* fetch-params with boostx; remove lotus binary dep in circleci config.yml

* use real proofs

Co-authored-by: Anton Evangelatov <anton.evangelatov@gmail.com>

.circleci/config.yml

@@ -45,38 +45,12 @@ commands:
             - 'v25-8mb-lotus-params'
           paths:
             - /var/tmp/filecoin-proof-parameters/
-      - run:  lotus fetch-params 8388608
+      - run:  ./boostx fetch-params 8388608
       - save_cache:
           name: Save parameters cache
           key: 'v25-8mb-lotus-params'
           paths:
             - /var/tmp/filecoin-proof-parameters/
-  lotus-artefacts:
-    steps:
-      - restore_cache:
-          name: Restore lotus artefacts cache
-          keys:
-            - 'lotus-d118ef92'
-          paths:
-            - /tmp/lotus-artefacts
-      - setup_remote_docker:
-          version: 19.03.13
-      - run:
-          name: copy latest stable lotus binaries
-          command: |
-            docker pull public.ecr.aws/p5f3y3g5/lotus-test:d118ef92
-            id=$(docker create public.ecr.aws/p5f3y3g5/lotus-test:d118ef92)
-            mkdir -p /tmp/lotus-artefacts
-            docker cp $id:/usr/local/bin/lotus-seed /tmp/lotus-artefacts/lotus-seed
-            docker cp $id:/usr/local/bin/lotus-miner /tmp/lotus-artefacts/lotus-miner
-            docker cp $id:/usr/local/bin/lotus /tmp/lotus-artefacts/lotus
-            docker cp $id:/lib/libhwloc.so.5 /tmp/lotus-artefacts/libhwloc.so.5
-            docker rm -v $id
-      - save_cache:
-          name: Save lotus artefacts cache
-          key: 'lotus-d118ef92'
-          paths:
-            - /tmp/lotus-artefacts
   git_fetch_all_tags:
     steps:
       - run:
@@ -135,14 +109,6 @@ jobs:
       - run:
           command: make debug
           no_output_timeout: 30m
-      - lotus-artefacts
-      - run:
-          name: copy library and add lotus binaries to PATH
-          command: |
-            sudo cp /tmp/lotus-artefacts/libhwloc.so.5 /lib/x86_64-linux-gnu/
-            sudo cp /tmp/lotus-artefacts/lotus /usr/local/bin/
-            sudo cp /tmp/lotus-artefacts/lotus-miner /usr/local/bin/
-            sudo cp /tmp/lotus-artefacts/lotus-seed /usr/local/bin/
       - download-params
       - run:
           name: go test
@@ -318,10 +284,27 @@ workflows:
             tags:
               only:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
+
       - test:
-          name: test-itest
-          suite: itest
-          target: "./itests/..."
+          name: test-itest-dummdeal_offline
+          suite: itest-dummdeal_offline
+          target: "./itests/dummydeal_offline_test.go"
+
+      - test:
+          name: test-itest-dummdeal
+          suite: itest-dummdeal
+          target: "./itests/dummydeal_test.go"
+
+      - test:
+          name: test-itest-markets_v1_deal
+          suite: itest-markets_v1_deal
+          target: "./itests/markets_v1_deal_test.go"
+
+      - test:
+          name: test-itest-markets_v1_offline_deal
+          suite: itest-markets_v1_offline_deal
+          target: "./itests/markets_v1_offline_deal_test.go"
+
       - test:
           name: test-all
           suite: all

cmd/boostx/main.go

@@ -32,6 +32,7 @@ func main() {
 			commpCmd,
 			generatecarCmd,
 			marketCmd,
+			fetchParamCmd,
 		},
 	}
 	app.Setup()

cmd/boostx/utils_cmd.go

@@ -22,8 +22,12 @@ import (
 	lcli "github.com/filecoin-project/lotus/cli"
 	"github.com/filecoin-project/lotus/lib/backupds"
 
+	"github.com/docker/go-units"
+	paramfetch "github.com/filecoin-project/go-paramfetch"
+	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/lib/unixfs"
 	"github.com/filecoin-project/lotus/node/modules"
+	lotus_repo "github.com/filecoin-project/lotus/node/repo"
 	"github.com/filecoin-project/lotus/node/repo/imports"
 	"github.com/ipfs/go-cidutil/cidenc"
 	"github.com/ipfs/go-datastore"
@@ -35,8 +39,6 @@ import (
 	"github.com/multiformats/go-multibase"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/xerrors"
-
-	lotus_repo "github.com/filecoin-project/lotus/node/repo"
 )
 
 var marketCmd = &cli.Command{
@@ -292,3 +294,28 @@ var generatecarCmd = &cli.Command{
 		return nil
 	},
 }
+
+var fetchParamCmd = &cli.Command{
+	Name:      "fetch-params",
+	Usage:     "Fetch proving parameters",
+	ArgsUsage: "[sectorSize]",
+	Action: func(cctx *cli.Context) error {
+		ctx := lcli.ReqContext(cctx)
+
+		if !cctx.Args().Present() {
+			return xerrors.Errorf("must pass sector size to fetch params for (specify as \"32GiB\", for instance)")
+		}
+		sectorSizeInt, err := units.RAMInBytes(cctx.Args().First())
+		if err != nil {
+			return xerrors.Errorf("error parsing sector size (specify as \"32GiB\", for instance): %w", err)
+		}
+		sectorSize := uint64(sectorSizeInt)
+
+		err = paramfetch.GetParams(ctx, build.ParametersJSON(), build.SrsJSON(), sectorSize)
+		if err != nil {
+			return xerrors.Errorf("fetching proof parameters: %w", err)
+		}
+
+		return nil
+	},
+}

go.mod

@@ -28,10 +28,11 @@ require (
 	github.com/filecoin-project/go-fil-markets v1.20.2-0.20220325122707-b18483a0834b
 	github.com/filecoin-project/go-jsonrpc v0.1.5
 	github.com/filecoin-project/go-padreader v0.0.1
+	github.com/filecoin-project/go-paramfetch v0.0.4
 	github.com/filecoin-project/go-state-types v0.1.3
 	github.com/filecoin-project/go-statestore v0.2.0
 	github.com/filecoin-project/index-provider v0.5.0
-	github.com/filecoin-project/lotus v1.15.1-0.20220321111228-3c1edca90295
+	github.com/filecoin-project/lotus v1.15.1-0.20220407132257-de8258c7a1bd
 	github.com/filecoin-project/specs-actors v0.9.14
 	github.com/filecoin-project/specs-actors/v2 v2.3.6
 	github.com/filecoin-project/specs-actors/v5 v5.0.4
@@ -79,7 +80,6 @@ require (
 	github.com/libp2p/go-libp2p-record v0.1.3
 	github.com/libp2p/go-libp2p-swarm v0.10.2
 	github.com/mattn/go-sqlite3 v1.14.9
-	github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/multiformats/go-multiaddr v0.5.0
 	github.com/multiformats/go-multibase v0.0.3

go.sum

@@ -373,8 +373,8 @@ github.com/filecoin-project/go-storedcounter v0.1.0 h1:Mui6wSUBC+cQGHbDUBcO7rfh5
 github.com/filecoin-project/go-storedcounter v0.1.0/go.mod h1:4ceukaXi4vFURIoxYMfKzaRF5Xv/Pinh2oTnoxpv+z8=
 github.com/filecoin-project/index-provider v0.5.0 h1:k2C1RFvOvxmA2i8bhmkb3b4qun7RDRDzzs/y25/TwQg=
 github.com/filecoin-project/index-provider v0.5.0/go.mod h1:KHVrP2vU3YuScb+fawObwTFoR882up9U07kk0ZrfP0c=
-github.com/filecoin-project/lotus v1.15.1-0.20220321111228-3c1edca90295 h1:mCENEontHYNg8ze6HNJioIqwJwhThlkJ19pcjxlatfQ=
-github.com/filecoin-project/lotus v1.15.1-0.20220321111228-3c1edca90295/go.mod h1:lWTkxHRmO6ir+uyMvvupnBcC0YMJXXQ+5cOdXuCyoFU=
+github.com/filecoin-project/lotus v1.15.1-0.20220407132257-de8258c7a1bd h1:JA8Vb0ytFYBHnTVOt8+UBCv0wnd6olN/Xk5CnUq0Mz4=
+github.com/filecoin-project/lotus v1.15.1-0.20220407132257-de8258c7a1bd/go.mod h1:lWTkxHRmO6ir+uyMvvupnBcC0YMJXXQ+5cOdXuCyoFU=
 github.com/filecoin-project/specs-actors v0.9.13/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao=
 github.com/filecoin-project/specs-actors v0.9.14 h1:68PVstg2UB3ZsMLF+DKFTAs/YKsqhKWynkr0IqmVRQY=
 github.com/filecoin-project/specs-actors v0.9.14/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao=

itests/dummydeal_offline_test.go

@@ -0,0 +1,46 @@
+package itests
+
+import (
+	"context"
+	"testing"
+
+	"github.com/filecoin-project/boost/itests/framework"
+	"github.com/filecoin-project/boost/testutil"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/lotus/itests/kit"
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDummydealOffline(t *testing.T) {
+	ctx := context.Background()
+
+	kit.QuietMiningLogs()
+	framework.SetLogLevel()
+	framework.SetPreCommitChallengeDelay(t, 5)
+	f := framework.NewTestFramework(ctx, t)
+	err := f.Start()
+	require.NoError(t, err)
+	defer f.Stop()
+
+	err = f.AddClientProviderBalance(abi.NewTokenAmount(1e15))
+	require.NoError(t, err)
+
+	// Create a CAR file
+	tempdir := t.TempDir()
+	randomFilepath, err := testutil.CreateRandomFile(tempdir, 5, 2000000)
+	require.NoError(t, err)
+	rootCid, carFilepath, err := testutil.CreateDenseCARv2(tempdir, randomFilepath)
+	require.NoError(t, err)
+
+	// make an offline deal
+	offlineDealUuid := uuid.New()
+	res, err := f.MakeDummyDeal(offlineDealUuid, carFilepath, rootCid, "", true)
+	require.NoError(t, err)
+	require.True(t, res.Accepted)
+	res, err = f.Boost.BoostOfflineDealWithData(context.Background(), offlineDealUuid, carFilepath)
+	require.NoError(t, err)
+	require.True(t, res.Accepted)
+	err = f.WaitForDealAddedToSector(offlineDealUuid)
+	require.NoError(t, err)
+}

itests/dummydeal_test.go

@@ -7,12 +7,29 @@ import (
 	"time"
 
 	"github.com/davecgh/go-spew/spew"
+	"github.com/filecoin-project/boost/itests/framework"
 	"github.com/filecoin-project/boost/testutil"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/lotus/itests/kit"
 	"github.com/google/uuid"
 	"github.com/stretchr/testify/require"
 )
 
 func TestDummydeal(t *testing.T) {
+	ctx := context.Background()
+	log := framework.Log
+
+	kit.QuietMiningLogs()
+	framework.SetLogLevel()
+	framework.SetPreCommitChallengeDelay(t, 5)
+	f := framework.NewTestFramework(ctx, t)
+	err := f.Start()
+	require.NoError(t, err)
+	defer f.Stop()
+
+	err = f.AddClientProviderBalance(abi.NewTokenAmount(1e15))
+	require.NoError(t, err)
+
 	// Create a CAR file
 	tempdir := t.TempDir()
 	log.Debugw("using tempdir", "dir", tempdir)
@@ -37,42 +54,35 @@ func TestDummydeal(t *testing.T) {
 	// Create a new dummy deal
 	dealUuid := uuid.New()
 
-	res, err := f.makeDummyDeal(dealUuid, carFilepath, rootCid, server.URL+"/"+filepath.Base(carFilepath), false)
+	// Make a deal
+	res, err := f.MakeDummyDeal(dealUuid, carFilepath, rootCid, server.URL+"/"+filepath.Base(carFilepath), false)
 	require.NoError(t, err)
 	require.True(t, res.Accepted)
 	log.Debugw("got response from MarketDummyDeal", "res", spew.Sdump(res))
 
 	time.Sleep(2 * time.Second)
 
+	// Make a second deal - it should fail because the first deal took up all
+	// available space
 	failingDealUuid := uuid.New()
-	res2, err2 := f.makeDummyDeal(failingDealUuid, failingCarFilepath, failingRootCid, server.URL+"/"+filepath.Base(failingCarFilepath), false)
+	res2, err2 := f.MakeDummyDeal(failingDealUuid, failingCarFilepath, failingRootCid, server.URL+"/"+filepath.Base(failingCarFilepath), false)
 	require.NoError(t, err2)
 	require.Equal(t, "cannot accept piece of size 2254421, on top of already allocated 2254421 bytes, because it would exceed max staging area size 4000000", res2.Reason)
 	log.Debugw("got response from MarketDummyDeal for failing deal", "res2", spew.Sdump(res2))
 
-	// Wait for the deal to be added to a sector and be cleanedup so space is made
-	err = f.waitForDealAddedToSector(dealUuid)
+	// Wait for the first deal to be added to a sector and cleaned up so space is made
+	err = f.WaitForDealAddedToSector(dealUuid)
 	require.NoError(t, err)
 	time.Sleep(100 * time.Millisecond)
 
+	// Make a third deal - it should succeed because the first deal has been cleaned up
 	passingDealUuid := uuid.New()
-	res2, err2 = f.makeDummyDeal(passingDealUuid, failingCarFilepath, failingRootCid, server.URL+"/"+filepath.Base(failingCarFilepath), false)
+	res2, err2 = f.MakeDummyDeal(passingDealUuid, failingCarFilepath, failingRootCid, server.URL+"/"+filepath.Base(failingCarFilepath), false)
 	require.NoError(t, err2)
 	require.True(t, res2.Accepted)
 	log.Debugw("got response from MarketDummyDeal", "res2", spew.Sdump(res2))
 
 	// Wait for the deal to be added to a sector
-	err = f.waitForDealAddedToSector(passingDealUuid)
-	require.NoError(t, err)
-
-	// make an offline deal
-	offlineDealUuid := uuid.New()
-	res, err = f.makeDummyDeal(offlineDealUuid, carFilepath, rootCid, server.URL+"/"+filepath.Base(carFilepath), true)
-	require.NoError(t, err)
-	require.True(t, res.Accepted)
-	res, err = f.boost.BoostOfflineDealWithData(context.Background(), offlineDealUuid, carFilepath)
-	require.NoError(t, err)
-	require.True(t, res.Accepted)
-	err = f.waitForDealAddedToSector(offlineDealUuid)
+	err = f.WaitForDealAddedToSector(passingDealUuid)
 	require.NoError(t, err)
 }