Add more storage deal filter parameters (filecoin-project#497)

* feat: add more storage deal filter parameters

* test: verify that deal filter is being called with correct params

gql/resolver.go

@@ -13,9 +13,9 @@ import (
 	gqltypes "github.com/filecoin-project/boost/gql/types"
 	"github.com/filecoin-project/boost/node/config"
 	"github.com/filecoin-project/boost/retrievalmarket/rtvllog"
-	"github.com/filecoin-project/boost/sealingpipeline"
 	"github.com/filecoin-project/boost/storagemanager"
 	"github.com/filecoin-project/boost/storagemarket"
+	"github.com/filecoin-project/boost/storagemarket/sealingpipeline"
 	"github.com/filecoin-project/boost/storagemarket/types"
 	"github.com/filecoin-project/boost/storagemarket/types/dealcheckpoints"
 	"github.com/filecoin-project/boost/transport"
@@ -88,14 +88,6 @@ func NewResolver(cfg *config.Boost, r lotus_repo.LockedRepo, h host.Host, dealsD
 	}
 }
 
-type storageResolver struct {
-	Staged      gqltypes.Uint64
-	Transferred gqltypes.Uint64
-	Pending     gqltypes.Uint64
-	Free        gqltypes.Uint64
-	MountPoint  string
-}
-
 // query: deal(id) Deal
 func (r *resolver) Deal(ctx context.Context, args struct{ ID graphql.ID }) (*dealResolver, error) {
 	id, err := toUuid(args.ID)

gql/resolver_funds.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	gqltypes "github.com/filecoin-project/boost/gql/types"
+	smfunds "github.com/filecoin-project/boost/storagemarket/funds"
 	"github.com/graph-gophers/graphql-go"
 )
 
@@ -29,40 +30,25 @@ type funds struct {
 
 // query: funds: Funds
 func (r *resolver) Funds(ctx context.Context) (*funds, error) {
-	tagged, err := r.fundMgr.TotalTagged(ctx)
+	fnds, err := smfunds.GetStatus(ctx, r.fundMgr)
 	if err != nil {
-		return nil, fmt.Errorf("getting total tagged: %w", err)
-	}
-
-	balMkt, err := r.fundMgr.BalanceMarket(ctx)
-	if err != nil {
-		return nil, fmt.Errorf("getting market balance: %w", err)
-	}
-
-	balPubMsg, err := r.fundMgr.BalancePublishMsg(ctx)
-	if err != nil {
-		return nil, fmt.Errorf("getting publish message balance: %w", err)
-	}
-
-	balCollateral, err := r.fundMgr.BalanceDealCollateral(ctx)
-	if err != nil {
-		return nil, fmt.Errorf("getting deal collateral balance: %w", err)
+		return nil, err
 	}
 
 	return &funds{
 		Escrow: fundsEscrow{
-			Tagged:    gqltypes.BigInt{Int: tagged.Collateral},
-			Available: gqltypes.BigInt{Int: balMkt.Available},
-			Locked:    gqltypes.BigInt{Int: balMkt.Locked},
+			Available: gqltypes.BigInt{Int: fnds.Escrow.Available},
+			Locked:    gqltypes.BigInt{Int: fnds.Escrow.Locked},
+			Tagged:    gqltypes.BigInt{Int: fnds.Escrow.Tagged},
 		},
 		Collateral: fundsWallet{
-			Address: r.fundMgr.AddressDealCollateral().String(),
-			Balance: gqltypes.BigInt{Int: balCollateral},
+			Address: fnds.Collateral.Address,
+			Balance: gqltypes.BigInt{Int: fnds.Collateral.Balance},
 		},
 		PubMsg: fundsWallet{
-			Address: r.fundMgr.AddressPublishMsg().String(),
-			Balance: gqltypes.BigInt{Int: balPubMsg},
-			Tagged:  gqltypes.BigInt{Int: tagged.PubMsg},
+			Address: fnds.PubMsg.Address,
+			Balance: gqltypes.BigInt{Int: fnds.PubMsg.Balance},
+			Tagged:  gqltypes.BigInt{Int: fnds.PubMsg.Tagged},
 		},
 	}, nil
 }

gql/resolver_storage.go

@@ -7,6 +7,14 @@ import (
 	"github.com/filecoin-project/boost/storagemarket/types/dealcheckpoints"
 )
 
+type storageResolver struct {
+	Staged      gqltypes.Uint64
+	Transferred gqltypes.Uint64
+	Pending     gqltypes.Uint64
+	Free        gqltypes.Uint64
+	MountPoint  string
+}
+
 // query: storage: [Storage]
 func (r *resolver) Storage(ctx context.Context) (*storageResolver, error) {
 	tagged, err := r.storageMgr.TotalTagged(ctx)

itests/dummydeal_test.go

@@ -33,10 +33,11 @@ func TestDummydealOnline(t *testing.T) {
 	tempdir := t.TempDir()
 	log.Debugw("using tempdir", "dir", tempdir)
 
-	randomFilepath, err := testutil.CreateRandomFile(tempdir, 5, 2000000)
+	fileSize := 2000000
+	randomFilepath, err := testutil.CreateRandomFile(tempdir, 5, fileSize)
 	require.NoError(t, err)
 
-	failingFilepath, err := testutil.CreateRandomFile(tempdir, 5, 2000000)
+	failingFilepath, err := testutil.CreateRandomFile(tempdir, 6, fileSize)
 	require.NoError(t, err)
 
 	// NOTE: these calls to CreateDenseCARv2 have the identity CID builder enabled so will
@@ -49,11 +50,13 @@ func TestDummydealOnline(t *testing.T) {
 	require.NoError(t, err)
 
 	// Start a web server to serve the car files
+	log.Debug("starting webserver")
 	server, err := testutil.HttpTestFileServer(t, tempdir)
 	require.NoError(t, err)
 	defer server.Close()
 
 	// Create a new dummy deal
+	log.Debug("creating dummy deal")
 	dealUuid := uuid.New()
 
 	// Make a deal

itests/framework/framework.go

@@ -181,9 +181,11 @@ func (f *TestFramework) Start() error {
 		wg.Done()
 	}()
 
-	// Create a wallet for publish storage deals with some funds
-	wg.Add(1)
+	// Create wallets for publish storage deals and deal collateral with
+	// some funds
+	wg.Add(2)
 	var psdWalletAddr address.Address
+	var dealCollatAddr address.Address
 	go func() {
 		Log.Info("Creating publish storage deals wallet")
 		psdWalletAddr, _ = fullnodeApi.WalletNew(f.ctx, chaintypes.KTBLS)
@@ -193,6 +195,15 @@ func (f *TestFramework) Start() error {
 		Log.Infof("Created publish storage deals wallet %s with %d attoFil", psdWalletAddr, amt)
 		wg.Done()
 	}()
+	go func() {
+		Log.Info("Creating deal collateral wallet")
+		dealCollatAddr, _ = fullnodeApi.WalletNew(f.ctx, chaintypes.KTBLS)
+
+		amt := abi.NewTokenAmount(1e18)
+		_ = sendFunds(f.ctx, fullnodeApi, dealCollatAddr, amt)
+		Log.Infof("Created deal collateral wallet %s with %d attoFil", dealCollatAddr, amt)
+		wg.Done()
+	}()
 	wg.Wait()
 
 	f.ClientAddr = clientAddr
@@ -267,6 +278,7 @@ func (f *TestFramework) Start() error {
 	cfg.SealerApiInfo = apiInfo
 	cfg.Wallets.Miner = minerAddr.String()
 	cfg.Wallets.PublishStorageDeals = psdWalletAddr.String()
+	cfg.Wallets.DealCollateral = dealCollatAddr.String()
 	cfg.LotusDealmaking.MaxDealsPerPublishMsg = 1
 	cfg.LotusDealmaking.PublishMsgPeriod = lotus_config.Duration(0)
 	val, err := ltypes.ParseFIL("0.1 FIL")

node/builder.go

@@ -21,10 +21,10 @@ import (
 	"github.com/filecoin-project/boost/protocolproxy"
 	"github.com/filecoin-project/boost/retrievalmarket/lp2pimpl"
 	"github.com/filecoin-project/boost/retrievalmarket/rtvllog"
-	"github.com/filecoin-project/boost/sealingpipeline"
 	"github.com/filecoin-project/boost/storagemanager"
 	"github.com/filecoin-project/boost/storagemarket"
 	"github.com/filecoin-project/boost/storagemarket/dealfilter"
+	"github.com/filecoin-project/boost/storagemarket/sealingpipeline"
 	smtypes "github.com/filecoin-project/boost/storagemarket/types"
 	"github.com/filecoin-project/boost/tracing"
 	"github.com/filecoin-project/dagstore"
@@ -555,7 +555,7 @@ func ConfigBoost(cfg *config.Boost) Option {
 		// Boost storage deal filter
 		Override(new(dtypes.StorageDealFilter), modules.BasicDealFilter(cfg.Dealmaking, nil)),
 		If(cfg.Dealmaking.Filter != "",
-			Override(new(dtypes.StorageDealFilter), modules.BasicDealFilter(cfg.Dealmaking, dealfilter.CliStorageDealFilter(cfg.Dealmaking.Filter))),
+			Override(new(dtypes.StorageDealFilter), modules.BasicDealFilter(cfg.Dealmaking, dtypes.StorageDealFilter(dealfilter.CliStorageDealFilter(cfg.Dealmaking.Filter)))),
 		),
 
 		// Lotus markets storage deal filter
@@ -567,7 +567,7 @@ func ConfigBoost(cfg *config.Boost) Option {
 		// Boost retrieval deal filter
 		Override(new(dtypes.RetrievalDealFilter), modules.RetrievalDealFilter(nil)),
 		If(cfg.Dealmaking.RetrievalFilter != "",
-			Override(new(dtypes.RetrievalDealFilter), modules.RetrievalDealFilter(dealfilter.CliRetrievalDealFilter(cfg.Dealmaking.RetrievalFilter))),
+			Override(new(dtypes.RetrievalDealFilter), modules.RetrievalDealFilter(dtypes.RetrievalDealFilter(dealfilter.CliRetrievalDealFilter(cfg.Dealmaking.RetrievalFilter)))),
 		),
 
 		// Lotus markets retrieval deal filter

node/impl/boost.go

@@ -18,8 +18,8 @@ import (
 	"github.com/filecoin-project/boost/gql"
 	"github.com/filecoin-project/boost/indexprovider"
 	"github.com/filecoin-project/boost/node/modules/dtypes"
-	"github.com/filecoin-project/boost/sealingpipeline"
 	"github.com/filecoin-project/boost/storagemarket"
+	"github.com/filecoin-project/boost/storagemarket/sealingpipeline"
 	"github.com/filecoin-project/boost/storagemarket/types"
 	"github.com/filecoin-project/dagstore"
 	"github.com/filecoin-project/dagstore/shard"

node/modules/dealfilter.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/filecoin-project/boost/node/config"
 	"github.com/filecoin-project/boost/node/modules/dtypes"
-	"github.com/filecoin-project/boost/storagemarket/types"
+	"github.com/filecoin-project/boost/storagemarket/dealfilter"
 	lotus_repo "github.com/filecoin-project/lotus/node/repo"
 )
 
@@ -28,7 +28,7 @@ func BasicDealFilter(cfg config.DealmakingConfig, userCmd dtypes.StorageDealFilt
 		startDelay dtypes.GetMaxDealStartDelayFunc,
 		r lotus_repo.LockedRepo,
 	) dtypes.StorageDealFilter {
-		return func(ctx context.Context, params types.DealFilterParams) (bool, string, error) {
+		return func(ctx context.Context, params dealfilter.DealFilterParams) (bool, string, error) {
 			deal := params.DealParams
 			pr := deal.ClientDealProposal.Proposal
 

node/modules/dtypes/miner.go

@@ -4,12 +4,11 @@ import (
 	"context"
 	"time"
 
-	"github.com/ipfs/go-cid"
-
-	"github.com/filecoin-project/boost/storagemarket/types"
+	"github.com/filecoin-project/boost/storagemarket/dealfilter"
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/lotus/storage/pipeline/sealiface"
+	"github.com/ipfs/go-cid"
 )
 
 type MinerID abi.ActorID
@@ -90,7 +89,7 @@ type GetExpectedSealDurationFunc func() (time.Duration, error)
 type SetMaxDealStartDelayFunc func(time.Duration) error
 type GetMaxDealStartDelayFunc func() (time.Duration, error)
 
-type StorageDealFilter func(ctx context.Context, deal types.DealFilterParams) (bool, string, error)
-type RetrievalDealFilter func(ctx context.Context, deal retrievalmarket.ProviderDealState) (bool, string, error)
+type StorageDealFilter dealfilter.StorageDealFilter
+type RetrievalDealFilter dealfilter.RetrievalDealFilter
 
 type RetrievalPricingFunc func(ctx context.Context, dealPricingParams retrievalmarket.PricingInput) (retrievalmarket.Ask, error)

node/modules/storageminer.go

@@ -17,11 +17,11 @@ import (
 	"github.com/filecoin-project/boost/node/modules/dtypes"
 	brm "github.com/filecoin-project/boost/retrievalmarket/lib"
 	"github.com/filecoin-project/boost/retrievalmarket/rtvllog"
-	"github.com/filecoin-project/boost/sealingpipeline"
 	"github.com/filecoin-project/boost/storagemanager"
 	"github.com/filecoin-project/boost/storagemarket"
 	"github.com/filecoin-project/boost/storagemarket/logs"
 	"github.com/filecoin-project/boost/storagemarket/lp2pimpl"
+	"github.com/filecoin-project/boost/storagemarket/sealingpipeline"
 	"github.com/filecoin-project/boost/storagemarket/types"
 	"github.com/filecoin-project/boost/tracing"
 	"github.com/filecoin-project/boost/transport/httptransport"

node/modules/storageminer_svc.go

@@ -9,7 +9,7 @@ import (
 	"go.uber.org/fx"
 
 	"github.com/filecoin-project/boost/api"
-	"github.com/filecoin-project/boost/sealingpipeline"
+	"github.com/filecoin-project/boost/storagemarket/sealingpipeline"
 
 	lapi "github.com/filecoin-project/lotus/api"
 	lclient "github.com/filecoin-project/lotus/api/client"

storagemanager/storagemanager.go

@@ -29,7 +29,7 @@ type Config struct {
 type StorageManager struct {
 	lr                 lotus_repo.LockedRepo
 	db                 *db.StorageDB
-	cfg                Config
+	Cfg                Config
 	StagingAreaDirPath string
 }
 
@@ -47,7 +47,7 @@ func New(cfg Config) func(lr lotus_repo.LockedRepo, sqldb *sql.DB) (*StorageMana
 
 		return &StorageManager{
 			db:                 db.NewStorageDB(sqldb),
-			cfg:                cfg,
+			Cfg:                cfg,
 			lr:                 lr,
 			StagingAreaDirPath: stagingPath,
 		}, nil
@@ -62,7 +62,7 @@ func (m *StorageManager) Free(ctx context.Context) (uint64, error) {
 		return 0, fmt.Errorf("getting total tagged: %w", err)
 	}
 
-	return m.cfg.MaxStagingDealsBytes - tagged, nil
+	return m.Cfg.MaxStagingDealsBytes - tagged, nil
 }
 
 // ErrNoSpaceLeft indicates that there is insufficient storage to accept a deal
@@ -72,23 +72,23 @@ var ErrNoSpaceLeft = errors.New("no space left")
 // If there is not enough space left, returns ErrNoSpaceLeft.
 func (m *StorageManager) Tag(ctx context.Context, dealUuid uuid.UUID, size uint64, host string) error {
 	// Get the total tagged storage, so that we know how much is available.
-	log.Debugw("tagging", "id", dealUuid, "size", size, "host", host, "maxbytes", m.cfg.MaxStagingDealsBytes)
+	log.Debugw("tagging", "id", dealUuid, "size", size, "host", host, "maxbytes", m.Cfg.MaxStagingDealsBytes)
 
-	if m.cfg.MaxStagingDealsBytes != 0 {
-		if m.cfg.MaxStagingDealsPercentPerHost != 0 {
+	if m.Cfg.MaxStagingDealsBytes != 0 {
+		if m.Cfg.MaxStagingDealsPercentPerHost != 0 {
 			// Get the total amount tagged for download from the host
 			tagged, err := m.TotalTaggedForHost(ctx, host)
 			if err != nil {
 				return fmt.Errorf("getting total tagged for host: %w", err)
 			}
 
 			// Check the amount tagged + the size of the proposed deal against the limit
-			limit := (m.cfg.MaxStagingDealsBytes * m.cfg.MaxStagingDealsPercentPerHost) / 100
+			limit := (m.Cfg.MaxStagingDealsBytes * m.Cfg.MaxStagingDealsPercentPerHost) / 100
 			if tagged+size >= limit {
 				return fmt.Errorf("%w: cannot accept piece of size %d from host %s, "+
 					"on top of already allocated %d bytes, because it would exceed max %d bytes: "+
 					"staging area size %d x per host limit %d%%",
-					ErrNoSpaceLeft, size, host, tagged, limit, m.cfg.MaxStagingDealsBytes, m.cfg.MaxStagingDealsPercentPerHost)
+					ErrNoSpaceLeft, size, host, tagged, limit, m.Cfg.MaxStagingDealsBytes, m.Cfg.MaxStagingDealsPercentPerHost)
 			}
 		}
 
@@ -99,9 +99,9 @@ func (m *StorageManager) Tag(ctx context.Context, dealUuid uuid.UUID, size uint6
 		}
 
 		// Check the amount tagged + the size of the proposed deal against the limit
-		if tagged+size >= m.cfg.MaxStagingDealsBytes {
+		if tagged+size >= m.Cfg.MaxStagingDealsBytes {
 			err := fmt.Errorf("%w: cannot accept piece of size %d, on top of already allocated %d bytes, because it would exceed max staging area size %d",
-				ErrNoSpaceLeft, size, tagged, m.cfg.MaxStagingDealsBytes)
+				ErrNoSpaceLeft, size, tagged, m.Cfg.MaxStagingDealsBytes)
 			return err
 		}
 	}

storagemarket/dealfilter/cli.go

@@ -6,33 +6,43 @@ import (
 	"encoding/json"
 	"os/exec"
 
-	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
-
-	"github.com/filecoin-project/boost/node/modules/dtypes"
+	"github.com/filecoin-project/boost/storagemarket/funds"
+	"github.com/filecoin-project/boost/storagemarket/sealingpipeline"
+	"github.com/filecoin-project/boost/storagemarket/storagespace"
 	"github.com/filecoin-project/boost/storagemarket/types"
+	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 )
 
 const agent = "boost"
-const jsonVersion = "2.0.0"
+const jsonVersion = "2.1.0"
+
+type StorageDealFilter func(ctx context.Context, deal DealFilterParams) (bool, string, error)
+type RetrievalDealFilter func(ctx context.Context, deal retrievalmarket.ProviderDealState) (bool, string, error)
 
-func CliStorageDealFilter(cmd string) dtypes.StorageDealFilter {
-	return func(ctx context.Context, deal types.DealFilterParams) (bool, string, error) {
+func CliStorageDealFilter(cmd string) StorageDealFilter {
+	return func(ctx context.Context, deal DealFilterParams) (bool, string, error) {
 		d := struct {
 			types.DealParams
-			DealType      string
-			FormatVersion string
-			Agent         string
+			SealingPipelineState sealingpipeline.Status
+			FundsState           funds.Status
+			StorageState         storagespace.Status
+			DealType             string
+			FormatVersion        string
+			Agent                string
 		}{
-			DealParams:    *deal.DealParams,
-			DealType:      "storage",
-			FormatVersion: jsonVersion,
-			Agent:         agent,
+			DealParams:           deal.DealParams,
+			SealingPipelineState: deal.SealingPipelineState,
+			FundsState:           deal.FundsState,
+			StorageState:         deal.StorageState,
+			DealType:             "storage",
+			FormatVersion:        jsonVersion,
+			Agent:                agent,
 		}
 		return runDealFilter(ctx, cmd, d)
 	}
 }
 
-func CliRetrievalDealFilter(cmd string) dtypes.RetrievalDealFilter {
+func CliRetrievalDealFilter(cmd string) RetrievalDealFilter {
 	return func(ctx context.Context, deal retrievalmarket.ProviderDealState) (bool, string, error) {
 		d := struct {
 			retrievalmarket.ProviderDealState