feat: make Boost curio compliant

cmd/boostd/init.go

@@ -87,7 +87,7 @@ var initCmd = &cli.Command{
 			}
 
 			rcfg.ConfigVersion = config.CurrentVersion
-			cerr = setMinerApiConfig(cctx, rcfg, true)
+			cerr = setMinerApiConfig(cctx, rcfg)
 			if cerr != nil {
 				return
 			}
@@ -118,9 +118,14 @@ var initCmd = &cli.Command{
 			return fmt.Errorf("writing config file %s: %w", string(newCfg), err)
 		}
 
+		miner, err := address.NewFromString(curCfg.Wallets.Miner)
+		if err != nil {
+			return fmt.Errorf("converting miner address: %w", err)
+		}
+
 		// Add the miner address to the metadata datastore
-		fmt.Printf("Adding miner address %s to datastore\n", bp.minerActor)
-		err = addMinerAddressToDatastore(ds, bp.minerActor)
+		fmt.Printf("Adding miner address %s to datastore\n", miner)
+		err = addMinerAddressToDatastore(ds, miner)
 		if err != nil {
 			return err
 		}
@@ -140,10 +145,9 @@ var initCmd = &cli.Command{
 }
 
 type boostParams struct {
-	repo       *lotus_repo.FsRepo
-	minerActor address.Address
-	walletPSD  address.Address
-	walletCP   address.Address
+	repo      *lotus_repo.FsRepo
+	walletPSD address.Address
+	walletCP  address.Address
 }
 
 func initBoost(ctx context.Context, cctx *cli.Context, marketsRepo lotus_repo.LockedRepo) (*boostParams, error) {
@@ -177,36 +181,6 @@ func initBoost(ctx context.Context, cctx *cli.Context, marketsRepo lotus_repo.Lo
 	}
 	defer closer()
 
-	var minerActor address.Address
-	if marketsRepo == nil {
-		// If this is not a migration from an existing repo, just query the
-		// miner directly for the actor address
-		smApi, smCloser, err := lcli.GetStorageMinerAPI(cctx)
-		if err != nil {
-			if strings.Contains(err.Error(), "could not get API info") {
-				err = fmt.Errorf("%w\nDo you need to set the environment variable MINER_API_INFO?", err)
-			}
-			return nil, err
-		}
-		defer smCloser()
-
-		minerActor, err = smApi.ActorAddress(ctx)
-		if err != nil {
-			return nil, fmt.Errorf("getting miner actor address: %w", err)
-		}
-	} else {
-		// This is a migration from an existing repo, so get the miner address
-		// from the repo datastore
-		ds, err := marketsRepo.Datastore(context.Background(), metadataNamespace)
-		if err != nil {
-			return nil, fmt.Errorf("getting legacy repo datastore: %w", err)
-		}
-		minerActor, err = getMinerAddressFromDatastore(ds)
-		if err != nil {
-			return nil, fmt.Errorf("getting miner actor address: %w", err)
-		}
-	}
-
 	fmt.Println("Checking full node sync status")
 
 	if err := lcli.SyncWait(ctx, &v0api.WrapperV1Full{FullNode: api}, false); err != nil {
@@ -248,56 +222,46 @@ func initBoost(ctx context.Context, cctx *cli.Context, marketsRepo lotus_repo.Lo
 	}
 
 	return &boostParams{
-		repo:       r,
-		minerActor: minerActor,
-		walletPSD:  walletPSD,
-		walletCP:   walletCP,
+		repo:      r,
+		walletPSD: walletPSD,
+		walletCP:  walletCP,
 	}, nil
 }
 
-func setMinerApiConfig(cctx *cli.Context, rcfg *config.Boost, dialCheck bool) error {
+func setMinerApiConfig(cctx *cli.Context, rcfg *config.Boost) error {
 	ctx := cctx.Context
-	asi, err := checkApiInfo(ctx, cctx.String("api-sector-index"), dialCheck)
+	asi, miner1, err := checkApiInfo(ctx, cctx.String("api-sector-index"))
 	if err != nil {
 		return fmt.Errorf("checking sector index API: %w", err)
 	}
 	fmt.Printf("Sector index api info: %s\n", asi)
 	rcfg.SectorIndexApiInfo = asi
 
-	ai, err := checkApiInfo(ctx, cctx.String("api-sealer"), dialCheck)
+	ai, miner2, err := checkApiInfo(ctx, cctx.String("api-sealer"))
 	if err != nil {
 		return fmt.Errorf("checking sealer API: %w", err)
 	}
 
+	if miner1 != miner2 {
+		return errors.New("sector index and sealer APIs belong to different miners")
+	}
+
 	fmt.Printf("Sealer api info: %s\n", ai)
+	fmt.Printf("Miner address: %s", miner1)
 	rcfg.SealerApiInfo = ai
+	rcfg.Wallets.Miner = miner1
 
 	return nil
 }
 
 func setCommonConfig(cctx *cli.Context, rcfg *config.Boost, bp *boostParams) {
 	rcfg.Dealmaking.MaxStagingDealsBytes = cctx.Int64("max-staging-deals-bytes")
-	rcfg.Wallets.Miner = bp.minerActor.String()
 	rcfg.Wallets.DealCollateral = bp.walletCP.String()
 	rcfg.Wallets.PublishStorageDeals = bp.walletPSD.String()
 }
 
 var minerAddrDSKey = datastore.NewKey("miner-address")
 
-func getMinerAddressFromDatastore(ds datastore.Batching) (address.Address, error) {
-	addr, err := ds.Get(context.Background(), minerAddrDSKey)
-	if err != nil {
-		return address.Address{}, fmt.Errorf("getting miner address from legacy datastore: %w", err)
-	}
-
-	minerAddr, err := address.NewFromBytes(addr)
-	if err != nil {
-		return address.Address{}, fmt.Errorf("parsing miner address from legacy datastore: %w", err)
-	}
-
-	return minerAddr, nil
-}
-
 func addMinerAddressToDatastore(ds datastore.Batching, minerActor address.Address) error {
 	return ds.Put(context.Background(), minerAddrDSKey, minerActor.Bytes())
 }
@@ -328,33 +292,34 @@ func checkV1ApiSupport(ctx context.Context, cctx *cli.Context) error {
 	return nil
 }
 
-func checkApiInfo(ctx context.Context, ai string, dialCheck bool) (string, error) {
+func checkApiInfo(ctx context.Context, ai string) (string, string, error) {
 	ai = strings.TrimPrefix(strings.TrimSpace(ai), "MINER_API_INFO=")
 	info := cliutil.ParseApiInfo(ai)
 	addr, err := info.DialArgs("v0")
 	if err != nil {
-		return "", fmt.Errorf("could not get DialArgs: %w", err)
-	}
-
-	if !dialCheck {
-		return ai, nil
+		return "", "", fmt.Errorf("could not get DialArgs: %w", err)
 	}
 
 	fmt.Printf("Checking miner api version of %s\n", addr)
 	api, closer, err := client.NewStorageMinerRPCV0(ctx, addr, info.AuthHeader())
 	if err != nil {
-		return "", err
+		return "", "", err
 	}
 	defer closer()
 
 	v, err := api.Version(ctx)
 	if err != nil {
-		return "", fmt.Errorf("checking version: %w", err)
+		return "", "", fmt.Errorf("checking version: %w", err)
 	}
 
 	if !v.APIVersion.EqMajorMinor(lapi.MinerAPIVersion0) {
-		return "", fmt.Errorf("remote service API version didn't match (expected %s, remote %s)", lapi.MinerAPIVersion0, v.APIVersion)
+		return "", "", fmt.Errorf("remote service API version didn't match (expected %s, remote %s)", lapi.MinerAPIVersion0, v.APIVersion)
+	}
+
+	miner, err := api.ActorAddress(ctx)
+	if err != nil {
+		return "", "", fmt.Errorf("getting miner address: %w", err)
 	}
 
-	return ai, nil
+	return ai, miner.String(), nil
 }

cmd/boostd/recover.go

@@ -25,7 +25,7 @@ import (
 	"github.com/filecoin-project/go-commp-utils/writer"
 	"github.com/filecoin-project/go-jsonrpc"
 	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/builtin/v9/miner"
+	"github.com/filecoin-project/go-state-types/builtin/v13/miner"
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/api/v1api"
 	"github.com/filecoin-project/lotus/chain/types"

cmd/booster-bitswap/run.go

@@ -194,7 +194,7 @@ var runCmd = &cli.Command{
 		}
 
 		// Connect to the storage API(s) and create a piece reader
-		sa := lib.NewMultiMinerAccessor(cctx.StringSlice("api-storage"), fullnodeApi)
+		sa := lib.NewMultiMinerAccessor(cctx.StringSlice("api-storage"), fullnodeApi, time.Second*60)
 		err = sa.Start(ctx, log)
 		if err != nil {
 			return err

cmd/booster-http/run.go

@@ -246,7 +246,7 @@ var runCmd = &cli.Command{
 		}
 
 		// Connect to the storage API(s) and create a piece reader
-		sa := lib.NewMultiMinerAccessor(cctx.StringSlice("api-storage"), fullnodeApi)
+		sa := lib.NewMultiMinerAccessor(cctx.StringSlice("api-storage"), fullnodeApi, 60*time.Second)
 		err = sa.Start(ctx, log)
 		if err != nil {
 			return err

cmd/lib/api.go

@@ -106,12 +106,14 @@ type MultiMinerAccessor struct {
 	sas             map[address.Address]dagstore.SectorAccessor
 	closeOnce       sync.Once
 	closers         []jsonrpc.ClientCloser
+	cachingDuration time.Duration
 }
 
-func NewMultiMinerAccessor(storageApiInfos []string, fullnodeApi v1api.FullNode) *MultiMinerAccessor {
+func NewMultiMinerAccessor(storageApiInfos []string, fullnodeApi v1api.FullNode, cacheTTL time.Duration) *MultiMinerAccessor {
 	return &MultiMinerAccessor{
 		storageApiInfos: storageApiInfos,
 		fullnodeApi:     fullnodeApi,
+		cachingDuration: cacheTTL,
 	}
 }
 
@@ -226,7 +228,7 @@ func CreateSectorAccessor(ctx context.Context, storageApiInfo string, fullnodeAp
 	// Create the piece provider
 	pp := sealer.NewPieceProvider(storage, storageService, storageService)
 	const maxCacheSize = 4096
-	newSectorAccessor := sectoraccessor.NewCachingSectorAccessor(maxCacheSize, 5*time.Minute)
+	newSectorAccessor := sectoraccessor.NewCachingSectorAccessor(maxCacheSize, 10*time.Second)
 	sa := newSectorAccessor(dtypes.MinerAddress(maddr), storageService, pp, fullnodeApi)
 	return &sectorAccessor{SectorAccessor: sa, maddr: maddr}, storageCloser, nil
 }

go.mod

@@ -27,10 +27,10 @@ require (
 	github.com/filecoin-project/go-commp-utils v0.1.4
 	github.com/filecoin-project/go-fil-commcid v0.1.0
 	github.com/filecoin-project/go-fil-commp-hashhash v0.2.0
-	github.com/filecoin-project/go-jsonrpc v0.3.1
+	github.com/filecoin-project/go-jsonrpc v0.3.2
 	github.com/filecoin-project/go-padreader v0.0.1
 	github.com/filecoin-project/go-paramfetch v0.0.4
-	github.com/filecoin-project/go-state-types v0.13.3
+	github.com/filecoin-project/go-state-types v0.14.0-dev
 	github.com/filecoin-project/go-statestore v0.2.0
 	github.com/filecoin-project/specs-actors v0.9.15
 	github.com/gbrlsnchs/jwt/v3 v3.0.1
@@ -95,7 +95,7 @@ require (
 	go.uber.org/atomic v1.11.0
 	go.uber.org/fx v1.20.1
 	go.uber.org/multierr v1.11.0
-	golang.org/x/crypto v0.19.0
+	golang.org/x/crypto v0.20.0
 	golang.org/x/exp v0.0.0-20240213143201-ec583247a57a
 	golang.org/x/sync v0.6.0
 	golang.org/x/text v0.14.0
@@ -197,7 +197,6 @@ require (
 	github.com/ipfs/go-ds-badger2 v0.1.3 // indirect
 	github.com/ipfs/go-ds-measure v0.2.0 // indirect
 	github.com/ipfs/go-fs-lock v0.0.7 // indirect
-	github.com/ipfs/go-ipfs-cmds v0.10.0 // indirect
 	github.com/ipfs/go-ipfs-delay v0.0.1 // indirect
 	github.com/ipfs/go-ipfs-pq v0.0.3 // indirect
 	github.com/ipfs/go-ipfs-util v0.0.3 // indirect
@@ -231,13 +230,13 @@ require (
 	github.com/libp2p/go-netroute v0.2.1 // indirect
 	github.com/libp2p/go-reuseport v0.4.0 // indirect
 	github.com/libp2p/go-yamux/v4 v4.0.1 // indirect
-	github.com/lucasb-eyer/go-colorful v1.0.3 // indirect
+	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/magefile/mage v1.9.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/mattn/go-runewidth v0.0.14 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/miekg/dns v1.1.58 // indirect
 	github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
 	github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
@@ -262,7 +261,7 @@ require (
 	github.com/prometheus/common v0.47.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/prometheus/statsd_exporter v0.22.7 // indirect
-	github.com/rivo/uniseg v0.4.4 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/rs/cors v1.7.0
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect
@@ -278,11 +277,10 @@ require (
 	github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11
 	github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
 	github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect
-	github.com/whyrusleeping/ledger-filecoin-go v0.9.1-0.20201010031517-c3dcc1bddce4 // indirect
 	github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
-	github.com/zondax/hid v0.9.1 // indirect
-	github.com/zondax/ledger-go v0.12.1 // indirect
+	github.com/zondax/hid v0.9.2 // indirect
+	github.com/zondax/ledger-go v0.14.3 // indirect
 	go.uber.org/dig v1.17.1 // indirect
 	go.uber.org/zap v1.27.0
 	go4.org v0.0.0-20230225012048-214862532bf5 // indirect
@@ -308,7 +306,7 @@ require (
 	github.com/filecoin-project/boost-graphsync v0.13.10
 	github.com/filecoin-project/boost/extern/boostd-data v0.0.0-20231124125934-3233c510357f
 	github.com/filecoin-project/go-data-segment v0.0.1
-	github.com/filecoin-project/lotus v1.26.3
+	github.com/filecoin-project/lotus v1.27.0-rc1.0.20240515154040-e14a85bcd32f
 	github.com/ipfs/boxo v0.18.0
 	github.com/ipfs/go-ds-leveldb v0.5.0
 	github.com/ipfs/kubo v0.22.0
@@ -325,42 +323,35 @@ require (
 
 require (
 	github.com/Jorropo/jsync v1.0.1 // indirect
+	github.com/KarpelesLab/reflink v1.0.1 // indirect
 	github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect
-	github.com/armon/go-metrics v0.3.9 // indirect
-	github.com/blang/semver/v4 v4.0.0 // indirect
-	github.com/boltdb/bolt v1.3.1 // indirect
+	github.com/bahlo/generic-list-go v0.2.0 // indirect
+	github.com/buger/jsonparser v1.1.1 // indirect
 	github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc7 // indirect
 	github.com/filecoin-project/go-fil-markets v1.28.3 // indirect
-	github.com/filecoin-project/kubo-api-client v0.27.0 // indirect
 	github.com/gammazero/channelqueue v0.2.1 // indirect
 	github.com/gammazero/deque v0.2.1 // indirect
 	github.com/georgysavva/scany/v2 v2.0.0 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5 // indirect
 	github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
-	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
-	github.com/hashicorp/go-msgpack v0.5.5 // indirect
 	github.com/hashicorp/go-retryablehttp v0.7.4 // indirect
 	github.com/hashicorp/golang-lru/arc/v2 v2.0.5 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
-	github.com/hashicorp/raft v1.3.10 // indirect
-	github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea // indirect
+	github.com/invopop/jsonschema v0.12.0 // indirect
 	github.com/ipfs/go-ipfs-blockstore v1.3.0 // indirect
 	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
 	github.com/jackc/pgconn v1.11.0 // indirect
 	github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa // indirect
 	github.com/jackc/pgio v1.0.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgproto3/v2 v2.2.0 // indirect
-	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
+	github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect
 	github.com/jackc/pgtype v1.10.0 // indirect
-	github.com/jackc/pgx/v5 v5.4.1 // indirect
 	github.com/jackc/puddle v1.2.1 // indirect
-	github.com/jackc/puddle/v2 v2.2.0 // indirect
+	github.com/jackc/puddle/v2 v2.2.1 // indirect
 	github.com/lib/pq v1.10.9 // indirect
-	github.com/libp2p/go-libp2p-consensus v0.0.1 // indirect
-	github.com/libp2p/go-libp2p-gorpc v0.6.0 // indirect
-	github.com/libp2p/go-libp2p-raft v0.4.0 // indirect
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/montanaflynn/stats v0.7.0 // indirect
 	github.com/onsi/ginkgo/v2 v2.15.0 // indirect
@@ -369,11 +360,13 @@ require (
 	github.com/quic-go/quic-go v0.42.0 // indirect
 	github.com/quic-go/webtransport-go v0.6.0 // indirect
 	github.com/samber/lo v1.39.0 // indirect
-	github.com/triplewz/poseidon v0.0.0-20220525065023-a7cdb0e183e7 // indirect
-	github.com/ugorji/go/codec v1.2.11 // indirect
+	github.com/triplewz/poseidon v0.0.0-20230828015038-79d8165c88ed // indirect
 	github.com/warpfork/go-testmark v0.12.1 // indirect
+	github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
 	github.com/yugabyte/gocql v0.0.0-20230831121436-1e2272bb6bb6 // indirect
 	github.com/yugabyte/pgx/v4 v4.14.5 // indirect
+	github.com/yugabyte/pgx/v5 v5.5.3-yb-2 // indirect
+	github.com/zondax/ledger-filecoin-go v0.11.1 // indirect
 	github.com/zyedidia/generic v1.2.1 // indirect
 	go.opentelemetry.io/otel/metric v1.21.0 // indirect
 	go.uber.org/mock v0.4.0 // indirect
@@ -383,3 +376,5 @@ require (
 )
 
 replace github.com/ugorji/go => github.com/ugorji/go v1.2.11
+
+replace github.com/KarpelesLab/reflink => github.com/magik6k/reflink v1.0.2-patch1