v3 H/AMT integration and migrations (#1328)

Co-authored-by: ZenGround0 <ZenGround0@users.noreply.github.com>
Co-authored-by: anorth <445306+anorth@users.noreply.github.com>

actors/builtin/init/init_actor_state.go

@@ -18,13 +18,13 @@ type State struct {
 }
 
 func ConstructState(store adt.Store, networkName string) (*State, error) {
-	emptyMapCid, err := adt.MakeEmptyMap(store, builtin.DefaultHamtBitwidth).Root()
+	emptyAddressMapCid, err := adt.StoreEmptyMap(store, builtin.DefaultHamtBitwidth)
 	if err != nil {
 		return nil, xerrors.Errorf("failed to create empty map: %w", err)
 	}
 
 	return &State{
-		AddressMap:  emptyMapCid,
+		AddressMap:  emptyAddressMapCid,
 		NextID:      abi.ActorID(builtin.FirstNonSingletonActorId),
 		NetworkName: networkName,
 	}, nil

actors/builtin/market/market_state.go

@@ -61,27 +61,36 @@ type State struct {
 }
 
 func ConstructState(store adt.Store) (*State, error) {
-	emptyArrayCid, err := adt.MakeEmptyArray(store).Root()
+	emptyProposalsArrayCid, err := adt.StoreEmptyArray(store, ProposalsAmtBitwidth)
 	if err != nil {
 		return nil, xerrors.Errorf("failed to create empty array: %w", err)
 	}
-	emptyMapCid, err := adt.MakeEmptyMap(store, builtin.DefaultHamtBitwidth).Root()
+	emptyStatesArrayCid, err := adt.StoreEmptyArray(store, StatesAmtBitwidth)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to create empty states array: %w", err)
+	}
+
+	emptyPendingProposalsMapCid, err := adt.StoreEmptyMap(store, builtin.DefaultHamtBitwidth)
 	if err != nil {
 		return nil, xerrors.Errorf("failed to create empty map: %w", err)
 	}
-	emptyMSetCid, err := MakeEmptySetMultimap(store, builtin.DefaultHamtBitwidth).Root()
+	emptyDealOpsHamtCid, err := MakeEmptySetMultimap(store, builtin.DefaultHamtBitwidth).Root()
 	if err != nil {
 		return nil, xerrors.Errorf("failed to create empty multiset: %w", err)
 	}
+	emptyBalanceTableCid, err := adt.StoreEmptyMap(store, adt.BalanceTableBitwidth)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to create empty balance table: %w", err)
+	}
 
 	return &State{
-		Proposals:        emptyArrayCid,
-		States:           emptyArrayCid,
-		PendingProposals: emptyMapCid,
-		EscrowTable:      emptyMapCid,
-		LockedTable:      emptyMapCid,
+		Proposals:        emptyProposalsArrayCid,
+		States:           emptyStatesArrayCid,
+		PendingProposals: emptyPendingProposalsMapCid,
+		EscrowTable:      emptyBalanceTableCid,
+		LockedTable:      emptyBalanceTableCid,
 		NextID:           abi.DealID(0),
-		DealOpsByEpoch:   emptyMSetCid,
+		DealOpsByEpoch:   emptyDealOpsHamtCid,
 		LastCron:         abi.ChainEpoch(-1),
 
 		TotalClientLockedCollateral:   abi.NewTokenAmount(0),

actors/builtin/market/market_test.go

@@ -82,10 +82,16 @@ func TestMarketActor(t *testing.T) {
 
 		store := adt.AsStore(rt)
 
-		emptyMap, err := adt.MakeEmptyMap(store, builtin.DefaultHamtBitwidth).Root()
+		emptyBalanceTable, err := adt.StoreEmptyMap(store, adt.BalanceTableBitwidth)
 		assert.NoError(t, err)
 
-		emptyArray, err := adt.MakeEmptyArray(store).Root()
+		emptyMap, err := adt.StoreEmptyMap(store, builtin.DefaultHamtBitwidth)
+		assert.NoError(t, err)
+
+		emptyProposalsArrayCid, err := adt.StoreEmptyArray(store, market.ProposalsAmtBitwidth)
+		assert.NoError(t, err)
+
+		emptyStatesArrayCid, err := adt.StoreEmptyArray(store, market.StatesAmtBitwidth)
 		assert.NoError(t, err)
 
 		emptyMultiMap, err := market.MakeEmptySetMultimap(store, builtin.DefaultHamtBitwidth).Root()
@@ -94,10 +100,11 @@ func TestMarketActor(t *testing.T) {
 		var state market.State
 		rt.GetState(&state)
 
-		assert.Equal(t, emptyArray, state.Proposals)
-		assert.Equal(t, emptyArray, state.States)
-		assert.Equal(t, emptyMap, state.EscrowTable)
-		assert.Equal(t, emptyMap, state.LockedTable)
+		assert.Equal(t, emptyProposalsArrayCid, state.Proposals)
+		assert.Equal(t, emptyStatesArrayCid, state.States)
+		assert.Equal(t, emptyMap, state.PendingProposals)
+		assert.Equal(t, emptyBalanceTable, state.EscrowTable)
+		assert.Equal(t, emptyBalanceTable, state.LockedTable)
 		assert.Equal(t, abi.DealID(0), state.NextID)
 		assert.Equal(t, emptyMultiMap, state.DealOpsByEpoch)
 		assert.Equal(t, abi.ChainEpoch(-1), state.LastCron)

actors/builtin/market/policy.go

@@ -26,6 +26,14 @@ var DealMaxDuration = abi.ChainEpoch(540 * builtin.EpochsInDay) // PARAM_SPEC
 // DealMaxLabelSize is the maximum size of a deal label.
 const DealMaxLabelSize = 256
 
+// Bitwidth of proposals AMT determined empirically from mutation patterns and
+// projections of mainnet data.
+const ProposalsAmtBitwidth = 5
+
+// Bitwidth of states AMT determined empirically from mutation patterns and
+// projections of mainnet data.
+const StatesAmtBitwidth = 6
+
 // Bounds (inclusive) on deal duration
 func DealDurationBounds(_ abi.PaddedPieceSize) (min abi.ChainEpoch, max abi.ChainEpoch) {
 	return DealMinDuration, DealMaxDuration

actors/builtin/market/set_multimap.go

@@ -3,7 +3,7 @@ package market
 import (
 	"reflect"
 
-	"github.com/filecoin-project/go-hamt-ipld/v2"
+	"github.com/filecoin-project/go-hamt-ipld/v3"
 	"github.com/filecoin-project/go-state-types/abi"
 	cid "github.com/ipfs/go-cid"
 	"github.com/pkg/errors"

actors/builtin/market/testing.go

@@ -60,7 +60,7 @@ func CheckStateInvariants(st *State, store adt.Store, balance abi.TokenAmount, c
 	expectedDealOps := make(map[abi.DealID]struct{})
 	totalProposalCollateral := abi.NewTokenAmount(0)
 
-	if proposals, err := adt.AsArray(store, st.Proposals); err != nil {
+	if proposals, err := adt.AsArray(store, st.Proposals, ProposalsAmtBitwidth); err != nil {
 		acc.Addf("error loading proposals: %v", err)
 	} else {
 		var proposal DealProposal
@@ -105,7 +105,7 @@ func CheckStateInvariants(st *State, store adt.Store, balance abi.TokenAmount, c
 	//
 
 	dealStateCount := uint64(0)
-	if dealStates, err := adt.AsArray(store, st.States); err != nil {
+	if dealStates, err := adt.AsArray(store, st.States, StatesAmtBitwidth); err != nil {
 		acc.Addf("error loading deal states: %v", err)
 	} else {
 		var dealState DealState

actors/builtin/market/types.go

@@ -15,7 +15,7 @@ type DealArray struct {
 
 // Interprets a store as balance table with root `r`.
 func AsDealProposalArray(s Store, r cid.Cid) (*DealArray, error) {
-	a, err := AsArray(s, r)
+	a, err := AsArray(s, r, ProposalsAmtBitwidth)
 	if err != nil {
 		return nil, err
 	}
@@ -56,7 +56,7 @@ type DealState struct {
 
 // Interprets a store as balance table with root `r`.
 func AsDealStateArray(s Store, r cid.Cid) (*DealMetaArray, error) {
-	dsa, err := AsArray(s, r)
+	dsa, err := AsArray(s, r, StatesAmtBitwidth)
 	if err != nil {
 		return nil, err
 	}

actors/builtin/miner/bitfield_queue.go

@@ -19,8 +19,8 @@ type BitfieldQueue struct {
 	quant QuantSpec
 }
 
-func LoadBitfieldQueue(store adt.Store, root cid.Cid, quant QuantSpec) (BitfieldQueue, error) {
-	arr, err := adt.AsArray(store, root)
+func LoadBitfieldQueue(store adt.Store, root cid.Cid, quant QuantSpec, bitwidth int) (BitfieldQueue, error) {
+	arr, err := adt.AsArray(store, root, bitwidth)
 	if err != nil {
 		return BitfieldQueue{}, xerrors.Errorf("failed to load epoch queue %v: %w", root, err)
 	}

actors/builtin/miner/bitfield_queue_test.go

@@ -15,9 +15,11 @@ import (
 	"github.com/filecoin-project/specs-actors/v3/support/mock"
 )
 
+const testAmtBitwidth = 3
+
 func TestBitfieldQueue(t *testing.T) {
 	t.Run("adds values to empty queue", func(t *testing.T) {
-		queue := emptyBitfieldQueue(t)
+		queue := emptyBitfieldQueue(t, testAmtBitwidth)
 
 		values := []uint64{1, 2, 3, 4}
 		epoch := abi.ChainEpoch(42)
@@ -29,7 +31,7 @@ func TestBitfieldQueue(t *testing.T) {
 	})
 
 	t.Run("adds bitfield to empty queue", func(t *testing.T) {
-		queue := emptyBitfieldQueue(t)
+		queue := emptyBitfieldQueue(t, testAmtBitwidth)
 
 		values := []uint64{1, 2, 3, 4}
 		epoch := abi.ChainEpoch(42)
@@ -42,7 +44,7 @@ func TestBitfieldQueue(t *testing.T) {
 	})
 
 	t.Run("quantizes added epochs according to quantization spec", func(t *testing.T) {
-		queue := emptyBitfieldQueueWithQuantizing(t, miner.NewQuantSpec(5, 3))
+		queue := emptyBitfieldQueueWithQuantizing(t, miner.NewQuantSpec(5, 3), testAmtBitwidth)
 
 		for _, val := range []uint64{0, 2, 3, 4, 7, 8, 9} {
 			require.NoError(t, queue.AddToQueueValues(abi.ChainEpoch(val), val))
@@ -57,7 +59,7 @@ func TestBitfieldQueue(t *testing.T) {
 	})
 
 	t.Run("quantizes added epochs according to quantization spec", func(t *testing.T) {
-		queue := emptyBitfieldQueueWithQuantizing(t, miner.NewQuantSpec(5, 3))
+		queue := emptyBitfieldQueueWithQuantizing(t, miner.NewQuantSpec(5, 3), testAmtBitwidth)
 
 		for _, val := range []uint64{0, 2, 3, 4, 7, 8, 9} {
 			err := queue.AddToQueueValues(abi.ChainEpoch(val), val)
@@ -73,7 +75,7 @@ func TestBitfieldQueue(t *testing.T) {
 	})
 
 	t.Run("merges values withing same epoch", func(t *testing.T) {
-		queue := emptyBitfieldQueue(t)
+		queue := emptyBitfieldQueue(t, testAmtBitwidth)
 
 		epoch := abi.ChainEpoch(42)
 
@@ -86,7 +88,7 @@ func TestBitfieldQueue(t *testing.T) {
 	})
 
 	t.Run("adds values to different epochs", func(t *testing.T) {
-		queue := emptyBitfieldQueue(t)
+		queue := emptyBitfieldQueue(t, testAmtBitwidth)
 
 		epoch1 := abi.ChainEpoch(42)
 		epoch2 := abi.ChainEpoch(93)
@@ -101,7 +103,7 @@ func TestBitfieldQueue(t *testing.T) {
 	})
 
 	t.Run("PouUntil from empty queue returns empty bitfield", func(t *testing.T) {
-		queue := emptyBitfieldQueue(t)
+		queue := emptyBitfieldQueue(t, testAmtBitwidth)
 
 		// TODO: broken pending https://github.com/filecoin-project/go-amt-ipld/issues/18
 		//emptyQueue, err := queue.Root()
@@ -122,7 +124,7 @@ func TestBitfieldQueue(t *testing.T) {
 	})
 
 	t.Run("PopUntil does nothing if 'until' parameter before first value", func(t *testing.T) {
-		queue := emptyBitfieldQueue(t)
+		queue := emptyBitfieldQueue(t, testAmtBitwidth)
 
 		epoch1 := abi.ChainEpoch(42)
 		epoch2 := abi.ChainEpoch(93)
@@ -148,7 +150,7 @@ func TestBitfieldQueue(t *testing.T) {
 	})
 
 	t.Run("PopUntil removes and returns entries before and including target epoch", func(t *testing.T) {
-		queue := emptyBitfieldQueue(t)
+		queue := emptyBitfieldQueue(t, testAmtBitwidth)
 
 		epoch1 := abi.ChainEpoch(42)
 		epoch2 := abi.ChainEpoch(93)
@@ -206,7 +208,7 @@ func TestBitfieldQueue(t *testing.T) {
 	})
 
 	t.Run("cuts elements", func(t *testing.T) {
-		queue := emptyBitfieldQueue(t)
+		queue := emptyBitfieldQueue(t, testAmtBitwidth)
 
 		epoch1 := abi.ChainEpoch(42)
 		epoch2 := abi.ChainEpoch(93)
@@ -223,19 +225,19 @@ func TestBitfieldQueue(t *testing.T) {
 
 }
 
-func emptyBitfieldQueueWithQuantizing(t *testing.T, quant miner.QuantSpec) miner.BitfieldQueue {
+func emptyBitfieldQueueWithQuantizing(t *testing.T, quant miner.QuantSpec, bitwidth int) miner.BitfieldQueue {
 	rt := mock.NewBuilder(context.Background(), address.Undef).Build(t)
 	store := adt.AsStore(rt)
-	root, err := adt.MakeEmptyArray(store).Root()
+	emptyArray, err := adt.StoreEmptyArray(store, bitwidth)
 	require.NoError(t, err)
 
-	queue, err := miner.LoadBitfieldQueue(store, root, quant)
+	queue, err := miner.LoadBitfieldQueue(store, emptyArray, quant, bitwidth)
 	require.NoError(t, err)
 	return queue
 }
 
-func emptyBitfieldQueue(t *testing.T) miner.BitfieldQueue {
-	return emptyBitfieldQueueWithQuantizing(t, miner.NoQuantization)
+func emptyBitfieldQueue(t *testing.T, bitwidth int) miner.BitfieldQueue {
+	return emptyBitfieldQueueWithQuantizing(t, miner.NoQuantization, bitwidth)
 }
 
 type bqExpectation struct {

actors/builtin/miner/deadline_state.go

@@ -56,6 +56,9 @@ type Deadline struct {
 	FaultyPower PowerPair
 }
 
+const DeadlinePartitionsAmtBitwidth = 3
+const DeadlineExpirationAmtBitwidth = 5
+
 //
 // Deadlines (plural)
 //
@@ -116,20 +119,29 @@ func (d *Deadlines) UpdateDeadline(store adt.Store, dlIdx uint64, deadline *Dead
 // Deadline (singular)
 //
 
-func ConstructDeadline(emptyArrayCid cid.Cid) *Deadline {
+func ConstructDeadline(store adt.Store) (*Deadline, error) {
+	emptyPartitionsArrayCid, err := adt.StoreEmptyArray(store, DeadlinePartitionsAmtBitwidth)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to construct empty partitions array: %w", err)
+	}
+	emptyDeadlineExpirationArrayCid, err := adt.StoreEmptyArray(store, DeadlineExpirationAmtBitwidth)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to construct empty deadline expiration array: %w", err)
+	}
+
 	return &Deadline{
-		Partitions:        emptyArrayCid,
-		ExpirationsEpochs: emptyArrayCid,
+		Partitions:        emptyPartitionsArrayCid,
+		ExpirationsEpochs: emptyDeadlineExpirationArrayCid,
 		PostSubmissions:   bitfield.New(),
 		EarlyTerminations: bitfield.New(),
 		LiveSectors:       0,
 		TotalSectors:      0,
 		FaultyPower:       NewPowerPairZero(),
-	}
+	}, nil
 }
 
 func (d *Deadline) PartitionsArray(store adt.Store) (*adt.Array, error) {
-	arr, err := adt.AsArray(store, d.Partitions)
+	arr, err := adt.AsArray(store, d.Partitions, DeadlinePartitionsAmtBitwidth)
 	if err != nil {
 		return nil, xc.ErrIllegalState.Wrapf("failed to load partitions: %w", err)
 	}
@@ -159,7 +171,7 @@ func (d *Deadline) AddExpirationPartitions(store adt.Store, expirationEpoch abi.
 		return nil
 	}
 
-	queue, err := LoadBitfieldQueue(store, d.ExpirationsEpochs, quant)
+	queue, err := LoadBitfieldQueue(store, d.ExpirationsEpochs, quant, DeadlineExpirationAmtBitwidth)
 	if err != nil {
 		return xerrors.Errorf("failed to load expiration queue: %w", err)
 	}
@@ -296,13 +308,11 @@ func (dl *Deadline) AddSectors(
 				return NewPowerPairZero(), err
 			} else if !found {
 				// This case will usually happen zero times.
-				// It would require adding more than a full partition in one go
-				// to happen more than once.
-				emptyArray, err := adt.MakeEmptyArray(store).Root()
+				// It would require adding more than a full partition in one go to happen more than once.
+				partition, err = ConstructPartition(store)
 				if err != nil {
 					return NewPowerPairZero(), err
 				}
-				partition = ConstructPartition(emptyArray)
 			}
 
 			// Figure out which (if any) sectors we want to add to this partition.
@@ -351,7 +361,7 @@ func (dl *Deadline) AddSectors(
 
 	// Next, update the expiration queue.
 	{
-		deadlineExpirations, err := LoadBitfieldQueue(store, dl.ExpirationsEpochs, quant)
+		deadlineExpirations, err := LoadBitfieldQueue(store, dl.ExpirationsEpochs, quant, DeadlineExpirationAmtBitwidth)
 		if err != nil {
 			return NewPowerPairZero(), xerrors.Errorf("failed to load expiration epochs: %w", err)
 		}
@@ -448,7 +458,7 @@ func (dl *Deadline) PopEarlyTerminations(store adt.Store, maxPartitions, maxSect
 
 // Returns nil if nothing was popped.
 func (dl *Deadline) popExpiredPartitions(store adt.Store, until abi.ChainEpoch, quant QuantSpec) (bitfield.BitField, bool, error) {
-	expirations, err := LoadBitfieldQueue(store, dl.ExpirationsEpochs, quant)
+	expirations, err := LoadBitfieldQueue(store, dl.ExpirationsEpochs, quant, DeadlineExpirationAmtBitwidth)
 	if err != nil {
 		return bitfield.BitField{}, false, err
 	}
@@ -569,7 +579,10 @@ func (dl *Deadline) RemovePartitions(store adt.Store, toRemove bitfield.BitField
 		return bitfield.BitField{}, bitfield.BitField{}, NewPowerPairZero(), xerrors.Errorf("cannot remove partitions from deadline with early terminations: %w", err)
 	}
 
-	newPartitions := adt.MakeEmptyArray(store)
+	newPartitions, err := adt.MakeEmptyArray(store, DeadlinePartitionsAmtBitwidth)
+	if err != nil {
+		return bitfield.BitField{}, bitfield.BitField{}, NewPowerPairZero(), xerrors.Errorf("failed to create empty array for initializing partitions: %w", err)
+	}
 	allDeadSectors := make([]bitfield.BitField, 0, len(toRemoveSet))
 	allLiveSectors := make([]bitfield.BitField, 0, len(toRemoveSet))
 	removedPower = NewPowerPairZero()
@@ -656,7 +669,7 @@ func (dl *Deadline) RemovePartitions(store adt.Store, toRemove bitfield.BitField
 
 	// Update expiration bitfields.
 	{
-		expirationEpochs, err := LoadBitfieldQueue(store, dl.ExpirationsEpochs, quant)
+		expirationEpochs, err := LoadBitfieldQueue(store, dl.ExpirationsEpochs, quant, DeadlineExpirationAmtBitwidth)
 		if err != nil {
 			return bitfield.BitField{}, bitfield.BitField{}, NewPowerPairZero(), xerrors.Errorf("failed to load expiration queue: %w", err)
 		}