From ca9c60294ba4420fae8307b271a2e28bbe84d334 Mon Sep 17 00:00:00 2001 From: ZenGround0 Date: Mon, 14 Dec 2020 11:58:50 -0500 Subject: [PATCH] Instate bitwidth tuning parameters --- actors/builtin/market/market_state.go | 25 +++++++--- actors/builtin/market/market_test.go | 24 +++++++--- actors/builtin/market/policy.go | 8 ++++ actors/builtin/market/testing.go | 4 +- actors/builtin/market/types.go | 5 +- actors/builtin/miner/bitfield_queue.go | 5 +- actors/builtin/miner/bitfield_queue_test.go | 30 ++++++------ actors/builtin/miner/deadline_state.go | 31 +++++++++---- actors/builtin/miner/deadline_state_test.go | 11 +++-- actors/builtin/miner/expiration_queue.go | 5 +- actors/builtin/miner/expiration_queue_test.go | 30 ++++++------ actors/builtin/miner/miner_state.go | 46 +++++++++++++++---- actors/builtin/miner/miner_state_test.go | 2 +- actors/builtin/miner/miner_test.go | 6 +-- actors/builtin/miner/partition_state.go | 31 +++++++------ actors/builtin/miner/partition_state_test.go | 21 +++++---- actors/builtin/miner/sectors.go | 3 +- actors/builtin/miner/testing.go | 10 ++-- actors/builtin/power/policy.go | 12 +++++ actors/builtin/power/power_actor.go | 10 ++-- actors/builtin/power/power_state.go | 4 +- actors/builtin/power/power_test.go | 6 +-- actors/builtin/power/testing.go | 4 +- actors/migration/nv9/market.go | 10 ++-- actors/migration/nv9/miner.go | 9 ++-- actors/migration/nv9/power.go | 9 +++- actors/util/adt/balancetable.go | 8 ++-- 27 files changed, 236 insertions(+), 133 deletions(-) diff --git a/actors/builtin/market/market_state.go b/actors/builtin/market/market_state.go index 6787ba4213..8803224e93 100644 --- a/actors/builtin/market/market_state.go +++ b/actors/builtin/market/market_state.go @@ -61,14 +61,23 @@ type State struct { } func ConstructState(store adt.Store) (*State, error) { - emptyArray, err := adt.MakeEmptyArray(store, builtin.DefaultAmtBitwidth) + emptyProposalsArray, err := adt.MakeEmptyArray(store, ProposalsAmtBitwidth) if err != nil { return nil, xerrors.Errorf("failed to create empty array: %w", err) } - emptyArrayCid, err := emptyArray.Root() + emptyProposalsArrayCid, err := emptyProposalsArray.Root() if err != nil { return nil, xerrors.Errorf("failed to persist empty array: %w", err) } + emptyStatesArray, err := adt.MakeEmptyArray(store, StatesAmtBitwidth) + if err != nil { + return nil, xerrors.Errorf("failed to create empty states array: %w", err) + } + emptyStatesArrayCid, err := emptyStatesArray.Root() + if err != nil { + return nil, xerrors.Errorf("failed to persist empty states array: %w", err) + } + emptyMapCid, err := adt.MakeEmptyMap(store, builtin.DefaultHamtBitwidth).Root() if err != nil { return nil, xerrors.Errorf("failed to create empty map: %w", err) @@ -77,13 +86,17 @@ func ConstructState(store adt.Store) (*State, error) { if err != nil { return nil, xerrors.Errorf("failed to create empty multiset: %w", err) } + emptyBalanceTableCid, err := adt.MakeEmptyMap(store, adt.BalanceTableBitwidth).Root() + if err != nil { + return nil, xerrors.Errorf("failed to create empty balance table: %w", err) + } return &State{ - Proposals: emptyArrayCid, - States: emptyArrayCid, + Proposals: emptyProposalsArrayCid, + States: emptyStatesArrayCid, PendingProposals: emptyMapCid, - EscrowTable: emptyMapCid, - LockedTable: emptyMapCid, + EscrowTable: emptyBalanceTableCid, + LockedTable: emptyBalanceTableCid, NextID: abi.DealID(0), DealOpsByEpoch: emptyMSetCid, LastCron: abi.ChainEpoch(-1), diff --git a/actors/builtin/market/market_test.go b/actors/builtin/market/market_test.go index e6d7d4b0aa..f4dc1c34fb 100644 --- a/actors/builtin/market/market_test.go +++ b/actors/builtin/market/market_test.go @@ -82,13 +82,22 @@ func TestMarketActor(t *testing.T) { store := adt.AsStore(rt) + emptyBalanceTable, err := adt.MakeEmptyMap(store, adt.BalanceTableBitwidth).Root() + assert.NoError(t, err) + emptyMap, err := adt.MakeEmptyMap(store, builtin.DefaultHamtBitwidth).Root() assert.NoError(t, err) - emptyArray, err := adt.MakeEmptyArray(store, builtin.DefaultAmtBitwidth) - require.NoError(t, err) + emptyProposalsArray, err := adt.MakeEmptyArray(store, market.ProposalsAmtBitwidth) + assert.NoError(t, err) + + emptyProposalsArrayCid, err := emptyProposalsArray.Root() + assert.NoError(t, err) + + emptyStatesArray, err := adt.MakeEmptyArray(store, market.StatesAmtBitwidth) + assert.NoError(t, err) - emptyArrayRoot, err := emptyArray.Root() + emptyStatesArrayCid, err := emptyStatesArray.Root() assert.NoError(t, err) emptyMultiMap, err := market.MakeEmptySetMultimap(store, builtin.DefaultHamtBitwidth).Root() @@ -97,10 +106,11 @@ func TestMarketActor(t *testing.T) { var state market.State rt.GetState(&state) - assert.Equal(t, emptyArrayRoot, state.Proposals) - assert.Equal(t, emptyArrayRoot, state.States) - assert.Equal(t, emptyMap, state.EscrowTable) - assert.Equal(t, emptyMap, state.LockedTable) + assert.Equal(t, emptyProposalsArrayCid, state.Proposals) + assert.Equal(t, emptyStatesArrayCid, state.States) + assert.Equal(t, emptyMap, state.PendingProposals) + assert.Equal(t, emptyBalanceTable, state.EscrowTable) + assert.Equal(t, emptyBalanceTable, state.LockedTable) assert.Equal(t, abi.DealID(0), state.NextID) assert.Equal(t, emptyMultiMap, state.DealOpsByEpoch) assert.Equal(t, abi.ChainEpoch(-1), state.LastCron) diff --git a/actors/builtin/market/policy.go b/actors/builtin/market/policy.go index c00f6554f3..f953949d0b 100644 --- a/actors/builtin/market/policy.go +++ b/actors/builtin/market/policy.go @@ -26,6 +26,14 @@ var DealMaxDuration = abi.ChainEpoch(540 * builtin.EpochsInDay) // PARAM_SPEC // DealMaxLabelSize is the maximum size of a deal label. const DealMaxLabelSize = 256 +// Bitwidth of proposals AMT determined empirically from mutation patterns and +// projections of mainnet data. +const ProposalsAmtBitwidth = 5 + +// Bitwidth of states AMT determined empirically from mutation patterns and +// projections of mainnet data. +const StatesAmtBitwidth = 6 + // Bounds (inclusive) on deal duration func DealDurationBounds(_ abi.PaddedPieceSize) (min abi.ChainEpoch, max abi.ChainEpoch) { return DealMinDuration, DealMaxDuration diff --git a/actors/builtin/market/testing.go b/actors/builtin/market/testing.go index a65550c087..281e4d7e33 100644 --- a/actors/builtin/market/testing.go +++ b/actors/builtin/market/testing.go @@ -60,7 +60,7 @@ func CheckStateInvariants(st *State, store adt.Store, balance abi.TokenAmount, c expectedDealOps := make(map[abi.DealID]struct{}) totalProposalCollateral := abi.NewTokenAmount(0) - if proposals, err := adt.AsArray(store, st.Proposals, builtin.DefaultAmtBitwidth); err != nil { + if proposals, err := adt.AsArray(store, st.Proposals, ProposalsAmtBitwidth); err != nil { acc.Addf("error loading proposals: %v", err) } else { var proposal DealProposal @@ -105,7 +105,7 @@ func CheckStateInvariants(st *State, store adt.Store, balance abi.TokenAmount, c // dealStateCount := uint64(0) - if dealStates, err := adt.AsArray(store, st.States, builtin.DefaultAmtBitwidth); err != nil { + if dealStates, err := adt.AsArray(store, st.States, StatesAmtBitwidth); err != nil { acc.Addf("error loading deal states: %v", err) } else { var dealState DealState diff --git a/actors/builtin/market/types.go b/actors/builtin/market/types.go index 263e7033e5..e94d57058f 100644 --- a/actors/builtin/market/types.go +++ b/actors/builtin/market/types.go @@ -2,7 +2,6 @@ package market import ( "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/specs-actors/v3/actors/builtin" . "github.com/filecoin-project/specs-actors/v3/actors/util/adt" "github.com/ipfs/go-cid" @@ -16,7 +15,7 @@ type DealArray struct { // Interprets a store as balance table with root `r`. func AsDealProposalArray(s Store, r cid.Cid) (*DealArray, error) { - a, err := AsArray(s, r, builtin.DefaultAmtBitwidth) + a, err := AsArray(s, r, ProposalsAmtBitwidth) if err != nil { return nil, err } @@ -57,7 +56,7 @@ type DealState struct { // Interprets a store as balance table with root `r`. func AsDealStateArray(s Store, r cid.Cid) (*DealMetaArray, error) { - dsa, err := AsArray(s, r, builtin.DefaultAmtBitwidth) + dsa, err := AsArray(s, r, StatesAmtBitwidth) if err != nil { return nil, err } diff --git a/actors/builtin/miner/bitfield_queue.go b/actors/builtin/miner/bitfield_queue.go index 919286d803..5b96f7844b 100644 --- a/actors/builtin/miner/bitfield_queue.go +++ b/actors/builtin/miner/bitfield_queue.go @@ -9,7 +9,6 @@ import ( "github.com/ipfs/go-cid" "golang.org/x/xerrors" - "github.com/filecoin-project/specs-actors/v3/actors/builtin" "github.com/filecoin-project/specs-actors/v3/actors/util/adt" ) @@ -20,8 +19,8 @@ type BitfieldQueue struct { quant QuantSpec } -func LoadBitfieldQueue(store adt.Store, root cid.Cid, quant QuantSpec) (BitfieldQueue, error) { - arr, err := adt.AsArray(store, root, builtin.DefaultAmtBitwidth) +func LoadBitfieldQueue(store adt.Store, root cid.Cid, quant QuantSpec, bitwidth int) (BitfieldQueue, error) { + arr, err := adt.AsArray(store, root, bitwidth) if err != nil { return BitfieldQueue{}, xerrors.Errorf("failed to load epoch queue %v: %w", root, err) } diff --git a/actors/builtin/miner/bitfield_queue_test.go b/actors/builtin/miner/bitfield_queue_test.go index e8419ba8e4..de178cf8f1 100644 --- a/actors/builtin/miner/bitfield_queue_test.go +++ b/actors/builtin/miner/bitfield_queue_test.go @@ -18,7 +18,7 @@ import ( func TestBitfieldQueue(t *testing.T) { t.Run("adds values to empty queue", func(t *testing.T) { - queue := emptyBitfieldQueue(t) + queue := emptyBitfieldQueue(t, builtin.DefaultAmtBitwidth) values := []uint64{1, 2, 3, 4} epoch := abi.ChainEpoch(42) @@ -30,7 +30,7 @@ func TestBitfieldQueue(t *testing.T) { }) t.Run("adds bitfield to empty queue", func(t *testing.T) { - queue := emptyBitfieldQueue(t) + queue := emptyBitfieldQueue(t, builtin.DefaultAmtBitwidth) values := []uint64{1, 2, 3, 4} epoch := abi.ChainEpoch(42) @@ -43,7 +43,7 @@ func TestBitfieldQueue(t *testing.T) { }) t.Run("quantizes added epochs according to quantization spec", func(t *testing.T) { - queue := emptyBitfieldQueueWithQuantizing(t, miner.NewQuantSpec(5, 3)) + queue := emptyBitfieldQueueWithQuantizing(t, miner.NewQuantSpec(5, 3), builtin.DefaultAmtBitwidth) for _, val := range []uint64{0, 2, 3, 4, 7, 8, 9} { require.NoError(t, queue.AddToQueueValues(abi.ChainEpoch(val), val)) @@ -58,7 +58,7 @@ func TestBitfieldQueue(t *testing.T) { }) t.Run("quantizes added epochs according to quantization spec", func(t *testing.T) { - queue := emptyBitfieldQueueWithQuantizing(t, miner.NewQuantSpec(5, 3)) + queue := emptyBitfieldQueueWithQuantizing(t, miner.NewQuantSpec(5, 3), builtin.DefaultAmtBitwidth) for _, val := range []uint64{0, 2, 3, 4, 7, 8, 9} { err := queue.AddToQueueValues(abi.ChainEpoch(val), val) @@ -74,7 +74,7 @@ func TestBitfieldQueue(t *testing.T) { }) t.Run("merges values withing same epoch", func(t *testing.T) { - queue := emptyBitfieldQueue(t) + queue := emptyBitfieldQueue(t, builtin.DefaultAmtBitwidth) epoch := abi.ChainEpoch(42) @@ -87,7 +87,7 @@ func TestBitfieldQueue(t *testing.T) { }) t.Run("adds values to different epochs", func(t *testing.T) { - queue := emptyBitfieldQueue(t) + queue := emptyBitfieldQueue(t, builtin.DefaultAmtBitwidth) epoch1 := abi.ChainEpoch(42) epoch2 := abi.ChainEpoch(93) @@ -102,7 +102,7 @@ func TestBitfieldQueue(t *testing.T) { }) t.Run("PouUntil from empty queue returns empty bitfield", func(t *testing.T) { - queue := emptyBitfieldQueue(t) + queue := emptyBitfieldQueue(t, builtin.DefaultAmtBitwidth) // TODO: broken pending https://github.com/filecoin-project/go-amt-ipld/issues/18 //emptyQueue, err := queue.Root() @@ -123,7 +123,7 @@ func TestBitfieldQueue(t *testing.T) { }) t.Run("PopUntil does nothing if 'until' parameter before first value", func(t *testing.T) { - queue := emptyBitfieldQueue(t) + queue := emptyBitfieldQueue(t, builtin.DefaultAmtBitwidth) epoch1 := abi.ChainEpoch(42) epoch2 := abi.ChainEpoch(93) @@ -149,7 +149,7 @@ func TestBitfieldQueue(t *testing.T) { }) t.Run("PopUntil removes and returns entries before and including target epoch", func(t *testing.T) { - queue := emptyBitfieldQueue(t) + queue := emptyBitfieldQueue(t, builtin.DefaultAmtBitwidth) epoch1 := abi.ChainEpoch(42) epoch2 := abi.ChainEpoch(93) @@ -207,7 +207,7 @@ func TestBitfieldQueue(t *testing.T) { }) t.Run("cuts elements", func(t *testing.T) { - queue := emptyBitfieldQueue(t) + queue := emptyBitfieldQueue(t, builtin.DefaultAmtBitwidth) epoch1 := abi.ChainEpoch(42) epoch2 := abi.ChainEpoch(93) @@ -224,21 +224,21 @@ func TestBitfieldQueue(t *testing.T) { } -func emptyBitfieldQueueWithQuantizing(t *testing.T, quant miner.QuantSpec) miner.BitfieldQueue { +func emptyBitfieldQueueWithQuantizing(t *testing.T, quant miner.QuantSpec, bitwidth int) miner.BitfieldQueue { rt := mock.NewBuilder(context.Background(), address.Undef).Build(t) store := adt.AsStore(rt) - emptyArray, err := adt.MakeEmptyArray(store, builtin.DefaultAmtBitwidth) + emptyArray, err := adt.MakeEmptyArray(store, bitwidth) require.NoError(t, err) root, err := emptyArray.Root() require.NoError(t, err) - queue, err := miner.LoadBitfieldQueue(store, root, quant) + queue, err := miner.LoadBitfieldQueue(store, root, quant, bitwidth) require.NoError(t, err) return queue } -func emptyBitfieldQueue(t *testing.T) miner.BitfieldQueue { - return emptyBitfieldQueueWithQuantizing(t, miner.NoQuantization) +func emptyBitfieldQueue(t *testing.T, bitwidth int) miner.BitfieldQueue { + return emptyBitfieldQueueWithQuantizing(t, miner.NoQuantization, bitwidth) } type bqExpectation struct { diff --git a/actors/builtin/miner/deadline_state.go b/actors/builtin/miner/deadline_state.go index 3d258bdb36..3c950d259e 100644 --- a/actors/builtin/miner/deadline_state.go +++ b/actors/builtin/miner/deadline_state.go @@ -57,6 +57,8 @@ type Deadline struct { FaultyPower PowerPair } +const DeadlineExpirationAmtBitwidth = 5 + // // Deadlines (plural) // @@ -117,10 +119,10 @@ func (d *Deadlines) UpdateDeadline(store adt.Store, dlIdx uint64, deadline *Dead // Deadline (singular) // -func ConstructDeadline(emptyArrayCid cid.Cid) *Deadline { +func ConstructDeadline(emptyPartitionsArrayCid, emptyDeadlineExpirationCid cid.Cid) *Deadline { return &Deadline{ - Partitions: emptyArrayCid, - ExpirationsEpochs: emptyArrayCid, + Partitions: emptyPartitionsArrayCid, + ExpirationsEpochs: emptyDeadlineExpirationCid, PostSubmissions: bitfield.New(), EarlyTerminations: bitfield.New(), LiveSectors: 0, @@ -160,7 +162,7 @@ func (d *Deadline) AddExpirationPartitions(store adt.Store, expirationEpoch abi. return nil } - queue, err := LoadBitfieldQueue(store, d.ExpirationsEpochs, quant) + queue, err := LoadBitfieldQueue(store, d.ExpirationsEpochs, quant, DeadlineExpirationAmtBitwidth) if err != nil { return xerrors.Errorf("failed to load expiration queue: %w", err) } @@ -299,15 +301,24 @@ func (dl *Deadline) AddSectors( // This case will usually happen zero times. // It would require adding more than a full partition in one go // to happen more than once. - emptyArray, err := adt.MakeEmptyArray(store, builtin.DefaultAmtBitwidth) + emptyExpirationArray, err := adt.MakeEmptyArray(store, PartitionExpirationAmtBitwidth) + if err != nil { + return NewPowerPairZero(), err + } + emptyExpirationArrayRoot, err := emptyExpirationArray.Root() + if err != nil { + return NewPowerPairZero(), err + } + + emptyEarlyTerminationArray, err := adt.MakeEmptyArray(store, builtin.DefaultAmtBitwidth) if err != nil { return NewPowerPairZero(), err } - emptyArrayRoot, err := emptyArray.Root() + emptyEarlyTerminationArrayRoot, err := emptyEarlyTerminationArray.Root() if err != nil { return NewPowerPairZero(), err } - partition = ConstructPartition(emptyArrayRoot) + partition = ConstructPartition(emptyExpirationArrayRoot, emptyEarlyTerminationArrayRoot) } // Figure out which (if any) sectors we want to add to this partition. @@ -356,7 +367,7 @@ func (dl *Deadline) AddSectors( // Next, update the expiration queue. { - deadlineExpirations, err := LoadBitfieldQueue(store, dl.ExpirationsEpochs, quant) + deadlineExpirations, err := LoadBitfieldQueue(store, dl.ExpirationsEpochs, quant, DeadlineExpirationAmtBitwidth) if err != nil { return NewPowerPairZero(), xerrors.Errorf("failed to load expiration epochs: %w", err) } @@ -453,7 +464,7 @@ func (dl *Deadline) PopEarlyTerminations(store adt.Store, maxPartitions, maxSect // Returns nil if nothing was popped. func (dl *Deadline) popExpiredPartitions(store adt.Store, until abi.ChainEpoch, quant QuantSpec) (bitfield.BitField, bool, error) { - expirations, err := LoadBitfieldQueue(store, dl.ExpirationsEpochs, quant) + expirations, err := LoadBitfieldQueue(store, dl.ExpirationsEpochs, quant, DeadlineExpirationAmtBitwidth) if err != nil { return bitfield.BitField{}, false, err } @@ -664,7 +675,7 @@ func (dl *Deadline) RemovePartitions(store adt.Store, toRemove bitfield.BitField // Update expiration bitfields. { - expirationEpochs, err := LoadBitfieldQueue(store, dl.ExpirationsEpochs, quant) + expirationEpochs, err := LoadBitfieldQueue(store, dl.ExpirationsEpochs, quant, DeadlineExpirationAmtBitwidth) if err != nil { return bitfield.BitField{}, bitfield.BitField{}, NewPowerPairZero(), xerrors.Errorf("failed to load expiration queue: %w", err) } diff --git a/actors/builtin/miner/deadline_state_test.go b/actors/builtin/miner/deadline_state_test.go index e7a4aa252e..76c73857a4 100644 --- a/actors/builtin/miner/deadline_state_test.go +++ b/actors/builtin/miner/deadline_state_test.go @@ -876,12 +876,17 @@ func TestDeadlines(t *testing.T) { } func emptyDeadline(t *testing.T, store adt.Store) *miner.Deadline { - emptyArray, err := adt.MakeEmptyArray(store, builtin.DefaultAmtBitwidth) + emptyPartitionsArray, err := adt.MakeEmptyArray(store, builtin.DefaultAmtBitwidth) require.NoError(t, err) - root, err := emptyArray.Root() + partitionsRoot, err := emptyPartitionsArray.Root() require.NoError(t, err) - return miner.ConstructDeadline(root) + emptyDeadlineExpirationArray, err := adt.MakeEmptyArray(store, miner.DeadlineExpirationAmtBitwidth) + require.NoError(t, err) + expirationsRoot, err := emptyDeadlineExpirationArray.Root() + require.NoError(t, err) + + return miner.ConstructDeadline(partitionsRoot, expirationsRoot) } // Helper type for validating deadline state. diff --git a/actors/builtin/miner/expiration_queue.go b/actors/builtin/miner/expiration_queue.go index 9742efa01e..35118938c8 100644 --- a/actors/builtin/miner/expiration_queue.go +++ b/actors/builtin/miner/expiration_queue.go @@ -10,7 +10,6 @@ import ( "github.com/ipfs/go-cid" "golang.org/x/xerrors" - "github.com/filecoin-project/specs-actors/v3/actors/builtin" "github.com/filecoin-project/specs-actors/v3/actors/util" "github.com/filecoin-project/specs-actors/v3/actors/util/adt" ) @@ -163,8 +162,8 @@ const entrySectorsMax = 10_000 // Loads a queue root. // Epochs provided to subsequent method calls will be quantized upwards to quanta mod offsetSeed before being // written to/read from queue entries. -func LoadExpirationQueue(store adt.Store, root cid.Cid, quant QuantSpec) (ExpirationQueue, error) { - arr, err := adt.AsArray(store, root, builtin.DefaultAmtBitwidth) +func LoadExpirationQueue(store adt.Store, root cid.Cid, quant QuantSpec, bitwidth int) (ExpirationQueue, error) { + arr, err := adt.AsArray(store, root, bitwidth) if err != nil { return ExpirationQueue{}, xerrors.Errorf("failed to load epoch queue %v: %w", root, err) } diff --git a/actors/builtin/miner/expiration_queue_test.go b/actors/builtin/miner/expiration_queue_test.go index 7431300486..fb1d9e9ace 100644 --- a/actors/builtin/miner/expiration_queue_test.go +++ b/actors/builtin/miner/expiration_queue_test.go @@ -242,7 +242,7 @@ func TestExpirationQueue(t *testing.T) { }) t.Run("quantizes added sectors by expiration", func(t *testing.T) { - queue := emptyExpirationQueueWithQuantizing(t, miner.NewQuantSpec(5, 3)) + queue := emptyExpirationQueueWithQuantizing(t, miner.NewQuantSpec(5, 3), builtin.DefaultAmtBitwidth) secNums, power, pledge, err := queue.AddActiveSectors(sectors, sectorSize) require.NoError(t, err) assertBitfieldEquals(t, secNums, 1, 2, 3, 4, 5, 6) @@ -343,7 +343,7 @@ func TestExpirationQueue(t *testing.T) { t.Run("reschedules sectors as faults", func(t *testing.T) { // Create 3 expiration sets with 2 sectors apiece - queue := emptyExpirationQueueWithQuantizing(t, miner.NewQuantSpec(4, 1)) + queue := emptyExpirationQueueWithQuantizing(t, miner.NewQuantSpec(4, 1), builtin.DefaultAmtBitwidth) _, _, _, err := queue.AddActiveSectors(sectors, sectorSize) require.NoError(t, err) @@ -404,7 +404,7 @@ func TestExpirationQueue(t *testing.T) { t.Run("reschedules all sectors as faults", func(t *testing.T) { // Create expiration 3 sets with 2 sectors apiece - queue := emptyExpirationQueueWithQuantizing(t, miner.NewQuantSpec(4, 1)) + queue := emptyExpirationQueueWithQuantizing(t, miner.NewQuantSpec(4, 1), builtin.DefaultAmtBitwidth) _, _, _, err := queue.AddActiveSectors(sectors, sectorSize) require.NoError(t, err) @@ -465,7 +465,7 @@ func TestExpirationQueue(t *testing.T) { t.Run("reschedule expirations then reschedule as fault", func(t *testing.T) { // Create expiration 3 sets with 2 sectors apiece - queue := emptyExpirationQueueWithQuantizing(t, miner.NewQuantSpec(4, 1)) + queue := emptyExpirationQueueWithQuantizing(t, miner.NewQuantSpec(4, 1), builtin.DefaultAmtBitwidth) _, _, _, err := queue.AddActiveSectors(sectors, sectorSize) require.NoError(t, err) @@ -502,7 +502,7 @@ func TestExpirationQueue(t *testing.T) { t.Run("reschedule recover restores all sector stats", func(t *testing.T) { // Create expiration 3 sets with 2 sectors apiece - queue := emptyExpirationQueueWithQuantizing(t, miner.NewQuantSpec(4, 1)) + queue := emptyExpirationQueueWithQuantizing(t, miner.NewQuantSpec(4, 1), builtin.DefaultAmtBitwidth) _, _, _, err := queue.AddActiveSectors(sectors, sectorSize) require.NoError(t, err) @@ -567,7 +567,7 @@ func TestExpirationQueue(t *testing.T) { t.Run("replaces sectors with new sectors", func(t *testing.T) { // Create expiration 3 sets - queue := emptyExpirationQueueWithQuantizing(t, miner.NewQuantSpec(4, 1)) + queue := emptyExpirationQueueWithQuantizing(t, miner.NewQuantSpec(4, 1), builtin.DefaultAmtBitwidth) // add sectors to each set _, _, _, err := queue.AddActiveSectors([]*miner.SectorOnChainInfo{sectors[0], sectors[1], sectors[3], sectors[5]}, sectorSize) @@ -621,7 +621,7 @@ func TestExpirationQueue(t *testing.T) { t.Run("removes sectors", func(t *testing.T) { // add all sectors into 3 sets - queue := emptyExpirationQueueWithQuantizing(t, miner.NewQuantSpec(4, 1)) + queue := emptyExpirationQueueWithQuantizing(t, miner.NewQuantSpec(4, 1), builtin.DefaultAmtBitwidth) _, _, _, err := queue.AddActiveSectors(sectors, sectorSize) require.NoError(t, err) @@ -683,21 +683,21 @@ func TestExpirationQueue(t *testing.T) { }) t.Run("adding no sectors leaves the queue empty", func(t *testing.T) { - queue := emptyExpirationQueueWithQuantizing(t, miner.NewQuantSpec(4, 1)) + queue := emptyExpirationQueueWithQuantizing(t, miner.NewQuantSpec(4, 1), builtin.DefaultAmtBitwidth) _, _, _, err := queue.AddActiveSectors(nil, sectorSize) require.NoError(t, err) assert.Zero(t, queue.Length()) }) t.Run("rescheduling no expirations leaves the queue empty", func(t *testing.T) { - queue := emptyExpirationQueueWithQuantizing(t, miner.NewQuantSpec(4, 1)) + queue := emptyExpirationQueueWithQuantizing(t, miner.NewQuantSpec(4, 1), builtin.DefaultAmtBitwidth) err := queue.RescheduleExpirations(10, nil, sectorSize) require.NoError(t, err) assert.Zero(t, queue.Length()) }) t.Run("rescheduling no expirations as faults leaves the queue empty", func(t *testing.T) { - queue := emptyExpirationQueueWithQuantizing(t, miner.NewQuantSpec(4, 1)) + queue := emptyExpirationQueueWithQuantizing(t, miner.NewQuantSpec(4, 1), builtin.DefaultAmtBitwidth) _, _, _, err := queue.AddActiveSectors(sectors, sectorSize) require.NoError(t, err) @@ -710,7 +710,7 @@ func TestExpirationQueue(t *testing.T) { }) t.Run("rescheduling all expirations as faults leaves the queue empty if it was empty", func(t *testing.T) { - queue := emptyExpirationQueueWithQuantizing(t, miner.NewQuantSpec(4, 1)) + queue := emptyExpirationQueueWithQuantizing(t, miner.NewQuantSpec(4, 1), builtin.DefaultAmtBitwidth) _, _, _, err := queue.AddActiveSectors(sectors, sectorSize) require.NoError(t, err) @@ -723,7 +723,7 @@ func TestExpirationQueue(t *testing.T) { }) t.Run("rescheduling no sectors as recovered leaves the queue empty", func(t *testing.T) { - queue := emptyExpirationQueueWithQuantizing(t, miner.NewQuantSpec(4, 1)) + queue := emptyExpirationQueueWithQuantizing(t, miner.NewQuantSpec(4, 1), builtin.DefaultAmtBitwidth) _, err := queue.RescheduleRecovered(nil, sectorSize) require.NoError(t, err) assert.Zero(t, queue.Length()) @@ -752,7 +752,7 @@ func requireNoExpirationGroupsBefore(t *testing.T, epoch abi.ChainEpoch, queue m require.True(t, empty) } -func emptyExpirationQueueWithQuantizing(t *testing.T, quant miner.QuantSpec) miner.ExpirationQueue { +func emptyExpirationQueueWithQuantizing(t *testing.T, quant miner.QuantSpec, bitwidth int) miner.ExpirationQueue { rt := mock.NewBuilder(context.Background(), address.Undef).Build(t) store := adt.AsStore(rt) emptyArray, err := adt.MakeEmptyArray(store, builtin.DefaultAmtBitwidth) @@ -760,11 +760,11 @@ func emptyExpirationQueueWithQuantizing(t *testing.T, quant miner.QuantSpec) min root, err := emptyArray.Root() require.NoError(t, err) - queue, err := miner.LoadExpirationQueue(store, root, quant) + queue, err := miner.LoadExpirationQueue(store, root, quant, bitwidth) require.NoError(t, err) return queue } func emptyExpirationQueue(t *testing.T) miner.ExpirationQueue { - return emptyExpirationQueueWithQuantizing(t, miner.NoQuantization) + return emptyExpirationQueueWithQuantizing(t, miner.NoQuantization, builtin.DefaultAmtBitwidth) } diff --git a/actors/builtin/miner/miner_state.go b/actors/builtin/miner/miner_state.go index 095c873f96..a44a673ad8 100644 --- a/actors/builtin/miner/miner_state.go +++ b/actors/builtin/miner/miner_state.go @@ -77,6 +77,9 @@ type State struct { EarlyTerminations bitfield.BitField } +const PrecommitExpiryAmtBitwidth = 6 +const SectorsAmtBitwidth = 6 + type MinerInfo struct { // Account that owns this miner. // - Income and returned collateral are paid to this address. @@ -169,20 +172,45 @@ func ConstructState(store adt.Store, infoCid cid.Cid, periodStart abi.ChainEpoch if err != nil { return nil, xerrors.Errorf("failed to construct empty map: %w", err) } - emptyArray, err := adt.MakeEmptyArray(store, builtin.DefaultAmtBitwidth) + emptyPrecommitsExpiryArray, err := adt.MakeEmptyArray(store, PrecommitExpiryAmtBitwidth) + if err != nil { + return nil, xerrors.Errorf("failed to construct empty precommits array: %w", err) + } + emptyPrecommitsArrayCid, err := emptyPrecommitsExpiryArray.Root() + if err != nil { + return nil, xerrors.Errorf("failed to persist empty precommits array: %w", err) + } + emptySectorsArray, err := adt.MakeEmptyArray(store, SectorsAmtBitwidth) + if err != nil { + return nil, xerrors.Errorf("failed to construct empty sectors array: %w", err) + } + emptySectorsArrayCid, err := emptySectorsArray.Root() if err != nil { - return nil, xerrors.Errorf("failed to construct empty array") + return nil, xerrors.Errorf("failed to persist empty sectors array: %w", err) } - emptyArrayCid, err := emptyArray.Root() + emptyPartitionsArray, err := adt.MakeEmptyArray(store, builtin.DefaultAmtBitwidth) if err != nil { - return nil, xerrors.Errorf("failed to persist empty array: %w", err) + return nil, xerrors.Errorf("failed to construct empty partitions array: %w", err) } + emptyPartitionsArrayCid, err := emptyPartitionsArray.Root() + if err != nil { + return nil, xerrors.Errorf("failed to persist empty partitions array: %w", err) + } + emptyDeadlineExpirationArray, err := adt.MakeEmptyArray(store, DeadlineExpirationAmtBitwidth) + if err != nil { + return nil, xerrors.Errorf("failed to construct empty deadline expiration array: %w", err) + } + emptyDeadlineExpirationArrayCid, err := emptyDeadlineExpirationArray.Root() + if err != nil { + return nil, xerrors.Errorf("failed to persist empty deadline expiration array: %w", err) + } + emptyBitfield := bitfield.NewFromSet(nil) emptyBitfieldCid, err := store.Put(store.Context(), emptyBitfield) if err != nil { return nil, xerrors.Errorf("failed to construct empty bitfield: %w", err) } - emptyDeadline := ConstructDeadline(emptyArrayCid) + emptyDeadline := ConstructDeadline(emptyPartitionsArrayCid, emptyDeadlineExpirationArrayCid) emptyDeadlineCid, err := store.Put(store.Context(), emptyDeadline) if err != nil { return nil, xerrors.Errorf("failed to construct empty deadline: %w", err) @@ -210,9 +238,9 @@ func ConstructState(store adt.Store, infoCid cid.Cid, periodStart abi.ChainEpoch InitialPledge: abi.NewTokenAmount(0), PreCommittedSectors: emptyMapCid, - PreCommittedSectorsExpiry: emptyArrayCid, + PreCommittedSectorsExpiry: emptyPrecommitsArrayCid, AllocatedSectors: emptyBitfieldCid, - Sectors: emptyArrayCid, + Sectors: emptySectorsArrayCid, ProvingPeriodStart: periodStart, CurrentDeadline: deadlineIndex, Deadlines: emptyDeadlinesCid, @@ -980,7 +1008,7 @@ func (st *State) QuantSpecEveryDeadline() QuantSpec { func (st *State) AddPreCommitExpiry(store adt.Store, expireEpoch abi.ChainEpoch, sectorNum abi.SectorNumber) error { // Load BitField Queue for sector expiry quant := st.QuantSpecEveryDeadline() - queue, err := LoadBitfieldQueue(store, st.PreCommittedSectorsExpiry, quant) + queue, err := LoadBitfieldQueue(store, st.PreCommittedSectorsExpiry, quant, PrecommitExpiryAmtBitwidth) if err != nil { return xerrors.Errorf("failed to load pre-commit expiry queue: %w", err) } @@ -1002,7 +1030,7 @@ func (st *State) ExpirePreCommits(store adt.Store, currEpoch abi.ChainEpoch) (de depositToBurn = abi.NewTokenAmount(0) // expire pre-committed sectors - expiryQ, err := LoadBitfieldQueue(store, st.PreCommittedSectorsExpiry, st.QuantSpecEveryDeadline()) + expiryQ, err := LoadBitfieldQueue(store, st.PreCommittedSectorsExpiry, st.QuantSpecEveryDeadline(), PrecommitExpiryAmtBitwidth) if err != nil { return depositToBurn, xerrors.Errorf("failed to load sector expiry queue: %w", err) } diff --git a/actors/builtin/miner/miner_state_test.go b/actors/builtin/miner/miner_state_test.go index a5a0e3df31..0a5076ee55 100644 --- a/actors/builtin/miner/miner_state_test.go +++ b/actors/builtin/miner/miner_state_test.go @@ -603,7 +603,7 @@ func TestAddPreCommitExpiry(t *testing.T) { // assert quant := harness.s.QuantSpecEveryDeadline() - queue, err := miner.LoadBitfieldQueue(harness.store, harness.s.PreCommittedSectorsExpiry, quant) + queue, err := miner.LoadBitfieldQueue(harness.store, harness.s.PreCommittedSectorsExpiry, quant, miner.PrecommitExpiryAmtBitwidth) require.NoError(t, err) require.EqualValues(t, 1, queue.Length()) diff --git a/actors/builtin/miner/miner_test.go b/actors/builtin/miner/miner_test.go index 31809742e0..06eedc7bba 100644 --- a/actors/builtin/miner/miner_test.go +++ b/actors/builtin/miner/miner_test.go @@ -1499,7 +1499,7 @@ func TestCCUpgrade(t *testing.T) { }, dQueue) // but partitions expiration set at that epoch is empty - queue, err := miner.LoadExpirationQueue(rt.AdtStore(), partition.ExpirationsEpochs, miner.QuantSpecForDeadline(dlInfo)) + queue, err := miner.LoadExpirationQueue(rt.AdtStore(), partition.ExpirationsEpochs, miner.QuantSpecForDeadline(dlInfo), miner.PartitionExpirationAmtBitwidth) require.NoError(t, err) var es miner.ExpirationSet expirationSetNotEmpty, err := queue.Get(uint64(expectedReplacedExpiration), &es) @@ -4854,7 +4854,7 @@ func (h *actorHarness) collectSectors(rt *mock.Runtime) map[abi.SectorNumber]*mi } func (h *actorHarness) collectDeadlineExpirations(rt *mock.Runtime, deadline *miner.Deadline) map[abi.ChainEpoch][]uint64 { - queue, err := miner.LoadBitfieldQueue(rt.AdtStore(), deadline.ExpirationsEpochs, miner.NoQuantization) + queue, err := miner.LoadBitfieldQueue(rt.AdtStore(), deadline.ExpirationsEpochs, miner.NoQuantization, miner.DeadlineExpirationAmtBitwidth) require.NoError(h.t, err) expirations := map[abi.ChainEpoch][]uint64{} _ = queue.ForEach(func(epoch abi.ChainEpoch, bf bitfield.BitField) error { @@ -4867,7 +4867,7 @@ func (h *actorHarness) collectDeadlineExpirations(rt *mock.Runtime, deadline *mi } func (h *actorHarness) collectPartitionExpirations(rt *mock.Runtime, partition *miner.Partition) map[abi.ChainEpoch]*miner.ExpirationSet { - queue, err := miner.LoadExpirationQueue(rt.AdtStore(), partition.ExpirationsEpochs, miner.NoQuantization) + queue, err := miner.LoadExpirationQueue(rt.AdtStore(), partition.ExpirationsEpochs, miner.NoQuantization, miner.PartitionExpirationAmtBitwidth) require.NoError(h.t, err) expirations := map[abi.ChainEpoch]*miner.ExpirationSet{} var es miner.ExpirationSet diff --git a/actors/builtin/miner/partition_state.go b/actors/builtin/miner/partition_state.go index 2c9e16e330..db07b41414 100644 --- a/actors/builtin/miner/partition_state.go +++ b/actors/builtin/miner/partition_state.go @@ -10,6 +10,7 @@ import ( "github.com/ipfs/go-cid" "golang.org/x/xerrors" + "github.com/filecoin-project/specs-actors/v3/actors/builtin" "github.com/filecoin-project/specs-actors/v3/actors/util" "github.com/filecoin-project/specs-actors/v3/actors/util/adt" ) @@ -50,6 +51,10 @@ type Partition struct { RecoveringPower PowerPair } +// Bitwidth of partition expiration AMT determined empirically from mutation +// patterns and projections of mainnet data. +const PartitionExpirationAmtBitwidth = 4 + // Value type for a pair of raw and QA power. type PowerPair struct { Raw abi.StoragePower @@ -57,15 +62,15 @@ type PowerPair struct { } // A set of sectors associated with a given epoch. -func ConstructPartition(emptyArray cid.Cid) *Partition { +func ConstructPartition(emptyExpirationArray, emptyEarlyTerminationArray cid.Cid) *Partition { return &Partition{ Sectors: bitfield.New(), Unproven: bitfield.New(), Faults: bitfield.New(), Recoveries: bitfield.New(), Terminated: bitfield.New(), - ExpirationsEpochs: emptyArray, - EarlyTerminated: emptyArray, + ExpirationsEpochs: emptyExpirationArray, + EarlyTerminated: emptyEarlyTerminationArray, LivePower: NewPowerPairZero(), UnprovenPower: NewPowerPairZero(), FaultyPower: NewPowerPairZero(), @@ -113,7 +118,7 @@ func (p *Partition) ActivePower() PowerPair { func (p *Partition) AddSectors( store adt.Store, proven bool, sectors []*SectorOnChainInfo, ssize abi.SectorSize, quant QuantSpec, ) (PowerPair, error) { - expirations, err := LoadExpirationQueue(store, p.ExpirationsEpochs, quant) + expirations, err := LoadExpirationQueue(store, p.ExpirationsEpochs, quant, PartitionExpirationAmtBitwidth) if err != nil { return NewPowerPairZero(), xerrors.Errorf("failed to load sector expirations: %w", err) } @@ -160,7 +165,7 @@ func (p *Partition) addFaults( ssize abi.SectorSize, quant QuantSpec, ) (powerDelta, newFaultyPower PowerPair, err error) { // Load expiration queue - queue, err := LoadExpirationQueue(store, p.ExpirationsEpochs, quant) + queue, err := LoadExpirationQueue(store, p.ExpirationsEpochs, quant, PartitionExpirationAmtBitwidth) if err != nil { return NewPowerPairZero(), NewPowerPairZero(), xerrors.Errorf("failed to load partition queue: %w", err) } @@ -294,7 +299,7 @@ func (p *Partition) RecoverFaults(store adt.Store, sectors Sectors, ssize abi.Se return NewPowerPairZero(), xerrors.Errorf("failed to load recovered sectors: %w", err) } // Load expiration queue - queue, err := LoadExpirationQueue(store, p.ExpirationsEpochs, quant) + queue, err := LoadExpirationQueue(store, p.ExpirationsEpochs, quant, PartitionExpirationAmtBitwidth) if err != nil { return NewPowerPairZero(), xerrors.Errorf("failed to load partition queue: %w", err) } @@ -436,7 +441,7 @@ func (p *Partition) RescheduleExpirations( return nil, err } - expirations, err := LoadExpirationQueue(store, p.ExpirationsEpochs, quant) + expirations, err := LoadExpirationQueue(store, p.ExpirationsEpochs, quant, PartitionExpirationAmtBitwidth) if err != nil { return nil, xerrors.Errorf("failed to load sector expirations: %w", err) } @@ -463,7 +468,7 @@ func (p *Partition) RescheduleExpirations( // Returns the delta to power and pledge requirement. func (p *Partition) ReplaceSectors(store adt.Store, oldSectors, newSectors []*SectorOnChainInfo, ssize abi.SectorSize, quant QuantSpec) (PowerPair, abi.TokenAmount, error) { - expirations, err := LoadExpirationQueue(store, p.ExpirationsEpochs, quant) + expirations, err := LoadExpirationQueue(store, p.ExpirationsEpochs, quant, PartitionExpirationAmtBitwidth) if err != nil { return NewPowerPairZero(), big.Zero(), xerrors.Errorf("failed to load sector expirations: %w", err) } @@ -508,7 +513,7 @@ func (p *Partition) ReplaceSectors(store adt.Store, oldSectors, newSectors []*Se // Record the epoch of any sectors expiring early, for termination fee calculation later. func (p *Partition) recordEarlyTermination(store adt.Store, epoch abi.ChainEpoch, sectors bitfield.BitField) error { - etQueue, err := LoadBitfieldQueue(store, p.EarlyTerminated, NoQuantization) + etQueue, err := LoadBitfieldQueue(store, p.EarlyTerminated, NoQuantization, builtin.DefaultAmtBitwidth) if err != nil { return xerrors.Errorf("failed to load early termination queue: %w", err) } @@ -541,7 +546,7 @@ func (p *Partition) TerminateSectors( if err != nil { return nil, err } - expirations, err := LoadExpirationQueue(store, p.ExpirationsEpochs, quant) + expirations, err := LoadExpirationQueue(store, p.ExpirationsEpochs, quant, PartitionExpirationAmtBitwidth) if err != nil { return nil, xerrors.Errorf("failed to load sector expirations: %w", err) } @@ -617,7 +622,7 @@ func (p *Partition) PopExpiredSectors(store adt.Store, until abi.ChainEpoch, qua return nil, xerrors.Errorf("cannot pop expired sectors from a partition with unproven sectors: %w", err) } - expirations, err := LoadExpirationQueue(store, p.ExpirationsEpochs, quant) + expirations, err := LoadExpirationQueue(store, p.ExpirationsEpochs, quant, PartitionExpirationAmtBitwidth) if err != nil { return nil, xerrors.Errorf("failed to load expiration queue: %w", err) } @@ -687,7 +692,7 @@ func (p *Partition) RecordMissedPost( ) (powerDelta, penalizedPower, newFaultyPower PowerPair, err error) { // Collapse tail of queue into the last entry, and mark all power faulty. // Load expiration queue - queue, err := LoadExpirationQueue(store, p.ExpirationsEpochs, quant) + queue, err := LoadExpirationQueue(store, p.ExpirationsEpochs, quant, PartitionExpirationAmtBitwidth) if err != nil { return NewPowerPairZero(), NewPowerPairZero(), NewPowerPairZero(), xerrors.Errorf("failed to load partition queue: %w", err) } @@ -733,7 +738,7 @@ func (p *Partition) PopEarlyTerminations(store adt.Store, maxSectors uint64) (re stopErr := errors.New("stop iter") // Load early terminations. - earlyTerminatedQ, err := LoadBitfieldQueue(store, p.EarlyTerminated, NoQuantization) + earlyTerminatedQ, err := LoadBitfieldQueue(store, p.EarlyTerminated, NoQuantization, builtin.DefaultAmtBitwidth) if err != nil { return TerminationResult{}, false, err } diff --git a/actors/builtin/miner/partition_state_test.go b/actors/builtin/miner/partition_state_test.go index e3407c3792..f1c31bdb91 100644 --- a/actors/builtin/miner/partition_state_test.go +++ b/actors/builtin/miner/partition_state_test.go @@ -449,7 +449,7 @@ func TestPartitions(t *testing.T) { }) // sectors should be added to early termination bitfield queue - queue, err := miner.LoadBitfieldQueue(store, partition.EarlyTerminated, miner.NoQuantization) + queue, err := miner.LoadBitfieldQueue(store, partition.EarlyTerminated, miner.NoQuantization, builtin.DefaultAmtBitwidth) require.NoError(t, err) ExpectBQ(). @@ -542,7 +542,7 @@ func TestPartitions(t *testing.T) { }) // sectors should be added to early termination bitfield queue - queue, err := miner.LoadBitfieldQueue(store, partition.EarlyTerminated, miner.NoQuantization) + queue, err := miner.LoadBitfieldQueue(store, partition.EarlyTerminated, miner.NoQuantization, builtin.DefaultAmtBitwidth) require.NoError(t, err) // only early termination appears in bitfield queue @@ -663,7 +663,7 @@ func TestPartitions(t *testing.T) { assert.True(t, hasMore) // expect terminations to still contain 3 and 5 - queue, err := miner.LoadBitfieldQueue(store, partition.EarlyTerminated, miner.NoQuantization) + queue, err := miner.LoadBitfieldQueue(store, partition.EarlyTerminated, miner.NoQuantization, builtin.DefaultAmtBitwidth) require.NoError(t, err) // only early termination appears in bitfield queue @@ -682,7 +682,7 @@ func TestPartitions(t *testing.T) { assert.False(t, hasMore) // expect early terminations to be empty - queue, err = miner.LoadBitfieldQueue(store, partition.EarlyTerminated, miner.NoQuantization) + queue, err = miner.LoadBitfieldQueue(store, partition.EarlyTerminated, miner.NoQuantization, builtin.DefaultAmtBitwidth) require.NoError(t, err) ExpectBQ().Equals(t, queue) }) @@ -863,7 +863,7 @@ type expectExpirationGroup struct { } func assertPartitionExpirationQueue(t *testing.T, store adt.Store, partition *miner.Partition, quant miner.QuantSpec, groups []expectExpirationGroup) { - queue, err := miner.LoadExpirationQueue(store, partition.ExpirationsEpochs, quant) + queue, err := miner.LoadExpirationQueue(store, partition.ExpirationsEpochs, quant, miner.PartitionExpirationAmtBitwidth) require.NoError(t, err) for _, group := range groups { @@ -923,12 +923,17 @@ func selectSectors(t *testing.T, sectors []*miner.SectorOnChainInfo, field bitfi } func emptyPartition(t *testing.T, store adt.Store) *miner.Partition { - emptyArray, err := adt.MakeEmptyArray(store, builtin.DefaultAmtBitwidth) + emptyExpirationArray, err := adt.MakeEmptyArray(store, miner.PartitionExpirationAmtBitwidth) require.NoError(t, err) - root, err := emptyArray.Root() + expirationRoot, err := emptyExpirationArray.Root() require.NoError(t, err) - return miner.ConstructPartition(root) + emptyEarlyTerminationArray, err := adt.MakeEmptyArray(store, builtin.DefaultAmtBitwidth) + require.NoError(t, err) + earlyTerminationRoot, err := emptyEarlyTerminationArray.Root() + require.NoError(t, err) + + return miner.ConstructPartition(expirationRoot, earlyTerminationRoot) } func rescheduleSectors(t *testing.T, target abi.ChainEpoch, sectors []*miner.SectorOnChainInfo, filter bitfield.BitField) []*miner.SectorOnChainInfo { diff --git a/actors/builtin/miner/sectors.go b/actors/builtin/miner/sectors.go index 8b121bd3b8..a25db2a1e5 100644 --- a/actors/builtin/miner/sectors.go +++ b/actors/builtin/miner/sectors.go @@ -9,12 +9,11 @@ import ( "github.com/ipfs/go-cid" "golang.org/x/xerrors" - "github.com/filecoin-project/specs-actors/v3/actors/builtin" "github.com/filecoin-project/specs-actors/v3/actors/util/adt" ) func LoadSectors(store adt.Store, root cid.Cid) (Sectors, error) { - sectorsArr, err := adt.AsArray(store, root, builtin.DefaultAmtBitwidth) + sectorsArr, err := adt.AsArray(store, root, SectorsAmtBitwidth) if err != nil { return Sectors{}, err } diff --git a/actors/builtin/miner/testing.go b/actors/builtin/miner/testing.go index 50733bd149..e2067fda7c 100644 --- a/actors/builtin/miner/testing.go +++ b/actors/builtin/miner/testing.go @@ -64,7 +64,7 @@ func CheckStateInvariants(st *State, store adt.Store, balance abi.TokenAmount) ( minerSummary.Deals = map[abi.DealID]DealSummary{} var allSectors map[abi.SectorNumber]*SectorOnChainInfo - if sectorsArr, err := adt.AsArray(store, st.Sectors, builtin.DefaultAmtBitwidth); err != nil { + if sectorsArr, err := adt.AsArray(store, st.Sectors, SectorsAmtBitwidth); err != nil { acc.Addf("error loading sectors") } else { allSectors = map[abi.SectorNumber]*SectorOnChainInfo{} @@ -255,7 +255,7 @@ func CheckDeadlineStateInvariants(deadline *Deadline, store adt.Store, quant Qua { // Validate partition expiration queue contains an entry for each partition and epoch with an expiration. // The queue may be a superset of the partitions that have expirations because we never remove from it. - if expirationEpochs, err := adt.AsArray(store, deadline.ExpirationsEpochs, builtin.DefaultAmtBitwidth); err != nil { + if expirationEpochs, err := adt.AsArray(store, deadline.ExpirationsEpochs, DeadlineExpirationAmtBitwidth); err != nil { acc.Addf("error loading expiration queue: %v", err) } else { for epoch, expiringPIdxs := range partitionsWithExpirations { // nolint:nomaprange @@ -423,7 +423,7 @@ func CheckPartitionStateInvariants( // Validate the expiration queue. var expirationEpochs []abi.ChainEpoch - if expQ, err := LoadExpirationQueue(store, partition.ExpirationsEpochs, quant); err != nil { + if expQ, err := LoadExpirationQueue(store, partition.ExpirationsEpochs, quant, PartitionExpirationAmtBitwidth); err != nil { acc.Addf("error loading expiration queue: %v", err) } else if liveSectors != nil { qsummary := CheckExpirationQueue(expQ, liveSectors, partition.Faults, quant, sectorSize, acc) @@ -439,7 +439,7 @@ func CheckPartitionStateInvariants( // Validate the early termination queue. earlyTerminationCount := 0 - if earlyQ, err := LoadBitfieldQueue(store, partition.EarlyTerminated, NoQuantization); err != nil { + if earlyQ, err := LoadBitfieldQueue(store, partition.EarlyTerminated, NoQuantization, builtin.DefaultAmtBitwidth); err != nil { acc.Addf("error loading early termination queue: %v", err) } else { earlyTerminationCount = CheckEarlyTerminationQueue(earlyQ, partition.Terminated, acc) @@ -708,7 +708,7 @@ func CheckPreCommits(st *State, store adt.Store, allocatedSectors map[uint64]boo // invert pre-commit expiry queue into a lookup by sector number expireEpochs := make(map[uint64]abi.ChainEpoch) - if expiryQ, err := LoadBitfieldQueue(store, st.PreCommittedSectorsExpiry, st.QuantSpecEveryDeadline()); err != nil { + if expiryQ, err := LoadBitfieldQueue(store, st.PreCommittedSectorsExpiry, st.QuantSpecEveryDeadline(), PrecommitExpiryAmtBitwidth); err != nil { acc.Addf("error loading pre-commit expiry queue: %v", err) } else { err = expiryQ.ForEach(func(epoch abi.ChainEpoch, bf bitfield.BitField) error { diff --git a/actors/builtin/power/policy.go b/actors/builtin/power/policy.go index 065d0f31cc..b97c77bbc9 100644 --- a/actors/builtin/power/policy.go +++ b/actors/builtin/power/policy.go @@ -10,3 +10,15 @@ const ConsensusMinerMinMiners = 4 // PARAM_SPEC // This limits the number of proof partitions we may need to load in the cron call path. // Onboarding 1EiB/year requires at least 32 prove-commits per epoch. const MaxMinerProveCommitsPerEpoch = 200 // PARAM_SPEC + +// Bitwidth of CronEventQueue HAMT determined empirically from mutation +// patterns and projections of mainnet data. +const CronQueueHamtBitwidth = 6 + +// Bitwidth of CronEventQueue AMT determined empirically from mutation +// patterns and projections of mainnet data. +const CronQueueAmtBitwidth = 6 + +// Bitwidth of ProofValidationBatch AMT determined empirically from mutation +// pattersn and projections of mainnet data. +const ProofValidationBatchAmtBitwidth = 4 diff --git a/actors/builtin/power/power_actor.go b/actors/builtin/power/power_actor.go index 9b312a7e87..6f90147786 100644 --- a/actors/builtin/power/power_actor.go +++ b/actors/builtin/power/power_actor.go @@ -198,7 +198,7 @@ func (a Actor) EnrollCronEvent(rt Runtime, params *EnrollCronEventParams) *abi.E var st State rt.StateTransaction(&st, func() { - events, err := adt.AsMultimap(adt.AsStore(rt), st.CronEventQueue, builtin.DefaultHamtBitwidth, builtin.DefaultAmtBitwidth) + events, err := adt.AsMultimap(adt.AsStore(rt), st.CronEventQueue, CronQueueHamtBitwidth, CronQueueAmtBitwidth) builtin.RequireNoErr(rt, err, exitcode.ErrIllegalState, "failed to load cron events") err = st.appendCronEvent(events, params.EventEpoch, &minerEvent) @@ -270,10 +270,10 @@ func (a Actor) SubmitPoRepForBulkVerify(rt Runtime, sealInfo *proof.SealVerifyIn store := adt.AsStore(rt) var mmap *adt.Multimap if st.ProofValidationBatch == nil { - mmap = adt.MakeEmptyMultimap(store, builtin.DefaultHamtBitwidth, builtin.DefaultAmtBitwidth) + mmap = adt.MakeEmptyMultimap(store, builtin.DefaultHamtBitwidth, ProofValidationBatchAmtBitwidth) } else { var err error - mmap, err = adt.AsMultimap(adt.AsStore(rt), *st.ProofValidationBatch, builtin.DefaultHamtBitwidth, builtin.DefaultAmtBitwidth) + mmap, err = adt.AsMultimap(adt.AsStore(rt), *st.ProofValidationBatch, builtin.DefaultHamtBitwidth, ProofValidationBatchAmtBitwidth) builtin.RequireNoErr(rt, err, exitcode.ErrIllegalState, "failed to load proof batch set") } @@ -348,7 +348,7 @@ func (a Actor) processBatchProofVerifies(rt Runtime) { if st.ProofValidationBatch == nil { return } - mmap, err := adt.AsMultimap(store, *st.ProofValidationBatch, builtin.DefaultHamtBitwidth, builtin.DefaultAmtBitwidth) + mmap, err := adt.AsMultimap(store, *st.ProofValidationBatch, builtin.DefaultHamtBitwidth, ProofValidationBatchAmtBitwidth) builtin.RequireNoErr(rt, err, exitcode.ErrIllegalState, "failed to load proofs validation batch") claims, err := adt.AsMap(adt.AsStore(rt), st.Claims, builtin.DefaultHamtBitwidth) @@ -430,7 +430,7 @@ func (a Actor) processDeferredCronEvents(rt Runtime) { var cronEvents []CronEvent var st State rt.StateTransaction(&st, func() { - events, err := adt.AsMultimap(adt.AsStore(rt), st.CronEventQueue, builtin.DefaultHamtBitwidth, builtin.DefaultAmtBitwidth) + events, err := adt.AsMultimap(adt.AsStore(rt), st.CronEventQueue, CronQueueHamtBitwidth, CronQueueAmtBitwidth) builtin.RequireNoErr(rt, err, exitcode.ErrIllegalState, "failed to load cron events") claims, err := adt.AsMap(adt.AsStore(rt), st.Claims, builtin.DefaultHamtBitwidth) diff --git a/actors/builtin/power/power_state.go b/actors/builtin/power/power_state.go index 9984da21c7..670ef451b2 100644 --- a/actors/builtin/power/power_state.go +++ b/actors/builtin/power/power_state.go @@ -77,7 +77,7 @@ func ConstructState(store adt.Store) (*State, error) { if err != nil { return nil, xerrors.Errorf("failed to create empty map: %w", err) } - emptyMMapCid, err := adt.MakeEmptyMultimap(store, builtin.DefaultHamtBitwidth, builtin.DefaultAmtBitwidth).Root() + emptyCronQueueMMapCid, err := adt.MakeEmptyMultimap(store, CronQueueHamtBitwidth, CronQueueAmtBitwidth).Root() if err != nil { return nil, xerrors.Errorf("failed to create empty multimap: %w", err) } @@ -93,7 +93,7 @@ func ConstructState(store adt.Store) (*State, error) { ThisEpochPledgeCollateral: abi.NewTokenAmount(0), ThisEpochQAPowerSmoothed: smoothing.NewEstimate(InitialQAPowerEstimatePosition, InitialQAPowerEstimateVelocity), FirstCronEpoch: 0, - CronEventQueue: emptyMMapCid, + CronEventQueue: emptyCronQueueMMapCid, Claims: emptyMapCid, MinerCount: 0, MinerAboveMinPowerCount: 0, diff --git a/actors/builtin/power/power_test.go b/actors/builtin/power/power_test.go index baa78794bf..d9d51a54cf 100644 --- a/actors/builtin/power/power_test.go +++ b/actors/builtin/power/power_test.go @@ -674,7 +674,7 @@ func TestCron(t *testing.T) { // assert used cron events are cleaned up st := getState(rt) - mmap, err := adt.AsMultimap(rt.AdtStore(), st.CronEventQueue, builtin.DefaultHamtBitwidth, builtin.DefaultAmtBitwidth) + mmap, err := adt.AsMultimap(rt.AdtStore(), st.CronEventQueue, power.CronQueueHamtBitwidth, power.CronQueueAmtBitwidth) require.NoError(t, err) var ev power.CronEvent @@ -868,7 +868,7 @@ func TestSubmitPoRepForBulkVerify(t *testing.T) { st := getState(rt) store := rt.AdtStore() require.NotNil(t, st.ProofValidationBatch) - mmap, err := adt.AsMultimap(store, *st.ProofValidationBatch, builtin.DefaultHamtBitwidth, builtin.DefaultAmtBitwidth) + mmap, err := adt.AsMultimap(store, *st.ProofValidationBatch, builtin.DefaultHamtBitwidth, power.ProofValidationBatchAmtBitwidth) require.NoError(t, err) arr, found, err := mmap.Get(abi.AddrKey(miner)) require.NoError(t, err) @@ -1278,7 +1278,7 @@ func (h *spActorHarness) getEnrolledCronTicks(rt *mock.Runtime, epoch abi.ChainE var st power.State rt.GetState(&st) - events, err := adt.AsMultimap(adt.AsStore(rt), st.CronEventQueue, builtin.DefaultHamtBitwidth, builtin.DefaultAmtBitwidth) + events, err := adt.AsMultimap(adt.AsStore(rt), st.CronEventQueue, power.CronQueueHamtBitwidth, power.CronQueueAmtBitwidth) builtin.RequireNoErr(rt, err, exitcode.ErrIllegalState, "failed to load cron events") evts, found, err := events.Get(abi.IntKey(int64(epoch))) diff --git a/actors/builtin/power/testing.go b/actors/builtin/power/testing.go index d4738f4ca0..3a632ca65b 100644 --- a/actors/builtin/power/testing.go +++ b/actors/builtin/power/testing.go @@ -57,7 +57,7 @@ func CheckStateInvariants(st *State, store adt.Store) (*StateSummary, *builtin.M func CheckCronInvariants(st *State, store adt.Store, acc *builtin.MessageAccumulator) CronEventsByAddress { byAddress := make(CronEventsByAddress) - queue, err := adt.AsMultimap(store, st.CronEventQueue, builtin.DefaultHamtBitwidth, builtin.DefaultAmtBitwidth) + queue, err := adt.AsMultimap(store, st.CronEventQueue, CronQueueHamtBitwidth, CronQueueAmtBitwidth) if err != nil { acc.Addf("error loading cron event queue: %v", err) // Bail here. @@ -152,7 +152,7 @@ func CheckProofValidationInvariants(st *State, store adt.Store, claims ClaimsByA } proofs := make(ProofsByAddress) - if queue, err := adt.AsMultimap(store, *st.ProofValidationBatch, builtin.DefaultHamtBitwidth, builtin.DefaultAmtBitwidth); err != nil { + if queue, err := adt.AsMultimap(store, *st.ProofValidationBatch, builtin.DefaultHamtBitwidth, ProofValidationBatchAmtBitwidth); err != nil { acc.Addf("error loading proof validation queue: %v", err) } else { err = queue.ForAll(func(key string, arr *adt.Array) error { diff --git a/actors/migration/nv9/market.go b/actors/migration/nv9/market.go index c488a53a7a..2d8a4afa79 100644 --- a/actors/migration/nv9/market.go +++ b/actors/migration/nv9/market.go @@ -6,6 +6,8 @@ import ( cid "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" + amt3 "github.com/filecoin-project/go-amt-ipld/v3" + hamt3 "github.com/filecoin-project/go-hamt-ipld/v3" market2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/market" adt2 "github.com/filecoin-project/specs-actors/v2/actors/util/adt" @@ -26,19 +28,19 @@ func (m marketMigrator) MigrateState(ctx context.Context, store cbor.IpldStore, if err != nil { return nil, err } - proposalsCidOut, err := migrateAMTRaw(ctx, store, inState.Proposals, adt3.DefaultAmtOptions) + proposalsCidOut, err := migrateAMTRaw(ctx, store, inState.Proposals, []amt3.Option{amt3.UseTreeBitWidth(market3.ProposalsAmtBitwidth)}) if err != nil { return nil, err } - statesCidOut, err := migrateAMTRaw(ctx, store, inState.States, adt3.DefaultAmtOptions) + statesCidOut, err := migrateAMTRaw(ctx, store, inState.States, []amt3.Option{amt3.UseTreeBitWidth(market3.StatesAmtBitwidth)}) if err != nil { return nil, err } - escrowTableCidOut, err := migrateHAMTRaw(ctx, store, inState.EscrowTable, adt3.DefaultHamtOptionsWithDefaultBitwidth) + escrowTableCidOut, err := migrateHAMTRaw(ctx, store, inState.EscrowTable, append(adt3.DefaultHamtOptions, hamt3.UseTreeBitWidth(adt3.BalanceTableBitwidth))) if err != nil { return nil, err } - lockedTableCidOut, err := migrateHAMTRaw(ctx, store, inState.LockedTable, adt3.DefaultHamtOptionsWithDefaultBitwidth) + lockedTableCidOut, err := migrateHAMTRaw(ctx, store, inState.LockedTable, append(adt3.DefaultHamtOptions, hamt3.UseTreeBitWidth(adt3.BalanceTableBitwidth))) if err != nil { return nil, err } diff --git a/actors/migration/nv9/miner.go b/actors/migration/nv9/miner.go index 303b2914f5..bb195811f7 100644 --- a/actors/migration/nv9/miner.go +++ b/actors/migration/nv9/miner.go @@ -3,6 +3,7 @@ package nv9 import ( "context" + amt3 "github.com/filecoin-project/go-amt-ipld/v3" miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" adt2 "github.com/filecoin-project/specs-actors/v2/actors/util/adt" miner3 "github.com/filecoin-project/specs-actors/v3/actors/builtin/miner" @@ -27,11 +28,11 @@ func (m minerMigrator) MigrateState(ctx context.Context, store cbor.IpldStore, i if err != nil { return nil, err } - preCommittedSectorsExpiryOut, err := migrateAMTRaw(ctx, store, inState.PreCommittedSectorsExpiry, adt3.DefaultAmtOptions) + preCommittedSectorsExpiryOut, err := migrateAMTRaw(ctx, store, inState.PreCommittedSectorsExpiry, append(adt3.DefaultAmtOptions, amt3.UseTreeBitWidth(miner3.PrecommitExpiryAmtBitwidth))) if err != nil { return nil, err } - sectorsOut, err := migrateAMTRaw(ctx, store, inState.Sectors, adt3.DefaultAmtOptions) + sectorsOut, err := migrateAMTRaw(ctx, store, inState.Sectors, append(adt3.DefaultAmtOptions, amt3.UseTreeBitWidth(miner3.SectorsAmtBitwidth))) if err != nil { return nil, err } @@ -89,7 +90,7 @@ func (m *minerMigrator) migrateDeadlines(ctx context.Context, store cbor.IpldSto return cid.Undef, xerrors.Errorf("partitions: %w", err) } - expirationEpochs, err := migrateAMTRaw(ctx, store, inDeadline.ExpirationsEpochs, adt3.DefaultAmtOptions) + expirationEpochs, err := migrateAMTRaw(ctx, store, inDeadline.ExpirationsEpochs, append(adt3.DefaultAmtOptions, amt3.UseTreeBitWidth(miner3.DeadlineExpirationAmtBitwidth))) if err != nil { return cid.Undef, xerrors.Errorf("bitfield queue: %w", err) } @@ -127,7 +128,7 @@ func (m *minerMigrator) migratePartitions(ctx context.Context, store cbor.IpldSt var inPartition miner2.Partition if err = inArray.ForEach(&inPartition, func(i int64) error { - expirationEpochs, err := migrateAMTRaw(ctx, store, inPartition.ExpirationsEpochs, adt3.DefaultAmtOptions) + expirationEpochs, err := migrateAMTRaw(ctx, store, inPartition.ExpirationsEpochs, append(adt3.DefaultAmtOptions, amt3.UseTreeBitWidth(miner3.PartitionExpirationAmtBitwidth))) if err != nil { return xerrors.Errorf("expiration queue: %w", err) } diff --git a/actors/migration/nv9/power.go b/actors/migration/nv9/power.go index 2d70254547..cb6db63c17 100644 --- a/actors/migration/nv9/power.go +++ b/actors/migration/nv9/power.go @@ -3,6 +3,8 @@ package nv9 import ( "context" + amt3 "github.com/filecoin-project/go-amt-ipld/v3" + hamt3 "github.com/filecoin-project/go-hamt-ipld/v3" power2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/power" cid "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" @@ -23,7 +25,8 @@ func (m powerMigrator) MigrateState(ctx context.Context, store cbor.IpldStore, i var proofValidationBatchOut *cid.Cid if inState.ProofValidationBatch != nil { - proofValidationBatchOutCID, err := migrateHAMTAMTRaw(ctx, store, *inState.ProofValidationBatch, adt3.DefaultHamtOptionsWithDefaultBitwidth, adt3.DefaultAmtOptions) + proofValidationBatchAmtOpts := append(adt3.DefaultAmtOptions, amt3.UseTreeBitWidth(power3.ProofValidationBatchAmtBitwidth)) + proofValidationBatchOutCID, err := migrateHAMTAMTRaw(ctx, store, *inState.ProofValidationBatch, adt3.DefaultHamtOptionsWithDefaultBitwidth, proofValidationBatchAmtOpts) if err != nil { return nil, err } @@ -35,7 +38,9 @@ func (m powerMigrator) MigrateState(ctx context.Context, store cbor.IpldStore, i return nil, err } - cronEventQueueOut, err := migrateHAMTAMTRaw(ctx, store, inState.CronEventQueue, adt3.DefaultHamtOptionsWithDefaultBitwidth, adt3.DefaultAmtOptions) + cronEventQueueHamtOpts := append(adt3.DefaultHamtOptions, hamt3.UseTreeBitWidth(power3.CronQueueHamtBitwidth)) + cronEventQueueAmtOpts := append(adt3.DefaultAmtOptions, amt3.UseTreeBitWidth(power3.CronQueueAmtBitwidth)) + cronEventQueueOut, err := migrateHAMTAMTRaw(ctx, store, inState.CronEventQueue, cronEventQueueHamtOpts, cronEventQueueAmtOpts) if err != nil { return nil, err } diff --git a/actors/util/adt/balancetable.go b/actors/util/adt/balancetable.go index dbffbe2c52..93232aca9d 100644 --- a/actors/util/adt/balancetable.go +++ b/actors/util/adt/balancetable.go @@ -6,17 +6,19 @@ import ( "github.com/filecoin-project/go-state-types/big" cid "github.com/ipfs/go-cid" "golang.org/x/xerrors" - - "github.com/filecoin-project/specs-actors/v3/actors/builtin" ) +// Bitwidth of balance table HAMTs, determined empirically from mutation +// patterns and projections of mainnet data +const BalanceTableBitwidth = 6 + // A specialization of a map of addresses to (positive) token amounts. // Absent keys implicitly have a balance of zero. type BalanceTable Map // Interprets a store as balance table with root `r`. func AsBalanceTable(s Store, r cid.Cid) (*BalanceTable, error) { - m, err := AsMap(s, r, builtin.DefaultHamtBitwidth) + m, err := AsMap(s, r, BalanceTableBitwidth) if err != nil { return nil, err }