Merge PR cosmos#3333: F1 storage efficiency improvements

PENDING.md

@@ -45,6 +45,7 @@ BREAKING CHANGES
     * `Delegation` -> `Value` in `MsgCreateValidator` and `MsgDelegate`
     * `MsgBeginUnbonding` -> `MsgUndelegate`
   * [\#3315] Increase decimal precision to 18
+  * \#3333 - F1 storage efficiency improvements - automatic withdrawals when unbonded, historical reward reference counting
   * \#3323 Update to Tendermint 0.29.0
   * [\#3328](https://github.com/cosmos/cosmos-sdk/issues/3328) [x/gov] Remove redundant action tag
 

cmd/gaia/app/export.go

@@ -74,10 +74,10 @@ func (app *GaiaApp) prepForZeroHeightGenesis(ctx sdk.Context) {
 	}
 
 	// clear validator slash events
-	app.distrKeeper.DeleteValidatorSlashEvents(ctx)
+	app.distrKeeper.DeleteAllValidatorSlashEvents(ctx)
 
 	// clear validator historical rewards
-	app.distrKeeper.DeleteValidatorHistoricalRewards(ctx)
+	app.distrKeeper.DeleteAllValidatorHistoricalRewards(ctx)
 
 	// set context height to zero
 	height := ctx.BlockHeight()

docs/spec/distribution/f1_reference_counting.md

@@ -0,0 +1,18 @@
+## Reference Counting in F1 Fee Distribution
+
+In F1 fee distribution, in order to calculate the rewards a delegator ought to receive when they
+withdraw their delegation, we must read the terms of the summation of rewards divided by tokens from
+the period which they ended when they delegated, and the final period (created when they withdraw).
+
+Additionally, as slashes change the amount of tokens a delegation will have (but we calculate this lazily,
+only when a delegator un-delegates), we must calculate rewards in separate periods before / after any slashes
+which occurred in between when a delegator delegated and when they withdrew their rewards. Thus slashes, like
+delegations, reference the period which was ended by the slash event.
+
+All stored historical rewards records for periods which are no longer referenced by any delegations
+or any slashes can thus be safely removed, as they will never be read (future delegations and future
+slashes will always reference future periods). This is implemented by tracking a `ReferenceCount`
+along with each historical reward storage entry. Each time a new object (delegation or slash)
+is created which might need to reference the historical record, the reference count is incremented.
+Each time one object which previously needed to reference the historical record is deleted, the reference
+count is decremented. If the reference count hits zero, the historical record is deleted.

x/distribution/keeper/delegation.go

@@ -8,14 +8,16 @@ import (
 
 // initialize starting info for a new delegation
 func (k Keeper) initializeDelegation(ctx sdk.Context, val sdk.ValAddress, del sdk.AccAddress) {
-	// period has already been incremented
-	period := k.GetValidatorCurrentRewards(ctx, val).Period
+	// period has already been incremented - we want to store the period ended by this delegation action
+	previousPeriod := k.GetValidatorCurrentRewards(ctx, val).Period - 1
+
 	validator := k.stakingKeeper.Validator(ctx, val)
 	delegation := k.stakingKeeper.Delegation(ctx, del, val)
+
 	// calculate delegation stake in tokens
 	// we don't store directly, so multiply delegation shares * (tokens per share)
 	stake := delegation.GetShares().Mul(validator.GetDelegatorShareExRate())
-	k.SetDelegatorStartingInfo(ctx, val, del, types.NewDelegatorStartingInfo(period-1, stake, uint64(ctx.BlockHeight())))
+	k.SetDelegatorStartingInfo(ctx, val, del, types.NewDelegatorStartingInfo(previousPeriod, stake, uint64(ctx.BlockHeight())))
 }
 
 // calculate the rewards accrued by a delegation between two periods
@@ -29,7 +31,7 @@ func (k Keeper) calculateDelegationRewardsBetween(ctx sdk.Context, val sdk.Valid
 	// return staking * (ending - starting)
 	starting := k.GetValidatorHistoricalRewards(ctx, val.GetOperator(), startingPeriod)
 	ending := k.GetValidatorHistoricalRewards(ctx, val.GetOperator(), endingPeriod)
-	difference := ending.Minus(starting)
+	difference := ending.CumulativeRewardRatio.Minus(starting.CumulativeRewardRatio)
 	rewards = difference.MulDec(staking)
 	return
 }
@@ -50,7 +52,7 @@ func (k Keeper) calculateDelegationRewards(ctx sdk.Context, val sdk.Validator, d
 	if endingHeight >= startingHeight {
 		k.IterateValidatorSlashEventsBetween(ctx, del.GetValidatorAddr(), startingHeight, endingHeight,
 			func(height uint64, event types.ValidatorSlashEvent) (stop bool) {
-				endingPeriod := event.ValidatorPeriod - 1
+				endingPeriod := event.ValidatorPeriod
 				rewards = rewards.Plus(k.calculateDelegationRewardsBetween(ctx, val, startingPeriod, endingPeriod, stake))
 				stake = stake.Mul(sdk.OneDec().Sub(event.Fraction))
 				startingPeriod = endingPeriod
@@ -67,10 +69,20 @@ func (k Keeper) calculateDelegationRewards(ctx sdk.Context, val sdk.Validator, d
 
 func (k Keeper) withdrawDelegationRewards(ctx sdk.Context, val sdk.Validator, del sdk.Delegation) sdk.Error {
 
+	// check existence of delegator starting info
+	if !k.HasDelegatorStartingInfo(ctx, del.GetValidatorAddr(), del.GetDelegatorAddr()) {
+		return types.ErrNoDelegationDistInfo(k.codespace)
+	}
+
 	// end current period and calculate rewards
 	endingPeriod := k.incrementValidatorPeriod(ctx, val)
 	rewards := k.calculateDelegationRewards(ctx, val, del, endingPeriod)
 
+	// decrement reference count of starting period
+	startingInfo := k.GetDelegatorStartingInfo(ctx, del.GetValidatorAddr(), del.GetDelegatorAddr())
+	startingPeriod := startingInfo.PreviousPeriod
+	k.decrementReferenceCount(ctx, del.GetValidatorAddr(), startingPeriod)
+
 	// truncate coins, return remainder to community pool
 	coins, remainder := rewards.TruncateDecimal()
 	outstanding := k.GetOutstandingRewards(ctx)
@@ -85,5 +97,8 @@ func (k Keeper) withdrawDelegationRewards(ctx sdk.Context, val sdk.Validator, de
 		return err
 	}
 
+	// remove delegator starting info
+	k.DeleteDelegatorStartingInfo(ctx, del.GetValidatorAddr(), del.GetDelegatorAddr())
+
 	return nil
 }

x/distribution/keeper/delegation_test.go

@@ -29,9 +29,15 @@ func TestCalculateRewardsBasic(t *testing.T) {
 	val := sk.Validator(ctx, valOpAddr1)
 	del := sk.Delegation(ctx, sdk.AccAddress(valOpAddr1), valOpAddr1)
 
+	// historical count should be 2 (once for validator init, once for delegation init)
+	require.Equal(t, uint64(2), k.GetValidatorHistoricalRewardCount(ctx))
+
 	// end period
 	endingPeriod := k.incrementValidatorPeriod(ctx, val)
 
+	// historical count should be 3 (since we ended the period, and haven't yet decremented a reference)
+	require.Equal(t, uint64(3), k.GetValidatorHistoricalRewardCount(ctx))
+
 	// calculate delegation rewards
 	rewards := k.calculateDelegationRewards(ctx, val, del, endingPeriod)
 
@@ -276,9 +282,15 @@ func TestWithdrawDelegationRewardsBasic(t *testing.T) {
 	tokens := sdk.DecCoins{{staking.DefaultBondDenom, sdk.NewDec(initial)}}
 	k.AllocateTokensToValidator(ctx, val, tokens)
 
+	// historical count should be 2 (initial + latest for delegation)
+	require.Equal(t, uint64(2), k.GetValidatorHistoricalRewardCount(ctx))
+
 	// withdraw rewards
 	require.Nil(t, k.WithdrawDelegationRewards(ctx, sdk.AccAddress(valOpAddr1), valOpAddr1))
 
+	// historical count should still be 2 (added one record, cleared one)
+	require.Equal(t, uint64(2), k.GetValidatorHistoricalRewardCount(ctx))
+
 	// assert correct balance
 	require.Equal(t, sdk.Coins{{staking.DefaultBondDenom, sdk.NewInt(balance - bond + (initial / 2))}}, ak.GetAccount(ctx, sdk.AccAddress(valOpAddr1)).GetCoins())
 
@@ -447,10 +459,16 @@ func TestCalculateRewardsMultiDelegatorMultWithdraw(t *testing.T) {
 	tokens := sdk.DecCoins{{staking.DefaultBondDenom, sdk.NewDec(initial)}}
 	k.AllocateTokensToValidator(ctx, val, tokens)
 
+	// historical count should be 2 (validator init, delegation init)
+	require.Equal(t, uint64(2), k.GetValidatorHistoricalRewardCount(ctx))
+
 	// second delegation
 	msg2 := staking.NewMsgDelegate(sdk.AccAddress(valOpAddr2), valOpAddr1, sdk.NewCoin(staking.DefaultBondDenom, sdk.NewInt(100)))
 	require.True(t, sh(ctx, msg2).IsOK())
 
+	// historical count should be 3 (second delegation init)
+	require.Equal(t, uint64(3), k.GetValidatorHistoricalRewardCount(ctx))
+
 	// fetch updated validator
 	val = sk.Validator(ctx, valOpAddr1)
 	del2 := sk.Delegation(ctx, sdk.AccAddress(valOpAddr2), valOpAddr1)
@@ -467,6 +485,9 @@ func TestCalculateRewardsMultiDelegatorMultWithdraw(t *testing.T) {
 	// second delegator withdraws
 	k.WithdrawDelegationRewards(ctx, sdk.AccAddress(valOpAddr2), valOpAddr1)
 
+	// historical count should be 3 (validator init + two delegations)
+	require.Equal(t, uint64(3), k.GetValidatorHistoricalRewardCount(ctx))
+
 	// validator withdraws commission
 	k.WithdrawValidatorCommission(ctx, valOpAddr1)
 

x/distribution/keeper/hooks.go

@@ -20,11 +20,41 @@ func (h Hooks) AfterValidatorCreated(ctx sdk.Context, valAddr sdk.ValAddress) {
 	h.k.initializeValidator(ctx, val)
 }
 func (h Hooks) BeforeValidatorModified(ctx sdk.Context, valAddr sdk.ValAddress) {
-	val := h.k.stakingKeeper.Validator(ctx, valAddr)
-	// increment period
-	h.k.incrementValidatorPeriod(ctx, val)
 }
 func (h Hooks) AfterValidatorRemoved(ctx sdk.Context, _ sdk.ConsAddress, valAddr sdk.ValAddress) {
+	// force-withdraw commission
+	commission := h.k.GetValidatorAccumulatedCommission(ctx, valAddr)
+	if !commission.IsZero() {
+		coins, remainder := commission.TruncateDecimal()
+
+		// remainder to community pool
+		feePool := h.k.GetFeePool(ctx)
+		feePool.CommunityPool = feePool.CommunityPool.Plus(remainder)
+		h.k.SetFeePool(ctx, feePool)
+
+		// update outstanding
+		outstanding := h.k.GetOutstandingRewards(ctx)
+		h.k.SetOutstandingRewards(ctx, outstanding.Minus(commission))
+
+		// add to validator account
+		accAddr := sdk.AccAddress(valAddr)
+		withdrawAddr := h.k.GetDelegatorWithdrawAddr(ctx, accAddr)
+
+		if _, _, err := h.k.bankKeeper.AddCoins(ctx, withdrawAddr, coins); err != nil {
+			panic(err)
+		}
+	}
+	// remove commission record
+	h.k.DeleteValidatorAccumulatedCommission(ctx, valAddr)
+
+	// clear slashes
+	h.k.DeleteValidatorSlashEvents(ctx, valAddr)
+
+	// clear historical rewards
+	h.k.DeleteValidatorHistoricalRewards(ctx, valAddr)
+
+	// clear current rewards
+	h.k.DeleteValidatorCurrentRewards(ctx, valAddr)
 }
 func (h Hooks) BeforeDelegationCreated(ctx sdk.Context, delAddr sdk.AccAddress, valAddr sdk.ValAddress) {
 	val := h.k.stakingKeeper.Validator(ctx, valAddr)
@@ -42,7 +72,7 @@ func (h Hooks) BeforeDelegationSharesModified(ctx sdk.Context, delAddr sdk.AccAd
 	}
 }
 func (h Hooks) BeforeDelegationRemoved(ctx sdk.Context, delAddr sdk.AccAddress, valAddr sdk.ValAddress) {
-	// nothing needed here since OnDelegationSharesModified will always also be called
+	// nothing needed here since BeforeDelegationSharesModified will always also be called
 }
 func (h Hooks) AfterDelegationModified(ctx sdk.Context, delAddr sdk.AccAddress, valAddr sdk.ValAddress) {
 	// create new delegation period record

x/distribution/keeper/key.go

@@ -112,6 +112,11 @@ func GetDelegatorStartingInfoKey(v sdk.ValAddress, d sdk.AccAddress) []byte {
 	return append(append(DelegatorStartingInfoPrefix, v.Bytes()...), d.Bytes()...)
 }
 
+// gets the prefix key for a validator's historical rewards
+func GetValidatorHistoricalRewardsPrefix(v sdk.ValAddress) []byte {
+	return append(ValidatorHistoricalRewardsPrefix, v.Bytes()...)
+}
+
 // gets the key for a validator's historical rewards
 func GetValidatorHistoricalRewardsKey(v sdk.ValAddress, k uint64) []byte {
 	b := make([]byte, 8)
@@ -129,6 +134,11 @@ func GetValidatorAccumulatedCommissionKey(v sdk.ValAddress) []byte {
 	return append(ValidatorAccumulatedCommissionPrefix, v.Bytes()...)
 }
 
+// gets the prefix key for a validator's slash fractions
+func GetValidatorSlashEventPrefix(v sdk.ValAddress) []byte {
+	return append(ValidatorSlashEventPrefix, v.Bytes()...)
+}
+
 // gets the key for a validator's slash fraction
 func GetValidatorSlashEventKey(v sdk.ValAddress, height uint64) []byte {
 	b := make([]byte, 8)

x/distribution/keeper/querier.go

@@ -3,10 +3,11 @@ package keeper
 import (
 	"fmt"
 
+	abci "github.com/tendermint/tendermint/abci/types"
+
 	"github.com/cosmos/cosmos-sdk/codec"
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	"github.com/cosmos/cosmos-sdk/x/distribution/types"
-	abci "github.com/tendermint/tendermint/abci/types"
 )
 
 // nolint

x/distribution/keeper/querier_test.go

@@ -6,11 +6,12 @@ import (
 
 	"github.com/stretchr/testify/require"
 
+	abci "github.com/tendermint/tendermint/abci/types"
+
 	"github.com/cosmos/cosmos-sdk/codec"
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	"github.com/cosmos/cosmos-sdk/x/distribution/types"
 	"github.com/cosmos/cosmos-sdk/x/staking"
-	abci "github.com/tendermint/tendermint/abci/types"
 )
 
 const custom = "custom"