Problem: transient store usage not compatible with parallel tx execution

Currently we use shared transient store keys to accumulate some states,
which cause issues when developing parallel tx execution

Solution:
- remove some transient stores.
- the others are used in a per-tx fasion.

Update CHANGELOG.md

Signed-off-by: yihuang <huang@crypto.com>

cleanup

fix test

Author: HuangYi

CHANGELOG.md

@@ -49,6 +49,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
 * (statedb) [#446](https://github.com/crypto-org-chain/ethermint/pull/446) Re-use the cache store implementation with sdk.
 * (evm) [#447](https://github.com/crypto-org-chain/ethermint/pull/447) Deduct fee through virtual bank transfer.
 * (evm) [#448](https://github.com/crypto-org-chain/ethermint/pull/448) Refactor the evm transfer to be more efficient.
+* (evm) [#450](https://github.com/crypto-org-chain/ethermint/pull/450) Refactor transient stores to be compatible with parallel tx execution.
 
 ### State Machine Breaking
 

app/ante/handler_options.go

@@ -87,7 +87,7 @@ func newEthAnteHandler(options HandlerOptions) sdk.AnteHandler {
 		}
 
 		// We need to setup an empty gas config so that the gas is consistent with Ethereum.
-		ctx, err = SetupEthContext(ctx, options.EvmKeeper)
+		ctx, err = SetupEthContext(ctx)
 		if err != nil {
 			return ctx, err
 		}

app/ante/interfaces.go

@@ -32,8 +32,6 @@ type EVMKeeper interface {
 	ChainID() *big.Int
 
 	DeductTxCostsFromUserBalance(ctx sdk.Context, fees sdk.Coins, from common.Address) error
-	ResetTransientGasUsed(ctx sdk.Context)
-	GetTxIndexTransient(ctx sdk.Context) uint64
 }
 
 type protoTxProvider interface {

app/ante/setup.go

@@ -30,16 +30,12 @@ import (
 
 // SetupEthContext is adapted from SetUpContextDecorator from cosmos-sdk, it ignores gas consumption
 // by setting the gas meter to infinite
-func SetupEthContext(ctx sdk.Context, evmKeeper EVMKeeper) (newCtx sdk.Context, err error) {
+func SetupEthContext(ctx sdk.Context) (newCtx sdk.Context, err error) {
 	// We need to setup an empty gas config so that the gas is consistent with Ethereum.
 	newCtx = ctx.WithGasMeter(storetypes.NewInfiniteGasMeter()).
 		WithKVGasConfig(storetypes.GasConfig{}).
 		WithTransientKVGasConfig(storetypes.GasConfig{})
 
-	// Reset transient gas used to prepare the execution of current cosmos tx.
-	// Transient gas-used is necessary to sum the gas-used of cosmos tx, when it contains multiple eth msgs.
-	evmKeeper.ResetTransientGasUsed(ctx)
-
 	return newCtx, nil
 }
 

app/ante/setup_test.go

@@ -28,7 +28,7 @@ func (suite *AnteTestSuite) TestEthSetupContextDecorator() {
 
 	for _, tc := range testCases {
 		suite.Run(tc.name, func() {
-			ctx, err := ante.SetupEthContext(suite.ctx, suite.app.EvmKeeper)
+			ctx, err := ante.SetupEthContext(suite.ctx)
 
 			if tc.expPass {
 				suite.Require().NoError(err)

app/app.go

@@ -314,6 +314,7 @@ func NewEthermintApp(
 	bApp.SetVersion(version.Version)
 	bApp.SetInterfaceRegistry(interfaceRegistry)
 	bApp.SetTxEncoder(txConfig.TxEncoder())
+	bApp.SetTxExecutor(DefaultTxExecutor)
 
 	keys := storetypes.NewKVStoreKeys(
 		// SDK keys

app/executor.go

@@ -0,0 +1,21 @@
+package app
+
+import (
+	"context"
+
+	storetypes "cosmossdk.io/store/types"
+	abci "github.com/cometbft/cometbft/abci/types"
+	evmtypes "github.com/evmos/ethermint/x/evm/types"
+)
+
+func DefaultTxExecutor(_ context.Context,
+	blockSize int,
+	ms storetypes.MultiStore,
+	deliverTxWithMultiStore func(int, storetypes.MultiStore) *abci.ExecTxResult,
+) ([]*abci.ExecTxResult, error) {
+	results := make([]*abci.ExecTxResult, blockSize)
+	for i := 0; i < blockSize; i++ {
+		results[i] = deliverTxWithMultiStore(i, ms)
+	}
+	return evmtypes.PatchTxResponses(results), nil
+}

tests/importer/importer_test.go

@@ -144,10 +144,10 @@ func (suite *ImporterTestSuite) TestImportBlocks() {
 			applyDAOHardFork(vmdb)
 		}
 
-		for _, tx := range block.Transactions() {
+		for i, tx := range block.Transactions() {
 
 			receipt, gas, err := applyTransaction(
-				ctx, chainConfig, chainContext, nil, gp, suite.app.EvmKeeper, vmdb, header, tx, usedGas, vmConfig,
+				ctx, chainConfig, chainContext, nil, gp, suite.app.EvmKeeper, vmdb, header, tx, usedGas, vmConfig, uint(i)
 			)
 			suite.Require().NoError(err, "failed to apply tx at block %d; tx: %X; gas %d; receipt:%v", block.NumberU64(), tx.Hash(), gas, receipt)
 			suite.Require().NotNil(receipt)
@@ -230,7 +230,7 @@ func applyDAOHardFork(vmdb ethvm.StateDB) {
 func applyTransaction(
 	ctx sdk.Context, config *ethparams.ChainConfig, bc ethcore.ChainContext, author *common.Address,
 	gp *ethcore.GasPool, evmKeeper *evmkeeper.Keeper, vmdb *statedb.StateDB, header *ethtypes.Header,
-	tx *ethtypes.Transaction, usedGas *uint64, cfg ethvm.Config,
+	tx *ethtypes.Transaction, usedGas *uint64, cfg ethvm.Config, index uint,
 ) (*ethtypes.Receipt, uint64, error) {
 	msg, err := ethcore.TransactionToMessage(tx, ethtypes.MakeSigner(config, header.Number), sdkmath.ZeroInt().BigInt())
 	if err != nil {
@@ -271,7 +271,7 @@ func applyTransaction(
 	receipt.Bloom = ethtypes.CreateBloom(ethtypes.Receipts{receipt})
 	receipt.BlockHash = header.Hash()
 	receipt.BlockNumber = header.Number
-	receipt.TransactionIndex = uint(evmKeeper.GetTxIndexTransient(ctx))
+	receipt.TransactionIndex = index
 
 	return receipt, execResult.UsedGas, err
 }

x/evm/keeper/abci.go

@@ -19,8 +19,6 @@ import (
 	"cosmossdk.io/store/types"
 
 	sdk "github.com/cosmos/cosmos-sdk/types"
-
-	ethtypes "github.com/ethereum/go-ethereum/core/types"
 )
 
 // BeginBlock sets the sdk Context and EIP155 chain id to the Keeper.
@@ -35,9 +33,6 @@ func (k *Keeper) BeginBlock(ctx sdk.Context) error {
 func (k *Keeper) EndBlock(ctx sdk.Context) error {
 	// Gas costs are handled within msg handler so costs should be ignored
 	infCtx := ctx.WithGasMeter(types.NewInfiniteGasMeter())
-
-	bloom := ethtypes.BytesToBloom(k.GetBlockBloomTransient(infCtx).Bytes())
-	k.EmitBlockBloomEvent(infCtx, bloom)
-
+	k.CollectTxBloom(infCtx)
 	return nil
 }

x/evm/keeper/bloom.go

@@ -0,0 +1,27 @@
+package keeper
+
+import (
+	"math/big"
+
+	"cosmossdk.io/store/prefix"
+	sdk "github.com/cosmos/cosmos-sdk/types"
+	"github.com/evmos/ethermint/x/evm/types"
+)
+
+func (k Keeper) SetTxBloom(ctx sdk.Context, bloom []byte) {
+	store := ctx.KVStore(k.transientKey)
+	store.Set(types.TransientBloomKey(ctx.TxIndex(), ctx.MsgIndex()), bloom)
+}
+
+func (k Keeper) CollectTxBloom(ctx sdk.Context) {
+	store := prefix.NewStore(ctx.KVStore(k.transientKey), types.KeyPrefixTransientBloom)
+	it := store.Iterator(nil, nil)
+	defer it.Close()
+
+	bloom := new(big.Int)
+	for ; it.Valid(); it.Next() {
+		bloom.Or(bloom, big.NewInt(0).SetBytes(it.Value()))
+	}
+
+	k.EmitBlockBloomEvent(ctx, bloom.Bytes())
+}