Allow validator to receive and directly incorporate bundles of transactions

Makefile

@@ -7,6 +7,7 @@ BUILDDIR ?= $(CURDIR)/build
 INVARIANT_CHECK_INTERVAL ?= $(INVARIANT_CHECK_INTERVAL:-0)
 export PROJECT_HOME=$(shell git rev-parse --show-toplevel)
 export GO_PKG_PATH=$(HOME)/go/pkg
+export GO_MOD_CACHE=$(shell go env GOMODCACHE)
 export GO111MODULE = on
 
 # process build tags
@@ -52,12 +53,6 @@ ldflags = -X github.com/cosmos/cosmos-sdk/version.Name=sei \
 			-X github.com/cosmos/cosmos-sdk/version.Commit=$(COMMIT) \
 			-X "github.com/cosmos/cosmos-sdk/version.BuildTags=$(build_tags_comma_sep)"
 
-# go 1.23+ needs a workaround to link memsize (see https://github.com/fjl/memsize).
-# NOTE: this is a terribly ugly and unstable way of comparing version numbers,
-# but that's what you get when you do anything nontrivial in a Makefile.
-ifeq ($(firstword $(sort go1.23 $(shell go env GOVERSION))), go1.23)
-	ldflags += -checklinkname=0
-endif
 ifeq ($(LINK_STATICALLY),true)
 	ldflags += -linkmode=external -extldflags "-Wl,-z,muldefs -static"
 endif
@@ -199,6 +194,12 @@ docker-cluster-start: docker-cluster-stop build-docker-node
 	@mkdir -p $(shell go env GOCACHE)
 	@cd docker && USERID=$(shell id -u) GROUPID=$(shell id -g) GOCACHE=$(shell go env GOCACHE) NUM_ACCOUNTS=10 INVARIANT_CHECK_INTERVAL=${INVARIANT_CHECK_INTERVAL} UPGRADE_VERSION_LIST=${UPGRADE_VERSION_LIST} docker compose up
 
+docker-mev-cluster-start: docker-mev-cluster-stop build-docker-node
+	@rm -rf $(PROJECT_HOME)/build/generated
+	@mkdir -p $(shell go env GOPATH)/pkg/mod
+	@mkdir -p $(shell go env GOCACHE)
+	@cd docker && USERID=$(shell id -u) GROUPID=$(shell id -g) GOCACHE=$(shell go env GOCACHE) NUM_ACCOUNTS=10 INVARIANT_CHECK_INTERVAL=${INVARIANT_CHECK_INTERVAL} UPGRADE_VERSION_LIST=${UPGRADE_VERSION_LIST} docker compose -f docker-compose.yml -f docker-compose.mev.yml up
+
 .PHONY: localnet-start
 
 # Use this to skip the seid build process
@@ -210,7 +211,11 @@ docker-cluster-start-skipbuild: docker-cluster-stop build-docker-node
 # Stop 4-node docker containers
 docker-cluster-stop:
 	@cd docker && USERID=$(shell id -u) GROUPID=$(shell id -g) GOCACHE=$(shell go env GOCACHE) docker compose down
-.PHONY: localnet-stop ddd
+
+docker-mev-cluster-stop:
+	@cd docker && USERID=$(shell id -u) GROUPID=$(shell id -g) GOCACHE=$(shell go env GOCACHE) docker compose -f docker-compose.yml -f docker-compose.mev.yml down
+
+.PHONY: localnet-stop
 
 
 # Implements test splitting and running. This is pulled directly from
@@ -233,4 +238,4 @@ $(BUILDDIR)/packages.txt:$(GO_TEST_FILES) $(BUILDDIR)
 split-test-packages:$(BUILDDIR)/packages.txt
 	split -d -n l/$(NUM_SPLIT) $< $<.
 test-group-%:split-test-packages
-	cat $(BUILDDIR)/packages.txt.$* | xargs go test -parallel 4 -mod=readonly -timeout=15m -race -coverprofile=$*.profile.out -covermode=atomic
+	cat $(BUILDDIR)/packages.txt.$* | xargs go test -parallel 4 -mod=readonly -timeout=10m -race -coverprofile=$*.profile.out -covermode=atomic

aclmapping/oracle/mappings_test.go

@@ -129,7 +129,7 @@ func TestMsgVoteDependencyGenerator(t *testing.T) {
 	tm := time.Now().UTC()
 	valPub := secp256k1.GenPrivKey().PubKey()
 
-	testWrapper := app.NewTestWrapper(t, tm, valPub, false)
+	testWrapper := app.NewTestWrapper(t, tm, valPub, false, false)
 
 	oracleVote := oracletypes.MsgAggregateExchangeRateVote{
 		ExchangeRates: "1usei",
@@ -147,7 +147,7 @@ func TestMsgVoteDependencyGeneratorInvalidMsgType(t *testing.T) {
 	tm := time.Now().UTC()
 	valPub := secp256k1.GenPrivKey().PubKey()
 
-	testWrapper := app.NewTestWrapper(t, tm, valPub, false)
+	testWrapper := app.NewTestWrapper(t, tm, valPub, false, false)
 	_, err := oracleacl.MsgVoteDependencyGenerator(testWrapper.App.AccessControlKeeper, testWrapper.Ctx, &banktypes.MsgSend{})
 	require.Error(t, err)
 }

aclmapping/staking/mappings_test.go

@@ -279,7 +279,7 @@ func (suite *KeeperTestSuite) TestMsgDelegateDependencies() {
 func TestGeneratorInvalidMessageTypes(t *testing.T) {
 	tm := time.Now().UTC()
 	valPub := secp256k1.GenPrivKey().PubKey()
-	testWrapper := app.NewTestWrapper(t, tm, valPub, false)
+	testWrapper := app.NewTestWrapper(t, tm, valPub, false, false)
 
 	stakingDelegate := stakingtypes.MsgDelegate{
 		DelegatorAddress: "delegator",

app/app.go

@@ -160,6 +160,10 @@ import (
 	// unnamed import of statik for openapi/swagger UI support
 	_ "github.com/sei-protocol/sei-chain/docs/swagger"
 	ssconfig "github.com/sei-protocol/sei-db/config"
+
+	mevbase "github.com/sei-protocol/sei-chain/mev"
+	mev "github.com/sei-protocol/sei-chain/x/mev"
+	mevtypes "github.com/sei-protocol/sei-chain/x/mev/types"
 )
 
 // this line is used by starport scaffolding # stargate/wasm/app/enabledProposals
@@ -212,6 +216,7 @@ var (
 		epochmodule.AppModuleBasic{},
 		tokenfactorymodule.AppModuleBasic{},
 		ctmodule.AppModuleBasic{},
+		mev.AppModuleBasic{},
 		// this line is used by starport scaffolding # stargate/app/moduleBasic
 	)
 
@@ -230,8 +235,8 @@ var (
 		evmtypes.ModuleName:            {authtypes.Minter, authtypes.Burner},
 		tokenfactorytypes.ModuleName:   {authtypes.Minter, authtypes.Burner},
 		// Confidential Transfers module is not live yet, but we add the permissions for testing
-		cttypes.ModuleName: nil,
-
+		cttypes.ModuleName:  nil,
+		mevtypes.ModuleName: nil,
 		// this line is used by starport scaffolding # stargate/app/maccPerms
 	}
 
@@ -342,6 +347,7 @@ type App struct {
 	WasmKeeper          wasm.Keeper
 	OracleKeeper        oraclekeeper.Keeper
 	EvmKeeper           evmkeeper.Keeper
+	MevKeeper           *mevbase.Keeper // pointer as this is used internally only, and synced via Mutex
 
 	// make scoped keepers public for test purposes
 	ScopedIBCKeeper      capabilitykeeper.ScopedKeeper
@@ -379,6 +385,7 @@ type App struct {
 
 	encodingConfig        appparams.EncodingConfig
 	evmRPCConfig          evmrpc.Config
+	mevConfig             mevbase.Config
 	lightInvarianceConfig LightInvarianceConfig
 
 	genesisImportConfig genesistypes.GenesisImportConfig
@@ -570,6 +577,13 @@ func New(
 		app.AccountKeeper,
 		app.BankKeeper)
 
+	keys[mevtypes.StoreKey] = storetypes.NewKVStoreKey(mevtypes.StoreKey)
+
+	app.MevKeeper = mevbase.NewKeeper(
+		appCodec,
+		keys[mevtypes.StoreKey],
+	)
+
 	// The last arguments can contain custom message handlers, and custom query handlers,
 	// if we want to allow any custom callbacks
 	supportedFeatures := "iterator,staking,stargate,sei"
@@ -637,6 +651,10 @@ func New(
 	if err != nil {
 		panic(fmt.Sprintf("error reading EVM config due to %s", err))
 	}
+	app.mevConfig, err = mevbase.ReadConfig(appOpts)
+	if err != nil {
+		panic(fmt.Sprintf("error reading MEV config due to %s", err))
+	}
 	evmQueryConfig, err := querier.ReadConfig(appOpts)
 	if err != nil {
 		panic(fmt.Sprintf("error reading evm query config due to %s", err))
@@ -777,6 +795,7 @@ func New(
 		authzmodule.NewAppModule(appCodec, app.AuthzKeeper, app.AccountKeeper, app.BankKeeper, app.interfaceRegistry),
 		ctmodule.NewAppModule(app.ConfidentialTransfersKeeper),
 		// this line is used by starport scaffolding # stargate/app/appModule
+		mev.NewAppModule(appCodec, app.MevKeeper),
 	)
 
 	// During begin block slashing happens after distr.BeginBlocker so that
@@ -809,6 +828,7 @@ func New(
 		tokenfactorytypes.ModuleName,
 		cttypes.ModuleName,
 		acltypes.ModuleName,
+		mevtypes.ModuleName,
 	)
 
 	app.mm.SetOrderMidBlockers(
@@ -841,6 +861,7 @@ func New(
 		tokenfactorytypes.ModuleName,
 		cttypes.ModuleName,
 		acltypes.ModuleName,
+		mevtypes.ModuleName,
 	)
 
 	// NOTE: The genutils module must occur after staking so that pools are
@@ -874,6 +895,7 @@ func New(
 		wasm.ModuleName,
 		evmtypes.ModuleName,
 		acltypes.ModuleName,
+		mevtypes.ModuleName,
 		// this line is used by starport scaffolding # stargate/app/initGenesis
 	)
 
@@ -1136,14 +1158,111 @@ func (app *App) InitChainer(ctx sdk.Context, req abci.RequestInitChain) abci.Res
 	return app.mm.InitGenesis(ctx, app.appCodec, genesisState, app.genesisImportConfig)
 }
 
-func (app *App) PrepareProposalHandler(_ sdk.Context, req *abci.RequestPrepareProposal) (*abci.ResponsePrepareProposal, error) {
+func (app *App) PrepareProposalHandler(ctx sdk.Context, req *abci.RequestPrepareProposal) (*abci.ResponsePrepareProposal, error) {
+	// Get all pending bundles for this height
+	ctx.Logger().Debug("Preparing proposal", "height", ctx.BlockHeight())
+
+	if app.mevConfig.Enabled {
+		return app.mevPrepareProposalHandler(ctx, req)
+	}
+
 	return &abci.ResponsePrepareProposal{
 		TxRecords: utils.Map(req.Txs, func(tx []byte) *abci.TxRecord {
 			return &abci.TxRecord{Action: abci.TxRecord_UNMODIFIED, Tx: tx}
 		}),
 	}, nil
 }
 
+func (app *App) mevPrepareProposalHandler(ctx sdk.Context, req *abci.RequestPrepareProposal) (*abci.ResponsePrepareProposal, error) {
+	bundles := app.MevKeeper.PendingBundles(ctx.BlockHeight())
+
+	if len(bundles) > 0 {
+		ctx.Logger().Debug("found pending bundles from mevkeeper", "count", len(bundles), "height", ctx.BlockHeight())
+	}
+
+	maxTxBytes := req.MaxTxBytes
+	selectedTxs := make([]*abci.TxRecord, 0)
+
+	var selectedTxsTotalSize = int64(0)
+	var remainingTxs [][]byte
+	// map tx hash to index in remainingTxs
+	var remainingTxsMap = make(map[string]int)
+
+	// First, add any system transactions (governance, etc.)
+	for _, tx := range req.Txs {
+		if app.isSystemTx(tx) {
+			selectedTxs = append(selectedTxs, &abci.TxRecord{
+				Action: abci.TxRecord_UNMODIFIED,
+				Tx:     tx,
+			})
+
+			selectedTxsTotalSize += int64(len(tx))
+		} else {
+			remainingTxs = append(remainingTxs, tx)
+			remainingTxsMap[string(tx)] = len(remainingTxs) - 1
+		}
+	}
+
+	// Next, add bundle transactions with the highest priority
+	for _, bundle := range bundles {
+		// Skip bundles not meant for this height
+		if bundle.BlockHeight != uint64(ctx.BlockHeight()) {
+			continue
+		}
+
+		// Calculate total size of this bundle
+		bundleSize := int64(0)
+		for _, txStr := range bundle.Transactions {
+			bundleSize += int64(len(txStr))
+		}
+
+		// Check if entire bundle fits
+		if selectedTxsTotalSize+bundleSize <= maxTxBytes {
+			// Add all transactions from this bundle
+			for _, tx := range bundle.Transactions {
+				selectedTxs = append(selectedTxs, &abci.TxRecord{
+					Action: abci.TxRecord_UNMODIFIED,
+					Tx:     tx,
+				})
+
+				//remove tx from remaining if it was already in a bundle
+				index, has := remainingTxsMap[string(tx)]
+				if has {
+					remainingTxs = append(remainingTxs[:index], remainingTxs[index+1:]...)
+					delete(remainingTxsMap, string(tx))
+				}
+			}
+			selectedTxsTotalSize += bundleSize
+		}
+	}
+
+	// Finally, add remaining transactions up to size limit
+	for _, tx := range remainingTxs {
+		if selectedTxsTotalSize+int64(len(tx)) <= maxTxBytes {
+			selectedTxs = append(selectedTxs, &abci.TxRecord{
+				Action: abci.TxRecord_UNMODIFIED,
+				Tx:     tx,
+			})
+			selectedTxsTotalSize += int64(len(tx))
+			if selectedTxsTotalSize >= maxTxBytes {
+				break
+			}
+		}
+	}
+
+	return &abci.ResponsePrepareProposal{
+		TxRecords: selectedTxs,
+	}, nil
+}
+
+func (app *App) isSystemTx(tx []byte) bool {
+	// Implement system transaction detection logic
+	// This could check for specific message types that should always be included
+	// like governance votes, IBC packets, etc.
+	// TODO get SEI team input, which tx should be put before bundles in a block
+	return false
+}
+
 func (app *App) GetOptimisticProcessingInfo() *OptimisticProcessingInfo {
 	return app.optimisticProcessingInfo
 }
@@ -1893,6 +2012,14 @@ func (app *App) RegisterTendermintService(clientCtx client.Context) {
 			panic(err)
 		}
 	}
+
+	if app.mevConfig.Enabled {
+		_, err := mevbase.NewPoller(app.GetCheckCtx().Context(), app.Logger(), app.mevConfig, app.MevKeeper, app.LastBlockHeight)
+		if err != nil {
+			panic(err)
+		}
+	}
+
 }
 
 // RegisterSwaggerAPI registers swagger route with API Server
@@ -2018,6 +2145,7 @@ func initParamsKeeper(appCodec codec.BinaryCodec, legacyAmino *codec.LegacyAmino
 	paramsKeeper.Subspace(epochmoduletypes.ModuleName)
 	paramsKeeper.Subspace(tokenfactorytypes.ModuleName)
 	paramsKeeper.Subspace(cttypes.ModuleName)
+	paramsKeeper.Subspace(mevtypes.ModuleName)
 	// this line is used by starport scaffolding # stargate/app/paramSubspace
 
 	return paramsKeeper