Skip to content

Add EVM execution microbenchmarks via go test -bench #2901

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
pdrobnjak wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

Add EVM execution microbenchmarks via go test -bench #2901

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 9 additions & 13 deletions app/test_helpers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,12 +121,12 @@ func NewTestWrapper(tb testing.TB, tm time.Time, valPub cryptotypes.PubKey, enab
return newTestWrapper(tb, tm, valPub, enableEVMCustomPrecompiles, false, TestAppOpts{}, baseAppOptions...)
}

func NewTestWrapperWithSc(t *testing.T, tm time.Time, valPub cryptotypes.PubKey, enableEVMCustomPrecompiles bool, baseAppOptions ...func(*bam.BaseApp)) *TestWrapper {
return newTestWrapper(t, tm, valPub, enableEVMCustomPrecompiles, true, TestAppOpts{UseSc: true}, baseAppOptions...)
func NewTestWrapperWithSc(tb testing.TB, tm time.Time, valPub cryptotypes.PubKey, enableEVMCustomPrecompiles bool, baseAppOptions ...func(*bam.BaseApp)) *TestWrapper {
return newTestWrapper(tb, tm, valPub, enableEVMCustomPrecompiles, true, TestAppOpts{UseSc: true}, baseAppOptions...)
}

func NewGigaTestWrapper(t *testing.T, tm time.Time, valPub cryptotypes.PubKey, enableEVMCustomPrecompiles bool, useOcc bool, baseAppOptions ...func(*bam.BaseApp)) *TestWrapper {
wrapper := newTestWrapper(t, tm, valPub, enableEVMCustomPrecompiles, true, TestAppOpts{UseSc: true, EnableGiga: true, EnableGigaOCC: useOcc}, baseAppOptions...)
func NewGigaTestWrapper(tb testing.TB, tm time.Time, valPub cryptotypes.PubKey, enableEVMCustomPrecompiles bool, useOcc bool, baseAppOptions ...func(*bam.BaseApp)) *TestWrapper {
wrapper := newTestWrapper(tb, tm, valPub, enableEVMCustomPrecompiles, true, TestAppOpts{UseSc: true, EnableGiga: true, EnableGigaOCC: useOcc}, baseAppOptions...)
genState := evmtypes.DefaultGenesis()
wrapper.App.EvmKeeper.InitGenesis(wrapper.Ctx, *genState)
return wrapper
Expand All @@ -140,9 +140,9 @@ func NewGigaTestWrapper(t *testing.T, tm time.Time, valPub cryptotypes.PubKey, e
// - Creates app with UseSc=true but EnableGiga=false (so GigaKVStore is NOT registered)
// - Manually enables Giga executor flags on the app
// - Sets GigaEvmKeeper.UseRegularStore=true so it uses ctx.KVStore instead of ctx.GigaKVStore
func NewGigaTestWrapperWithRegularStore(t *testing.T, tm time.Time, valPub cryptotypes.PubKey, enableEVMCustomPrecompiles bool, useOcc bool, baseAppOptions ...func(*bam.BaseApp)) *TestWrapper {
func NewGigaTestWrapperWithRegularStore(tb testing.TB, tm time.Time, valPub cryptotypes.PubKey, enableEVMCustomPrecompiles bool, useOcc bool, baseAppOptions ...func(*bam.BaseApp)) *TestWrapper {
// Create wrapper with Sc but WITHOUT EnableGiga - this means GigaKVStore won't be registered
wrapper := newTestWrapper(t, tm, valPub, enableEVMCustomPrecompiles, true, TestAppOpts{UseSc: true, EnableGiga: false, EnableGigaOCC: false}, baseAppOptions...)
wrapper := newTestWrapper(tb, tm, valPub, enableEVMCustomPrecompiles, true, TestAppOpts{UseSc: true, EnableGiga: false, EnableGigaOCC: false}, baseAppOptions...)

// Manually enable Giga executor on the app
wrapper.App.GigaExecutorEnabled = true
Expand Down Expand Up @@ -179,11 +179,7 @@ func newTestWrapper(tb testing.TB, tm time.Time, valPub cryptotypes.PubKey, enab
DefaultNodeHome = originalHome
})
if UseSc {
if testT, ok := tb.(*testing.T); ok {
appPtr = SetupWithSc(testT, false, enableEVMCustomPrecompiles, testAppOpts, baseAppOptions...)
} else {
panic("SetupWithSc requires *testing.T, cannot use with *testing.B")
}
appPtr = SetupWithSc(tb, false, enableEVMCustomPrecompiles, testAppOpts, baseAppOptions...)
} else {
appPtr = Setup(tb, false, enableEVMCustomPrecompiles, false, baseAppOptions...)
}
Expand Down Expand Up @@ -471,7 +467,7 @@ func SetupWithDB(tb testing.TB, db dbm.DB, isCheckTx bool, enableEVMCustomPrecom
return res
}

func SetupWithSc(t *testing.T, isCheckTx bool, enableEVMCustomPrecompiles bool, testAppOpts TestAppOpts, baseAppOptions ...func(*bam.BaseApp)) (res *App) {
func SetupWithSc(tb testing.TB, isCheckTx bool, enableEVMCustomPrecompiles bool, testAppOpts TestAppOpts, baseAppOptions ...func(*bam.BaseApp)) (res *App) {
db := dbm.NewMemDB()
encodingConfig := MakeEncodingConfig()
cdc := encodingConfig.Marshaler
Expand All @@ -492,7 +488,7 @@ func SetupWithSc(t *testing.T, isCheckTx bool, enableEVMCustomPrecompiles bool,
nil,
true,
map[int64]bool{},
t.TempDir(),
tb.TempDir(),
1,
enableEVMCustomPrecompiles,
config.TestConfig(),
Expand Down
131 changes: 131 additions & 0 deletions occ_tests/evm_benchmark_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
package occ

import (
"testing"
"time"

"github.com/cosmos/cosmos-sdk/server/config"
"github.com/sei-protocol/sei-chain/occ_tests/messages"
"github.com/sei-protocol/sei-chain/occ_tests/utils"
)

type benchConfig struct {
name string
workers int
occEnabled bool
}

var occConfigs = []benchConfig{
{name: "occ", workers: config.DefaultConcurrencyWorkers, occEnabled: true},
{name: "sequential", workers: 1, occEnabled: false},
}

func runEVMBenchmark(b *testing.B, txCount int, cfg benchConfig, genTxs func(tCtx *utils.TestContext) []*utils.TestMessage) {
b.Helper()
blockTime := time.Now()
accts := utils.NewTestAccounts(5)

var totalGasUsed int64
var totalTimedDuration time.Duration

b.ResetTimer()
for i := 0; i < b.N; i++ {
b.StopTimer()
iterCtx := utils.NewTestContext(b, accts, blockTime, cfg.workers, cfg.occEnabled)
txMsgs := genTxs(iterCtx)
txs := utils.ToTxBytes(iterCtx, txMsgs)
b.StartTimer()

startTime := time.Now()
_, txResults, _, err := utils.ProcessBlockDirect(iterCtx, txs, cfg.occEnabled)
timedDuration := time.Since(startTime)
totalTimedDuration += timedDuration

if err != nil {
b.Fatalf("ProcessBlock error: %v", err)
}
if len(txResults) != len(txs) {
b.Fatalf("expected %d tx results, got %d", len(txs), len(txResults))
}

for _, result := range txResults {
totalGasUsed += result.GasUsed
}
}

b.ReportMetric(float64(txCount), "txns/op")
avgGasPerOp := float64(totalGasUsed) / float64(b.N)
b.ReportMetric(avgGasPerOp, "gas/op")
avgTimeSeconds := totalTimedDuration.Seconds() / float64(b.N)
if avgTimeSeconds > 0 {
b.ReportMetric(avgGasPerOp/avgTimeSeconds, "gas/sec")
b.ReportMetric(float64(txCount)/avgTimeSeconds, "tps")
b.ReportMetric(avgTimeSeconds/float64(txCount)*1e9, "ns/tx")
}
}

func BenchmarkEVMTransfer(b *testing.B) {
txCounts := []int{100, 500, 1000, 5000}
for _, tc := range txCounts {
for _, cfg := range occConfigs {
tc := tc
cfg := cfg
b.Run(benchName(tc, cfg.name), func(b *testing.B) {
runEVMBenchmark(b, tc, cfg, func(tCtx *utils.TestContext) []*utils.TestMessage {
return utils.JoinMsgs(messages.EVMTransferNonConflicting(tCtx, tc))
})
})
}
}
}

func BenchmarkEVMTransferConflicting(b *testing.B) {
txCounts := []int{100, 1000}
for _, tc := range txCounts {
for _, cfg := range occConfigs {
tc := tc
cfg := cfg
b.Run(benchName(tc, cfg.name), func(b *testing.B) {
runEVMBenchmark(b, tc, cfg, func(tCtx *utils.TestContext) []*utils.TestMessage {
return utils.JoinMsgs(messages.EVMTransferConflicting(tCtx, tc))
})
})
}
}
}

func BenchmarkEVMTransferMixed(b *testing.B) {
txCounts := []int{1000}
for _, tc := range txCounts {
for _, cfg := range occConfigs {
tc := tc
cfg := cfg
half := tc / 2
b.Run(benchName(tc, cfg.name), func(b *testing.B) {
runEVMBenchmark(b, tc, cfg, func(tCtx *utils.TestContext) []*utils.TestMessage {
conflicting := messages.EVMTransferConflicting(tCtx, half)
nonConflicting := messages.EVMTransferNonConflicting(tCtx, tc-half)
return utils.Shuffle(utils.JoinMsgs(conflicting, nonConflicting))
})
})
}
}
}

func benchName(txCount int, mode string) string {
return "txs_" + itoa(txCount) + "/" + mode
}

func itoa(n int) string {
if n == 0 {
return "0"
}
buf := [20]byte{}
i := len(buf)
for n > 0 {
i--
buf[i] = byte('0' + n%10)
n /= 10
}
return string(buf[i:])
}
Loading