Add limit to filter logs (#1558)

Author: codchen

cmd/seid/cmd/root.go

@@ -496,6 +496,15 @@ checktx_timeout = "{{ .EVM.CheckTxTimeout }}"
 # controls whether to have txns go through one by one
 slow = {{ .EVM.Slow }}
 
+# Deny list defines list of methods that EVM RPC should fail fast
+deny_list = {{ .EVM.DenyList }}
+
+# max number of logs returned if block range is open-ended
+max_log_no_block = {{ .EVM.MaxLogNoBlock }}
+
+# max number of blocks to query logs for
+max_blocks_for_log = {{ .EVM.MaxBlocksForLog }}
+
 [eth_replay]
 eth_replay_enabled = {{ .ETHReplay.Enabled }}
 eth_rpc = "{{ .ETHReplay.EthRPC }}"

evmrpc/config.go

@@ -73,6 +73,12 @@ type Config struct {
 
 	// Deny list defines list of methods that EVM RPC should fail fast
 	DenyList []string `mapstructure:"deny_list"`
+
+	// max number of logs returned if block range is open-ended
+	MaxLogNoBlock int64 `mapstructure:"max_log_no_block"`
+
+	// max number of blocks to query logs for
+	MaxBlocksForLog int64 `mapstructure:"max_blocks_for_log"`
 }
 
 var DefaultConfig = Config{
@@ -93,6 +99,8 @@ var DefaultConfig = Config{
 	MaxTxPoolTxs:         1000,
 	Slow:                 false,
 	DenyList:             make([]string, 0),
+	MaxLogNoBlock:        10000,
+	MaxBlocksForLog:      2000,
 }
 
 const (
@@ -113,6 +121,8 @@ const (
 	flagCheckTxTimeout       = "evm.checktx_timeout"
 	flagSlow                 = "evm.slow"
 	flagDenyList             = "evm.deny_list"
+	flagMaxLogNoBlock        = "evm.max_log_no_block"
+	flagMaxBlocksForLog      = "evm.max_blocks_for_log"
 )
 
 func ReadConfig(opts servertypes.AppOptions) (Config, error) {
@@ -203,5 +213,15 @@ func ReadConfig(opts servertypes.AppOptions) (Config, error) {
 			return cfg, err
 		}
 	}
+	if v := opts.Get(flagMaxLogNoBlock); v != nil {
+		if cfg.MaxLogNoBlock, err = cast.ToInt64E(v); err != nil {
+			return cfg, err
+		}
+	}
+	if v := opts.Get(flagMaxBlocksForLog); v != nil {
+		if cfg.MaxBlocksForLog, err = cast.ToInt64E(v); err != nil {
+			return cfg, err
+		}
+	}
 	return cfg, nil
 }

evmrpc/config_test.go

@@ -26,6 +26,8 @@ type opts struct {
 	maxTxPoolTxs         interface{}
 	slow                 interface{}
 	denyList             interface{}
+	maxLogNoBlock        interface{}
+	maxBlocksForLog      interface{}
 }
 
 func (o *opts) Get(k string) interface{} {
@@ -80,6 +82,12 @@ func (o *opts) Get(k string) interface{} {
 	if k == "evm.deny_list" {
 		return o.denyList
 	}
+	if k == "evm.max_log_no_block" {
+		return o.maxLogNoBlock
+	}
+	if k == "evm.max_blocks_for_log" {
+		return o.maxBlocksForLog
+	}
 	panic("unknown key")
 }
 
@@ -102,6 +110,8 @@ func TestReadConfig(t *testing.T) {
 		1000,
 		false,
 		make([]string, 0),
+		20000,
+		1000,
 	}
 	_, err := evmrpc.ReadConfig(&goodOpts)
 	require.Nil(t, err)

evmrpc/filter.go

@@ -51,7 +51,9 @@ type FilterAPI struct {
 }
 
 type FilterConfig struct {
-	timeout time.Duration
+	timeout  time.Duration
+	maxLog   int64
+	maxBlock int64
 }
 
 type EventItemDataWrapper struct {
@@ -60,6 +62,7 @@ type EventItemDataWrapper struct {
 }
 
 func NewFilterAPI(tmClient rpcclient.Client, logFetcher *LogFetcher, filterConfig *FilterConfig) *FilterAPI {
+	logFetcher.filterConfig = filterConfig
 	filters := make(map[ethrpc.ID]filter)
 	api := &FilterAPI{
 		tmClient:     tmClient,
@@ -261,9 +264,10 @@ func (a *FilterAPI) UninstallFilter(
 }
 
 type LogFetcher struct {
-	tmClient    rpcclient.Client
-	k           *keeper.Keeper
-	ctxProvider func(int64) sdk.Context
+	tmClient     rpcclient.Client
+	k            *keeper.Keeper
+	ctxProvider  func(int64) sdk.Context
+	filterConfig *FilterConfig
 }
 
 func (f *LogFetcher) GetLogsByFilters(ctx context.Context, crit filters.FilterCriteria, lastToHeight int64) ([]*ethtypes.Log, int64, error) {
@@ -275,6 +279,7 @@ func (f *LogFetcher) GetLogsByFilters(ctx context.Context, crit filters.FilterCr
 		}
 		return f.GetLogsForBlock(ctx, block, crit, bloomIndexes), block.Block.Height, nil
 	}
+	applyOpenEndedLogLimit := f.filterConfig.maxLog > 0 && (crit.FromBlock == nil || crit.ToBlock == nil)
 	latest := f.ctxProvider(LatestCtxHeight).BlockHeight()
 	begin, end := latest, latest
 	if crit.FromBlock != nil {
@@ -290,6 +295,9 @@ func (f *LogFetcher) GetLogsByFilters(ctx context.Context, crit filters.FilterCr
 	if lastToHeight > begin {
 		begin = lastToHeight
 	}
+	if !applyOpenEndedLogLimit && f.filterConfig.maxBlock > 0 && end >= (begin+f.filterConfig.maxBlock) {
+		end = begin + f.filterConfig.maxBlock - 1
+	}
 	// begin should always be <= end block at this point
 	if begin > end {
 		return nil, 0, fmt.Errorf("fromBlock %d is after toBlock %d", begin, end)
@@ -303,6 +311,10 @@ func (f *LogFetcher) GetLogsByFilters(ctx context.Context, crit filters.FilterCr
 			return nil, 0, err
 		}
 		res = append(res, f.GetLogsForBlock(ctx, block, crit, bloomIndexes)...)
+		if applyOpenEndedLogLimit && int64(len(res)) >= f.filterConfig.maxLog {
+			res = res[:int(f.filterConfig.maxLog)]
+			break
+		}
 	}
 
 	return res, end, nil

evmrpc/filter_test.go

@@ -246,7 +246,7 @@ func TestFilterGetFilterChanges(t *testing.T) {
 
 	resObj = sendRequest(t, TestPort, "getFilterChanges", filterId)
 	logs := resObj["result"].([]interface{})
-	require.Equal(t, 5, len(logs))
+	require.Equal(t, 4, len(logs)) // limited by MaxLogNoBlock config to 4
 	logObj := logs[0].(map[string]interface{})
 	require.Equal(t, "0x2", logObj["blockNumber"].(string))
 

evmrpc/server.go

@@ -72,7 +72,7 @@ func NewEVMHTTPServer(
 		},
 		{
 			Namespace: "eth",
-			Service:   NewFilterAPI(tmClient, &LogFetcher{tmClient: tmClient, k: k, ctxProvider: ctxProvider}, &FilterConfig{timeout: config.FilterTimeout}),
+			Service:   NewFilterAPI(tmClient, &LogFetcher{tmClient: tmClient, k: k, ctxProvider: ctxProvider}, &FilterConfig{timeout: config.FilterTimeout, maxLog: config.MaxLogNoBlock, maxBlock: config.MaxBlocksForLog}),
 		},
 		{
 			Namespace: "sei",

evmrpc/setup_test.go

@@ -407,6 +407,7 @@ func init() {
 	goodConfig.HTTPPort = TestPort
 	goodConfig.WSPort = TestWSPort
 	goodConfig.FilterTimeout = 500 * time.Millisecond
+	goodConfig.MaxLogNoBlock = 4
 	infoLog, err := log.NewDefaultLogger("text", "info")
 	if err != nil {
 		panic(err)

evmrpc/subscribe.go

@@ -33,6 +33,7 @@ type SubscriptionConfig struct {
 }
 
 func NewSubscriptionAPI(tmClient rpcclient.Client, logFetcher *LogFetcher, subscriptionConfig *SubscriptionConfig) *SubscriptionAPI {
+	logFetcher.filterConfig = &FilterConfig{}
 	return &SubscriptionAPI{
 		tmClient:            tmClient,
 		subscriptionManager: NewSubscriptionManager(tmClient),