Add support for eth_getBlockReceipts RPC method (#102)

### TL;DR

Added support for using `eth_getBlockReceipts` instead of `eth_getLogs` and enhanced transaction data storage.

### What changed?

- Introduced `eth_getBlockReceipts` RPC method support
- Added new configuration options for block receipts
- Enhanced transaction structure with additional fields from receipts
- Updated ClickHouse schema to accommodate new transaction fields
- Improved serialization logic to handle block receipts data

### How to test?

1. Update the configuration file to enable block receipts:
   ```yaml
   rpc:
     blockReceipts:
       enabled: true
       blocksPerRequest: 500
       batchDelay: 100
   ```
2. Run the indexer with the updated configuration
3. Verify that transactions in the ClickHouse database contain the new fields (e.g., `contract_address`, `gas_used`, `status`)
4. Check the logs to ensure that `eth_getBlockReceipts` method is being used when enabled

### Why make this change?

This change improves the efficiency and completeness of data collection:

1. Using `eth_getBlockReceipts` can be more efficient than fetching logs and receipts separately
2. Additional transaction data from receipts provides more comprehensive information for analysis and querying
3. The enhanced schema allows for more detailed transaction insights without the need for additional RPC calls

README.md

@@ -136,6 +136,43 @@ rpc:
     batchDelay: 100
 ```
 
+#### RPC Block Receipts Enabled
+If this is `true`, will use `eth_getBlockReceipts` instead of `eth_getLogs` if the RPC supports it. Allows getting receipt data for transactions, but is not supported by every RPC. Default is `false`.
+
+cmd: `--rpc-block-receipts-enabled`
+env: `RPC_BLOCKRECEIPTS_ENABLED`
+yaml:
+```yaml
+rpc:
+  blockReceipts:
+    enabled: true
+```
+
+#### RPC Block Receipts Blocks Per Request
+How many blocks at a time to fetch block receipts for from the RPC. Default is 250.
+Has no effect if it's larger than RPC blocks per request.
+
+cmd: `--rpc-block-receipts-blocksPerRequest`
+env: `RPC_BLOCKRECEIPTS_BLOCKSPERREQUEST`
+yaml:
+```yaml
+rpc:
+  blockReceipts:
+    blocksPerRequest: 100
+```
+
+#### RPC Block Receipts Batch Delay
+Milliseconds to wait between batches of block receipts when fetching from the RPC. Default is 0.
+
+cmd: `--rpc-block-receipts-batchDelay`
+env: `RPC_BLOCKRECEIPTS_BATCHDELAY`
+yaml:
+```yaml
+rpc:
+  blockReceipts:
+    batchDelay: 100
+```
+
 #### RPC Traces Enabled
 Whether to enable fetching traces from the RPC. Default is `true`, but it will try to detect if the RPC supports traces automatically.
 

cmd/root.go

@@ -41,6 +41,9 @@ func init() {
 	rootCmd.PersistentFlags().Int("rpc-blocks-batchDelay", 0, "Milliseconds to wait between batches of blocks when fetching from the RPC")
 	rootCmd.PersistentFlags().Int("rpc-logs-blocksPerRequest", 0, "How many blocks to fetch logs per request")
 	rootCmd.PersistentFlags().Int("rpc-logs-batchDelay", 0, "Milliseconds to wait between batches of logs when fetching from the RPC")
+	rootCmd.PersistentFlags().Bool("rpc-blockReceipts-enabled", false, "Whether to enable fetching block receipts from the RPC")
+	rootCmd.PersistentFlags().Int("rpc-blockReceipts-blocksPerRequest", 0, "How many blocks to fetch receipts for per request")
+	rootCmd.PersistentFlags().Int("rpc-blockReceipts-batchDelay", 0, "Milliseconds to wait between batches of receipts when fetching from the RPC")
 	rootCmd.PersistentFlags().Bool("rpc-traces-enabled", true, "Whether to enable fetching traces from the RPC")
 	rootCmd.PersistentFlags().Int("rpc-traces-blocksPerRequest", 0, "How many blocks to fetch traces per request")
 	rootCmd.PersistentFlags().Int("rpc-traces-batchDelay", 0, "Milliseconds to wait between batches of traces when fetching from the RPC")
@@ -89,6 +92,9 @@ func init() {
 	viper.BindPFlag("rpc.blocks.batchDelay", rootCmd.PersistentFlags().Lookup("rpc-blocks-batchDelay"))
 	viper.BindPFlag("rpc.logs.blocksPerRequest", rootCmd.PersistentFlags().Lookup("rpc-logs-blocksPerRequest"))
 	viper.BindPFlag("rpc.logs.batchDelay", rootCmd.PersistentFlags().Lookup("rpc-logs-batchDelay"))
+	viper.BindPFlag("rpc.blockReceipts.enabled", rootCmd.PersistentFlags().Lookup("rpc-blockReceipts-enabled"))
+	viper.BindPFlag("rpc.blockReceipts.blocksPerRequest", rootCmd.PersistentFlags().Lookup("rpc-blockReceipts-blocksPerRequest"))
+	viper.BindPFlag("rpc.blockReceipts.batchDelay", rootCmd.PersistentFlags().Lookup("rpc-blockReceipts-batchDelay"))
 	viper.BindPFlag("rpc.traces.enabled", rootCmd.PersistentFlags().Lookup("rpc-traces-enabled"))
 	viper.BindPFlag("rpc.traces.blocksPerRequest", rootCmd.PersistentFlags().Lookup("rpc-traces-blocksPerRequest"))
 	viper.BindPFlag("rpc.traces.batchDelay", rootCmd.PersistentFlags().Lookup("rpc-traces-batchDelay"))

configs/config.example.yml

@@ -5,6 +5,10 @@ rpc:
   logs:
     blocksPerRequest: 400
     batchDelay: 100
+  blockReceipts:
+    enabled: true
+    blocksPerRequest: 500
+    batchDelay: 100
   traces:
     enabled: true
     blocksPerRequest: 200
@@ -29,6 +33,11 @@ failureRecoverer:
   interval: 10000
   blocksPerRun: 100
 
+reorgHandler:
+  enabled: true
+  interval: 1000
+  blocksPerScan: 50
+
 storage:
   main:
     clickhouse:

configs/config.go

@@ -81,22 +81,22 @@ type RedisConfig struct {
 	DB       int    `mapstructure:"db"`
 }
 
-type RPCBatchSizeConfig struct {
+type RPCBatchRequestConfig struct {
 	BlocksPerRequest int `mapstructure:"blocksPerRequest"`
 	BatchDelay       int `mapstructure:"batchDelay"`
 }
 
-type RPCTracesConfig struct {
-	Enabled          bool `mapstructure:"enabled"`
-	BlocksPerRequest int  `mapstructure:"blocksPerRequest"`
-	BatchDelay       int  `mapstructure:"batchDelay"`
+type ToggleableRPCBatchRequestConfig struct {
+	Enabled bool `mapstructure:"enabled"`
+	RPCBatchRequestConfig
 }
 
 type RPCConfig struct {
-	URL    string             `mapstructure:"url"`
-	Blocks RPCBatchSizeConfig `mapstructure:"blocks"`
-	Logs   RPCBatchSizeConfig `mapstructure:"logs"`
-	Traces RPCTracesConfig    `mapstructure:"traces"`
+	URL           string                          `mapstructure:"url"`
+	Blocks        RPCBatchRequestConfig           `mapstructure:"blocks"`
+	Logs          RPCBatchRequestConfig           `mapstructure:"logs"`
+	BlockReceipts ToggleableRPCBatchRequestConfig `mapstructure:"blockReceipts"`
+	Traces        ToggleableRPCBatchRequestConfig `mapstructure:"traces"`
 }
 
 type APIConfig struct {

internal/common/log.go

@@ -18,3 +18,5 @@ type Log struct {
 }
 
 type RawLogs = []map[string]interface{}
+type RawReceipts = []RawReceipt
+type RawReceipt = map[string]interface{}

internal/common/transaction.go

@@ -25,5 +25,13 @@ type Transaction struct {
 	R                    *big.Int `json:"r"`
 	S                    *big.Int `json:"s"`
 	V                    *big.Int `json:"v"`
-	AccessListJson       string   `json:"access_list_json"`
+	AccessListJson       *string  `json:"access_list_json"`
+	ContractAddress      *string  `json:"contract_address"`
+	GasUsed              *uint64  `json:"gas_used"`
+	CumulativeGasUsed    *uint64  `json:"cumulative_gas_used"`
+	EffectiveGasPrice    *big.Int `json:"effective_gas_price"`
+	BlobGasUsed          *uint64  `json:"blob_gas_used"`
+	BlobGasPrice         *big.Int `json:"blob_gas_price"`
+	LogsBloom            *string  `json:"logs_bloom"`
+	Status               *uint64  `json:"status"`
 }

internal/rpc/params.go

@@ -21,3 +21,7 @@ func GetLogsParams(blockNum *big.Int) []interface{} {
 func TraceBlockParams(blockNum *big.Int) []interface{} {
 	return []interface{}{hexutil.EncodeBig(blockNum)}
 }
+
+func GetBlockReceiptsParams(blockNum *big.Int) []interface{} {
+	return []interface{}{hexutil.EncodeBig(blockNum)}
+}

internal/rpc/rpc.go

@@ -27,9 +27,10 @@ type GetBlocksResult struct {
 }
 
 type BlocksPerRequestConfig struct {
-	Blocks int
-	Logs   int
-	Traces int
+	Blocks   int
+	Logs     int
+	Traces   int
+	Receipts int
 }
 
 type IRPCClient interface {
@@ -44,13 +45,14 @@ type IRPCClient interface {
 }
 
 type Client struct {
-	RPCClient          *gethRpc.Client
-	EthClient          *ethclient.Client
-	supportsTraceBlock bool
-	isWebsocket        bool
-	url                string
-	chainID            *big.Int
-	blocksPerRequest   BlocksPerRequestConfig
+	RPCClient             *gethRpc.Client
+	EthClient             *ethclient.Client
+	supportsTraceBlock    bool
+	supportsBlockReceipts bool
+	isWebsocket           bool
+	url                   string
+	chainID               *big.Int
+	blocksPerRequest      BlocksPerRequestConfig
 }
 
 func Initialize() (IRPCClient, error) {
@@ -111,28 +113,75 @@ func (rpc *Client) Close() {
 }
 
 func (rpc *Client) checkSupportedMethods() error {
+	if err := rpc.checkGetBlockByNumberSupport(); err != nil {
+		return err
+	}
+	if err := rpc.checkGetBlockReceiptsSupport(); err != nil {
+		return err
+	}
+	if err := rpc.checkGetLogsSupport(); err != nil {
+		return err
+	}
+	if err := rpc.checkTraceBlockSupport(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (rpc *Client) checkGetBlockByNumberSupport() error {
 	var blockByNumberResult interface{}
 	err := rpc.RPCClient.Call(&blockByNumberResult, "eth_getBlockByNumber", "latest", true)
 	if err != nil {
 		return fmt.Errorf("eth_getBlockByNumber method not supported: %v", err)
 	}
 	log.Debug().Msg("eth_getBlockByNumber method supported")
+	return nil
+}
 
+func (rpc *Client) checkGetBlockReceiptsSupport() error {
+	if config.Cfg.RPC.BlockReceipts.Enabled {
+		var getBlockReceiptsResult interface{}
+		receiptsErr := rpc.RPCClient.Call(&getBlockReceiptsResult, "eth_getBlockReceipts", "latest")
+		if receiptsErr != nil {
+			log.Warn().Err(receiptsErr).Msg("eth_getBlockReceipts method not supported")
+			return fmt.Errorf("eth_getBlockReceipts method not supported: %v", receiptsErr)
+		} else {
+			rpc.supportsBlockReceipts = true
+			log.Debug().Msg("eth_getBlockReceipts method supported")
+		}
+	} else {
+		rpc.supportsBlockReceipts = false
+		log.Debug().Msg("eth_getBlockReceipts method disabled")
+	}
+	return nil
+}
+
+func (rpc *Client) checkGetLogsSupport() error {
+	if rpc.supportsBlockReceipts {
+		return nil
+	}
 	var getLogsResult interface{}
 	logsErr := rpc.RPCClient.Call(&getLogsResult, "eth_getLogs", map[string]string{"fromBlock": "0x0", "toBlock": "0x0"})
 	if logsErr != nil {
 		return fmt.Errorf("eth_getLogs method not supported: %v", logsErr)
 	}
 	log.Debug().Msg("eth_getLogs method supported")
+	return nil
+}
 
-	var traceBlockResult interface{}
+func (rpc *Client) checkTraceBlockSupport() error {
 	if config.Cfg.RPC.Traces.Enabled {
+		var traceBlockResult interface{}
 		if traceBlockErr := rpc.RPCClient.Call(&traceBlockResult, "trace_block", "latest"); traceBlockErr != nil {
 			log.Warn().Err(traceBlockErr).Msg("Optional method trace_block not supported")
+		} else {
+			rpc.supportsTraceBlock = true
+			log.Debug().Msg("trace_block method supported")
 		}
+	} else {
+		rpc.supportsTraceBlock = false
+		log.Debug().Msg("trace_block method disabled")
 	}
-	rpc.supportsTraceBlock = traceBlockResult != nil
-	log.Debug().Msgf("trace_block method supported: %v", rpc.supportsTraceBlock)
 	return nil
 }
 
@@ -147,33 +196,44 @@ func (rpc *Client) setChainID() error {
 
 func (rpc *Client) GetFullBlocks(blockNumbers []*big.Int) []GetFullBlockResult {
 	var wg sync.WaitGroup
-	var blocks []RPCFetchBatchResult[common.RawBlock]
-	var logs []RPCFetchBatchResult[common.RawLogs]
-	var traces []RPCFetchBatchResult[common.RawTraces]
-
+	var blocks *[]RPCFetchBatchResult[common.RawBlock]
+	var logs *[]RPCFetchBatchResult[common.RawLogs]
+	var traces *[]RPCFetchBatchResult[common.RawTraces]
+	var receipts *[]RPCFetchBatchResult[common.RawReceipts]
 	wg.Add(2)
 
 	go func() {
 		defer wg.Done()
-		blocks = RPCFetchBatch[common.RawBlock](rpc, blockNumbers, "eth_getBlockByNumber", GetBlockWithTransactionsParams)
+		result := RPCFetchBatch[common.RawBlock](rpc, blockNumbers, "eth_getBlockByNumber", GetBlockWithTransactionsParams)
+		blocks = &result
 	}()
 
-	go func() {
-		defer wg.Done()
-		logs = RPCFetchInBatches[common.RawLogs](rpc, blockNumbers, rpc.blocksPerRequest.Logs, config.Cfg.RPC.Logs.BatchDelay, "eth_getLogs", GetLogsParams)
-	}()
+	if rpc.supportsBlockReceipts {
+		go func() {
+			defer wg.Done()
+			result := RPCFetchInBatches[common.RawReceipts](rpc, blockNumbers, rpc.blocksPerRequest.Receipts, config.Cfg.RPC.BlockReceipts.BatchDelay, "eth_getBlockReceipts", GetBlockReceiptsParams)
+			receipts = &result
+		}()
+	} else {
+		go func() {
+			defer wg.Done()
+			result := RPCFetchInBatches[common.RawLogs](rpc, blockNumbers, rpc.blocksPerRequest.Logs, config.Cfg.RPC.Logs.BatchDelay, "eth_getLogs", GetLogsParams)
+			logs = &result
+		}()
+	}
 
 	if rpc.supportsTraceBlock {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			traces = RPCFetchInBatches[common.RawTraces](rpc, blockNumbers, rpc.blocksPerRequest.Traces, config.Cfg.RPC.Traces.BatchDelay, "trace_block", TraceBlockParams)
+			result := RPCFetchInBatches[common.RawTraces](rpc, blockNumbers, rpc.blocksPerRequest.Traces, config.Cfg.RPC.Traces.BatchDelay, "trace_block", TraceBlockParams)
+			traces = &result
 		}()
 	}
 
 	wg.Wait()
 
-	return SerializeFullBlocks(rpc.chainID, blocks, logs, traces)
+	return SerializeFullBlocks(rpc.chainID, blocks, logs, traces, receipts)
 }
 
 func (rpc *Client) GetBlocks(blockNumbers []*big.Int) []GetBlocksResult {