New overlays feature for doing ad-hoc simulations of existing contracts with modified bytecode

cmd/rpcdaemon/cli/config.go

@@ -137,6 +137,8 @@ func RootCommand() (*cobra.Command, *httpcfg.HttpCfg) {
 	rootCmd.PersistentFlags().DurationVar(&cfg.HTTPTimeouts.WriteTimeout, "http.timeouts.write", rpccfg.DefaultHTTPTimeouts.WriteTimeout, "Maximum duration before timing out writes of the response. It is reset whenever a new request's header is read")
 	rootCmd.PersistentFlags().DurationVar(&cfg.HTTPTimeouts.IdleTimeout, "http.timeouts.idle", rpccfg.DefaultHTTPTimeouts.IdleTimeout, "Maximum amount of time to wait for the next request when keep-alives are enabled. If http.timeouts.idle is zero, the value of http.timeouts.read is used")
 	rootCmd.PersistentFlags().DurationVar(&cfg.EvmCallTimeout, "rpc.evmtimeout", rpccfg.DefaultEvmCallTimeout, "Maximum amount of time to wait for the answer from EVM call.")
+	rootCmd.PersistentFlags().DurationVar(&cfg.OverlayGetLogsTimeout, "rpc.overlay.getlogstimeout", rpccfg.DefaultOverlayGetLogsTimeout, "Maximum amount of time to wait for the answer from the overlay_getLogs call.")
+	rootCmd.PersistentFlags().DurationVar(&cfg.OverlayReplayBlockTimeout, "rpc.overlay.replayblocktimeout", rpccfg.DefaultOverlayReplayBlockTimeout, "Maximum amount of time to wait for the answer to replay a single block when called from an overlay_getLogs call.")
 	rootCmd.PersistentFlags().IntVar(&cfg.BatchLimit, utils.RpcBatchLimit.Name, utils.RpcBatchLimit.Value, utils.RpcBatchLimit.Usage)
 	rootCmd.PersistentFlags().IntVar(&cfg.ReturnDataLimit, utils.RpcReturnDataLimit.Name, utils.RpcReturnDataLimit.Value, utils.RpcReturnDataLimit.Usage)
 	rootCmd.PersistentFlags().BoolVar(&cfg.AllowUnprotectedTxs, utils.AllowUnprotectedTxs.Name, utils.AllowUnprotectedTxs.Value, utils.AllowUnprotectedTxs.Usage)

cmd/rpcdaemon/cli/httpcfg/http_cfg.go

@@ -66,11 +66,14 @@ type HttpCfg struct {
 	SocketServerEnabled bool
 	SocketListenUrl     string
 
-	JWTSecretPath   string // Engine API Authentication
-	TraceRequests   bool   // Always trace requests in INFO level
-	HTTPTimeouts    rpccfg.HTTPTimeouts
-	AuthRpcTimeouts rpccfg.HTTPTimeouts
-	EvmCallTimeout  time.Duration
+	JWTSecretPath             string // Engine API Authentication
+	TraceRequests             bool   // Always trace requests in INFO level
+	HTTPTimeouts              rpccfg.HTTPTimeouts
+	AuthRpcTimeouts           rpccfg.HTTPTimeouts
+	EvmCallTimeout            time.Duration
+	OverlayGetLogsTimeout     time.Duration
+	OverlayReplayBlockTimeout time.Duration
+
 	LogDirVerbosity string
 	LogDirPath      string
 

cmd/rpcdaemon/postman/README.md

@@ -1,9 +1,10 @@
 # Postman testing
 
-There are two files here:
+There are three files here:
 
 - RPC_Testing.json
 - Trace_Testing.json
+- Overlay_Testing.json
 
 You can import them into Postman using these
 instructions: https://github.com/ledgerwatch/erigon/wiki/Using-Postman-to-Test-TurboGeth-RPC
@@ -14,5 +15,7 @@ release. There is basically one test for each of the 81 RPC endpoints.
 The second file contains 31 test cases specifically for the nine trace routines (five tests for five of the routines,
 three for another, one each for the other three).
 
+The third file contains 12 test cases for the overlay API for CREATE and CREATE2.
+
 Another collection of related tests can be found
 here: https://github.com/Great-Hill-Corporation/trueblocks-core/tree/develop/src/other/trace_tests

cmd/rpctest/main.go

@@ -240,6 +240,19 @@ func main() {
 	}
 	with(benchEthGetLogsCmd, withErigonUrl, withGethUrl, withNeedCompare, withBlockNum, withRecord, withErrorFile)
 
+	var benchOverlayGetLogsCmd = &cobra.Command{
+		Use:   "benchOverlayGetLogs",
+		Short: "",
+		Long:  ``,
+		Run: func(cmd *cobra.Command, args []string) {
+			err := rpctest.BenchOverlayGetLogs(erigonURL, needCompare, blockFrom, blockTo, recordFile, errorFile)
+			if err != nil {
+				logger.Error(err.Error())
+			}
+		},
+	}
+	with(benchOverlayGetLogsCmd, withErigonUrl, withGethUrl, withNeedCompare, withBlockNum, withRecord, withErrorFile)
+
 	var bench9Cmd = &cobra.Command{
 		Use:   "bench9",
 		Short: "",
@@ -457,6 +470,7 @@ func main() {
 		bench6Cmd,
 		bench7Cmd,
 		benchEthGetLogsCmd,
+		benchOverlayGetLogsCmd,
 		bench9Cmd,
 		benchTraceCallCmd,
 		benchTraceCallManyCmd,

cmd/rpctest/rpctest/bench_overlaygetlogs.go

@@ -0,0 +1,132 @@
+package rpctest
+
+import (
+	"bufio"
+	"fmt"
+	"math/rand"
+	"net/http"
+	"os"
+	"time"
+)
+
+// BenchOverlayGetLogs compares response of Erigon with Geth
+// but also can be used for comparing RPCDaemon with Geth or infura
+// parameters:
+// needCompare - if false - doesn't call Erigon and doesn't compare responses
+//
+//	false value - to generate vegeta files, it's faster but we can generate vegeta files for Geth and Erigon
+//	recordFile stores all eth_getlogs returned with success
+//	errorFile stores information when erigon and geth doesn't return same data
+func BenchOverlayGetLogs(erigonURL string, needCompare bool, blockFrom uint64, blockTo uint64, recordFile string, errorFile string) error {
+	setRoutes(erigonURL, erigonURL)
+	var client = &http.Client{
+		Timeout: time.Second * 600,
+	}
+
+	var rec *bufio.Writer
+	if recordFile != "" {
+		f, err := os.Create(recordFile)
+		if err != nil {
+			return fmt.Errorf("Cannot create file %s for recording: %v\n", recordFile, err)
+		}
+		defer f.Close()
+		rec = bufio.NewWriter(f)
+		defer rec.Flush()
+	}
+	var errs *bufio.Writer
+	if errorFile != "" {
+		ferr, err := os.Create(errorFile)
+		if err != nil {
+			return fmt.Errorf("Cannot create file %s for error output: %v\n", errorFile, err)
+		}
+		defer ferr.Close()
+		errs = bufio.NewWriter(ferr)
+		defer errs.Flush()
+	}
+
+	var resultsCh chan CallResult = nil
+	if !needCompare {
+		resultsCh = make(chan CallResult, 1000)
+		defer close(resultsCh)
+		go vegetaWrite(true, []string{"debug_getModifiedAccountsByNumber", "eth_getLogs"}, resultsCh)
+	}
+
+	var res CallResult
+	reqGen := &RequestGenerator{
+		client: client,
+	}
+
+	reqGen.reqID++
+	var blockNumber EthBlockNumber
+	res = reqGen.Erigon("eth_blockNumber", reqGen.blockNumber(), &blockNumber)
+	if res.Err != nil {
+		return fmt.Errorf("Could not get block number: %v\n", res.Err)
+	}
+	if blockNumber.Error != nil {
+		return fmt.Errorf("Error getting block number: %d %s\n", blockNumber.Error.Code, blockNumber.Error.Message)
+	}
+	fmt.Printf("Last block: %d\n", blockNumber.Number)
+
+	prevBn := blockFrom
+	rnd := rand.New(rand.NewSource(42)) // nolint:gosec
+	for bn := blockFrom + 100; bn < blockTo; bn += 100 {
+
+		// Checking modified accounts
+		reqGen.reqID++
+		var mag DebugModifiedAccounts
+		res = reqGen.Erigon("debug_getModifiedAccountsByNumber", reqGen.getModifiedAccountsByNumber(prevBn, bn), &mag)
+		if res.Err != nil {
+			return fmt.Errorf("Could not get modified accounts (Erigon): %v\n", res.Err)
+		}
+		if mag.Error != nil {
+			return fmt.Errorf("Error getting modified accounts (Erigon): %d %s\n", mag.Error.Code, mag.Error.Message)
+		}
+		if res.Err == nil && mag.Error == nil {
+			accountSet := extractAccountMap(&mag)
+			for account := range accountSet {
+				reqGen.reqID++
+				requestEth := reqGen.getLogs(prevBn, bn, account)
+				requestOverlay := reqGen.getOverlayLogs(prevBn, bn, account)
+				errCtx := fmt.Sprintf("account %x blocks %d-%d", account, prevBn, bn)
+				if err := requestAndCompareErigon(requestEth, requestOverlay, "eth_getLogs", "overlay_getLogs", errCtx, reqGen, needCompare, rec, errs, resultsCh,
+					/* insertOnlyIfSuccess */ false); err != nil {
+					fmt.Println(err)
+					return err
+				}
+				topics := getTopics(res.Result)
+				// All combination of account and one topic
+				for _, topic := range topics {
+					reqGen.reqID++
+					requestEth = reqGen.getLogs1(prevBn, bn+10000, account, topic)
+					requestOverlay := reqGen.getOverlayLogs1(prevBn, bn+10000, account, topic)
+					errCtx := fmt.Sprintf("account %x topic %x blocks %d-%d", account, topic, prevBn, bn)
+					if err := requestAndCompareErigon(requestEth, requestOverlay, "eth_getLogs", "overlay_getLogs", errCtx, reqGen, needCompare, rec, errs, resultsCh,
+						/* insertOnlyIfSuccess */ false); err != nil {
+						fmt.Println(err)
+						return err
+					}
+				}
+				// Random combinations of two topics
+				if len(topics) >= 2 {
+					idx1 := rnd.Int31n(int32(len(topics)))
+					idx2 := rnd.Int31n(int32(len(topics) - 1))
+					if idx2 >= idx1 {
+						idx2++
+					}
+					reqGen.reqID++
+					requestEth = reqGen.getLogs2(prevBn, bn+100000, account, topics[idx1], topics[idx2])
+					requestOverlay := reqGen.getOverlayLogs2(prevBn, bn+100000, account, topics[idx1], topics[idx2])
+					errCtx := fmt.Sprintf("account %x topic1 %x topic2 %x blocks %d-%d", account, topics[idx1], topics[idx2], prevBn, bn)
+					if err := requestAndCompareErigon(requestEth, requestOverlay, "eth_getLogs", "overlay_getLogs", errCtx, reqGen, needCompare, rec, errs, resultsCh,
+						/* insertOnlyIfSuccess */ false); err != nil {
+						fmt.Println(err)
+						return err
+					}
+				}
+			}
+		}
+		fmt.Printf("Done blocks %d-%d, modified accounts: %d\n", prevBn, bn, len(mag.Result))
+		prevBn = bn
+	}
+	return nil
+}

cmd/rpctest/rpctest/request_generator.go

@@ -89,16 +89,31 @@ func (g *RequestGenerator) getLogs(prevBn uint64, bn uint64, account libcommon.A
 	return fmt.Sprintf(template, prevBn, bn, account, g.reqID)
 }
 
+func (g *RequestGenerator) getOverlayLogs(prevBn uint64, bn uint64, account libcommon.Address) string {
+	const template = `{"jsonrpc":"2.0","method":"overlay_getLogs","params":[{"fromBlock": "0x%x", "toBlock": "0x%x", "address": "0x%x"},{}],"id":%d}`
+	return fmt.Sprintf(template, prevBn, bn, account, g.reqID)
+}
+
 func (g *RequestGenerator) getLogs1(prevBn uint64, bn uint64, account libcommon.Address, topic libcommon.Hash) string {
 	const template = `{"jsonrpc":"2.0","method":"eth_getLogs","params":[{"fromBlock": "0x%x", "toBlock": "0x%x", "address": "0x%x", "topics": ["0x%x"]}],"id":%d}`
 	return fmt.Sprintf(template, prevBn, bn, account, topic, g.reqID)
 }
 
+func (g *RequestGenerator) getOverlayLogs1(prevBn uint64, bn uint64, account libcommon.Address, topic libcommon.Hash) string {
+	const template = `{"jsonrpc":"2.0","method":"eth_getLogs","params":[{"fromBlock": "0x%x", "toBlock": "0x%x", "address": "0x%x", "topics": ["0x%x"]},{}],"id":%d}`
+	return fmt.Sprintf(template, prevBn, bn, account, topic, g.reqID)
+}
+
 func (g *RequestGenerator) getLogs2(prevBn uint64, bn uint64, account libcommon.Address, topic1, topic2 libcommon.Hash) string {
 	const template = `{"jsonrpc":"2.0","method":"eth_getLogs","params":[{"fromBlock": "0x%x", "toBlock": "0x%x", "address": "0x%x", "topics": ["0x%x", "0x%x"]}],"id":%d}`
 	return fmt.Sprintf(template, prevBn, bn, account, topic1, topic2, g.reqID)
 }
 
+func (g *RequestGenerator) getOverlayLogs2(prevBn uint64, bn uint64, account libcommon.Address, topic1, topic2 libcommon.Hash) string {
+	const template = `{"jsonrpc":"2.0","method":"eth_getLogs","params":[{"fromBlock": "0x%x", "toBlock": "0x%x", "address": "0x%x", "topics": ["0x%x", "0x%x"]},{}],"id":%d}`
+	return fmt.Sprintf(template, prevBn, bn, account, topic1, topic2, g.reqID)
+}
+
 func (g *RequestGenerator) accountRange(bn uint64, page []byte, num int) string { //nolint
 	const template = `{ "jsonrpc": "2.0", "method": "debug_accountRange", "params": ["0x%x", "%s", %d, false, false, false], "id":%d}`
 	encodedKey := base64.StdEncoding.EncodeToString(page)

cmd/rpctest/rpctest/utils.go

@@ -290,6 +290,66 @@ func requestAndCompare(request string, methodName string, errCtx string, reqGen
 	return nil
 }
 
+func requestAndCompareErigon(requestA, requestB string, methodNameA, methodNameB string, errCtx string, reqGen *RequestGenerator, needCompare bool, rec *bufio.Writer, errs *bufio.Writer, channel chan CallResult, insertOnlyIfSuccess bool) error {
+	recording := rec != nil
+	res := reqGen.Erigon2(methodNameA, requestA)
+	if res.Err != nil {
+		return fmt.Errorf("could not invoke %s (Erigon): %w", methodNameA, res.Err)
+	}
+	errVal := res.Result.Get("error")
+	if errVal != nil {
+		if !needCompare && channel == nil {
+			return fmt.Errorf("error invoking %s (Erigon): %d %s", methodNameA, errVal.GetInt("code"), errVal.GetStringBytes("message"))
+		}
+	}
+	if needCompare {
+		resg := reqGen.Erigon2(methodNameB, requestB)
+		if resg.Err != nil {
+			return fmt.Errorf("could not invoke %s (Geth/OE): %w", methodNameB, res.Err)
+		}
+		errValg := resg.Result.Get("error")
+		if errVal == nil && errValg == nil {
+			if err := compareResults(res.Result, resg.Result); err != nil {
+				recording = false
+				if errs != nil {
+					fmt.Printf("different results for methods %s, %s, errCtx: %s: %v\n", methodNameA, methodNameB, errCtx, err)
+					fmt.Fprintf(errs, "\nDifferent results for methods %s, %s, errCtx %s: %v\n", methodNameA, methodNameB, errCtx, err)
+					fmt.Fprintf(errs, "Request=====================================\n%s\n", requestA)
+					fmt.Fprintf(errs, "%s response=================================\n%s\n", methodNameA, res.Response)
+					fmt.Fprintf(errs, "%s response=================================\n%s\n", methodNameB, resg.Response)
+					errs.Flush() // nolint:errcheck
+					// Keep going
+				} else {
+					reqFile, _ := os.Create("request.json")                //nolint:errcheck
+					reqFile.Write([]byte(requestA))                        //nolint:errcheck
+					reqFile.Close()                                        //nolint:errcheck
+					erigonRespFile, _ := os.Create("erigon-response.json") //nolint:errcheck
+					erigonRespFile.Write(res.Response)                     //nolint:errcheck
+					erigonRespFile.Close()                                 //nolint:errcheck
+					oeRespFile, _ := os.Create("oe-response.json")         //nolint:errcheck
+					oeRespFile.Write(resg.Response)                        //nolint:errcheck
+					oeRespFile.Close()                                     //nolint:errcheck
+					return fmt.Errorf("different results for methods %s, %s, errCtx %s: %v\nRequest in file request.json, Erigon response in file erigon-response.json, Geth/OE response in file oe-response.json", methodNameA, methodNameB, errCtx, err)
+				}
+			}
+		} else {
+			//TODO fix for two methods
+			return compareErrors(errVal, errValg, methodNameA, errCtx, errs)
+		}
+	} else {
+		if channel != nil {
+			if insertOnlyIfSuccess == false || (insertOnlyIfSuccess && errVal == nil) {
+				channel <- res
+			}
+		}
+	}
+
+	if recording {
+		fmt.Fprintf(rec, "%s\n%s\n\n", requestA, res.Response)
+	}
+	return nil
+}
+
 func compareBalances(balance, balanceg *EthBalance) bool {
 	if balance.Balance.ToInt().Cmp(balanceg.Balance.ToInt()) != 0 {
 		fmt.Printf("Different balance: %d %d\n", balance.Balance.ToInt(), balanceg.Balance.ToInt())

common/compiler/test.v.py

@@ -1,3 +1,3 @@
-@public
+@external
 def test():
-    hello: int128
+    hello: int128 = 13

core/vm/evm.go

@@ -322,13 +322,21 @@ type codeAndHash struct {
 	hash libcommon.Hash
 }
 
+func NewCodeAndHash(code []byte) *codeAndHash {
+	return &codeAndHash{code: code}
+}
+
 func (c *codeAndHash) Hash() libcommon.Hash {
 	if c.hash == (libcommon.Hash{}) {
 		c.hash = crypto.Keccak256Hash(c.code)
 	}
 	return c.hash
 }
 
+func (evm *EVM) OverlayCreate(caller ContractRef, codeAndHash *codeAndHash, gas uint64, value *uint256.Int, address libcommon.Address, typ OpCode, incrementNonce bool) ([]byte, libcommon.Address, uint64, error) {
+	return evm.create(caller, codeAndHash, gas, value, address, typ, incrementNonce)
+}
+
 // create creates a new contract using code as deployment code.
 func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, value *uint256.Int, address libcommon.Address, typ OpCode, incrementNonce bool) ([]byte, libcommon.Address, uint64, error) {
 	var ret []byte

rpc/rpccfg/rpccfg.go

@@ -37,6 +37,8 @@ var DefaultHTTPTimeouts = HTTPTimeouts{
 }
 
 const DefaultEvmCallTimeout = 5 * time.Minute
+const DefaultOverlayGetLogsTimeout = 5 * time.Minute
+const DefaultOverlayReplayBlockTimeout = 10 * time.Second
 
 var SlowLogBlackList = []string{
 	"eth_getBlock", "eth_getBlockByNumber", "eth_getBlockByHash", "eth_blockNumber",

turbo/cli/default_flags.go

@@ -89,6 +89,8 @@ var DefaultFlags = []cli.Flag{
 	&AuthRpcWriteTimeoutFlag,
 	&AuthRpcIdleTimeoutFlag,
 	&EvmCallTimeoutFlag,
+	&OverlayGetLogsFlag,
+	&OverlayReplayBlockFlag,
 
 	&utils.SnapKeepBlocksFlag,
 	&utils.SnapStopFlag,

turbo/cli/flags.go

@@ -234,6 +234,18 @@ var (
 		Value: rpccfg.DefaultEvmCallTimeout,
 	}
 
+	OverlayGetLogsFlag = cli.DurationFlag{
+		Name:  "rpc.overlay.getlogstimeout",
+		Usage: "Maximum amount of time to wait for the answer from the overlay_getLogs call.",
+		Value: rpccfg.DefaultOverlayGetLogsTimeout,
+	}
+
+	OverlayReplayBlockFlag = cli.DurationFlag{
+		Name:  "rpc.overlay.replayblocktimeout",
+		Usage: "Maximum amount of time to wait for the answer to replay a single block when called from an overlay_getLogs call.",
+		Value: rpccfg.DefaultOverlayReplayBlockTimeout,
+	}
+
 	TxPoolCommitEvery = cli.DurationFlag{
 		Name:  "txpool.commit.every",
 		Usage: "How often transactions should be committed to the storage",
@@ -465,6 +477,8 @@ func setEmbeddedRpcDaemon(ctx *cli.Context, cfg *nodecfg.Config, logger log.Logg
 			IdleTimeout:  ctx.Duration(HTTPIdleTimeoutFlag.Name),
 		},
 		EvmCallTimeout:              ctx.Duration(EvmCallTimeoutFlag.Name),
+		OverlayGetLogsTimeout:       ctx.Duration(OverlayGetLogsFlag.Name),
+		OverlayReplayBlockTimeout:   ctx.Duration(OverlayReplayBlockFlag.Name),
 		WebsocketPort:               ctx.Int(utils.WSPortFlag.Name),
 		WebsocketEnabled:            ctx.IsSet(utils.WSEnabledFlag.Name),
 		RpcBatchConcurrency:         ctx.Uint(utils.RpcBatchConcurrencyFlag.Name),