core/vm, cmd/evm: implement eof validation (ethereum#30418)

The bulk of this PR is authored by @lightclient , in the original
EOF-work. More recently, the code has been picked up and reworked for the new EOF
specification, by @MariusVanDerWijden , in ethereum#29518, and also @shemnon has contributed with fixes.

This PR is an attempt to start eating the elephant one small bite at a
time, by selecting only the eof-validation as a standalone piece which
can be merged without interfering too much in the core stuff.

In this PR: 

- [x] Validation of eof containers, lifted from ethereum#29518, along with
test-vectors from consensus-tests and fuzzing, to ensure that the move
did not lose any functionality.
- [x] Definition of eof opcodes, which is a prerequisite for validation
- [x] Addition of `undefined` to a jumptable entry item. I'm not
super-happy with this, but for the moment it seems the least invasive
way to do it. A better way might be to go back and allowing nil-items or
nil execute-functions to denote "undefined".
- [x] benchmarks of eof validation speed 


---------

Co-authored-by: lightclient <lightclient@protonmail.com>
Co-authored-by: Marius van der Wijden <m.vanderwijden@live.de>
Co-authored-by: Danno Ferrin <danno.ferrin@shemnon.com>

cmd/evm/eofparse.go

@@ -0,0 +1,200 @@
+// Copyright 2023 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bufio"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/urfave/cli/v2"
+)
+
+func init() {
+	jt = vm.NewPragueEOFInstructionSetForTesting()
+}
+
+var (
+	jt       vm.JumpTable
+	initcode = "INITCODE"
+)
+
+func eofParseAction(ctx *cli.Context) error {
+	// If `--test` is set, parse and validate the reference test at the provided path.
+	if ctx.IsSet(refTestFlag.Name) {
+		var (
+			file          = ctx.String(refTestFlag.Name)
+			executedTests int
+			passedTests   int
+		)
+		err := filepath.Walk(file, func(path string, info fs.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+			if info.IsDir() {
+				return nil
+			}
+			log.Debug("Executing test", "name", info.Name())
+			passed, tot, err := executeTest(path)
+			passedTests += passed
+			executedTests += tot
+			return err
+		})
+		if err != nil {
+			return err
+		}
+		log.Info("Executed tests", "passed", passedTests, "total executed", executedTests)
+		return nil
+	}
+	// If `--hex` is set, parse and validate the hex string argument.
+	if ctx.IsSet(hexFlag.Name) {
+		if _, err := parseAndValidate(ctx.String(hexFlag.Name), false); err != nil {
+			return fmt.Errorf("err: %w", err)
+		}
+		fmt.Println("OK")
+		return nil
+	}
+	// If neither are passed in, read input from stdin.
+	scanner := bufio.NewScanner(os.Stdin)
+	scanner.Buffer(make([]byte, 1024*1024), 10*1024*1024)
+	for scanner.Scan() {
+		l := strings.TrimSpace(scanner.Text())
+		if strings.HasPrefix(l, "#") || l == "" {
+			continue
+		}
+		if _, err := parseAndValidate(l, false); err != nil {
+			fmt.Printf("err: %v\n", err)
+		} else {
+			fmt.Println("OK")
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		fmt.Println(err.Error())
+	}
+	return nil
+}
+
+type refTests struct {
+	Vectors map[string]eOFTest `json:"vectors"`
+}
+
+type eOFTest struct {
+	Code          string              `json:"code"`
+	Results       map[string]etResult `json:"results"`
+	ContainerKind string              `json:"containerKind"`
+}
+
+type etResult struct {
+	Result    bool   `json:"result"`
+	Exception string `json:"exception,omitempty"`
+}
+
+func executeTest(path string) (int, int, error) {
+	src, err := os.ReadFile(path)
+	if err != nil {
+		return 0, 0, err
+	}
+	var testsByName map[string]refTests
+	if err := json.Unmarshal(src, &testsByName); err != nil {
+		return 0, 0, err
+	}
+	passed, total := 0, 0
+	for testsName, tests := range testsByName {
+		for name, tt := range tests.Vectors {
+			for fork, r := range tt.Results {
+				total++
+				_, err := parseAndValidate(tt.Code, tt.ContainerKind == initcode)
+				if r.Result && err != nil {
+					log.Error("Test failure, expected validation success", "name", testsName, "idx", name, "fork", fork, "err", err)
+					continue
+				}
+				if !r.Result && err == nil {
+					log.Error("Test failure, expected validation error", "name", testsName, "idx", name, "fork", fork, "have err", r.Exception, "err", err)
+					continue
+				}
+				passed++
+			}
+		}
+	}
+	return passed, total, nil
+}
+
+func parseAndValidate(s string, isInitCode bool) (*vm.Container, error) {
+	if len(s) >= 2 && strings.HasPrefix(s, "0x") {
+		s = s[2:]
+	}
+	b, err := hex.DecodeString(s)
+	if err != nil {
+		return nil, fmt.Errorf("unable to decode data: %w", err)
+	}
+	return parse(b, isInitCode)
+}
+
+func parse(b []byte, isInitCode bool) (*vm.Container, error) {
+	var c vm.Container
+	if err := c.UnmarshalBinary(b, isInitCode); err != nil {
+		return nil, err
+	}
+	if err := c.ValidateCode(&jt, isInitCode); err != nil {
+		return nil, err
+	}
+	return &c, nil
+}
+
+func eofDumpAction(ctx *cli.Context) error {
+	// If `--hex` is set, parse and validate the hex string argument.
+	if ctx.IsSet(hexFlag.Name) {
+		return eofDump(ctx.String(hexFlag.Name))
+	}
+	// Otherwise read from stdin
+	scanner := bufio.NewScanner(os.Stdin)
+	scanner.Buffer(make([]byte, 1024*1024), 10*1024*1024)
+	for scanner.Scan() {
+		l := strings.TrimSpace(scanner.Text())
+		if strings.HasPrefix(l, "#") || l == "" {
+			continue
+		}
+		if err := eofDump(l); err != nil {
+			return err
+		}
+		fmt.Println("")
+	}
+	return scanner.Err()
+}
+
+func eofDump(hexdata string) error {
+	if len(hexdata) >= 2 && strings.HasPrefix(hexdata, "0x") {
+		hexdata = hexdata[2:]
+	}
+	b, err := hex.DecodeString(hexdata)
+	if err != nil {
+		return fmt.Errorf("unable to decode data: %w", err)
+	}
+	var c vm.Container
+	if err := c.UnmarshalBinary(b, false); err != nil {
+		return err
+	}
+	fmt.Println(c.String())
+	return nil
+}

cmd/evm/eofparse_test.go

@@ -0,0 +1,166 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/hex"
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/vm"
+)
+
+func FuzzEofParsing(f *testing.F) {
+	// Seed with corpus from execution-spec-tests
+	for i := 0; ; i++ {
+		fname := fmt.Sprintf("testdata/eof/eof_corpus_%d.txt", i)
+		corpus, err := os.Open(fname)
+		if err != nil {
+			break
+		}
+		f.Logf("Reading seed data from %v", fname)
+		scanner := bufio.NewScanner(corpus)
+		scanner.Buffer(make([]byte, 1024), 10*1024*1024)
+		for scanner.Scan() {
+			s := scanner.Text()
+			if len(s) >= 2 && strings.HasPrefix(s, "0x") {
+				s = s[2:]
+			}
+			b, err := hex.DecodeString(s)
+			if err != nil {
+				panic(err) // rotten corpus
+			}
+			f.Add(b)
+		}
+		corpus.Close()
+		if err := scanner.Err(); err != nil {
+			panic(err) // rotten corpus
+		}
+	}
+	// And do the fuzzing
+	f.Fuzz(func(t *testing.T, data []byte) {
+		var (
+			jt = vm.NewPragueEOFInstructionSetForTesting()
+			c  vm.Container
+		)
+		cpy := common.CopyBytes(data)
+		if err := c.UnmarshalBinary(data, true); err == nil {
+			c.ValidateCode(&jt, true)
+			if have := c.MarshalBinary(); !bytes.Equal(have, data) {
+				t.Fatal("Unmarshal-> Marshal failure!")
+			}
+		}
+		if err := c.UnmarshalBinary(data, false); err == nil {
+			c.ValidateCode(&jt, false)
+			if have := c.MarshalBinary(); !bytes.Equal(have, data) {
+				t.Fatal("Unmarshal-> Marshal failure!")
+			}
+		}
+		if !bytes.Equal(cpy, data) {
+			panic("data modified during unmarshalling")
+		}
+	})
+}
+
+func TestEofParseInitcode(t *testing.T) {
+	testEofParse(t, true, "testdata/eof/results.initcode.txt")
+}
+
+func TestEofParseRegular(t *testing.T) {
+	testEofParse(t, false, "testdata/eof/results.regular.txt")
+}
+
+func testEofParse(t *testing.T, isInitCode bool, wantFile string) {
+	var wantFn func() string
+	var wantLoc = 0
+	{ // Configure the want-reader
+		wants, err := os.Open(wantFile)
+		if err != nil {
+			t.Fatal(err)
+		}
+		scanner := bufio.NewScanner(wants)
+		scanner.Buffer(make([]byte, 1024), 10*1024*1024)
+		wantFn = func() string {
+			if scanner.Scan() {
+				wantLoc++
+				return scanner.Text()
+			}
+			return "end of file reached"
+		}
+	}
+
+	for i := 0; ; i++ {
+		fname := fmt.Sprintf("testdata/eof/eof_corpus_%d.txt", i)
+		corpus, err := os.Open(fname)
+		if err != nil {
+			break
+		}
+		t.Logf("# Reading seed data from %v", fname)
+		scanner := bufio.NewScanner(corpus)
+		scanner.Buffer(make([]byte, 1024), 10*1024*1024)
+		line := 1
+		for scanner.Scan() {
+			s := scanner.Text()
+			if len(s) >= 2 && strings.HasPrefix(s, "0x") {
+				s = s[2:]
+			}
+			b, err := hex.DecodeString(s)
+			if err != nil {
+				panic(err) // rotten corpus
+			}
+			have := "OK"
+			if _, err := parse(b, isInitCode); err != nil {
+				have = fmt.Sprintf("ERR: %v", err)
+			}
+			if false { // Change this to generate the want-output
+				fmt.Printf("%v\n", have)
+			} else {
+				want := wantFn()
+				if have != want {
+					if len(want) > 100 {
+						want = want[:100]
+					}
+					if len(b) > 100 {
+						b = b[:100]
+					}
+					t.Errorf("%v:%d\n%v\ninput %x\nisInit: %v\nhave: %q\nwant: %q\n",
+						fname, line, fmt.Sprintf("%v:%d", wantFile, wantLoc), b, isInitCode, have, want)
+				}
+			}
+			line++
+		}
+		corpus.Close()
+	}
+}
+
+func BenchmarkEofParse(b *testing.B) {
+	corpus, err := os.Open("testdata/eof/eof_benches.txt")
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer corpus.Close()
+	scanner := bufio.NewScanner(corpus)
+	scanner.Buffer(make([]byte, 1024), 10*1024*1024)
+	line := 1
+	for scanner.Scan() {
+		s := scanner.Text()
+		if len(s) >= 2 && strings.HasPrefix(s, "0x") {
+			s = s[2:]
+		}
+		data, err := hex.DecodeString(s)
+		if err != nil {
+			b.Fatal(err) // rotten corpus
+		}
+		b.Run(fmt.Sprintf("test-%d", line), func(b *testing.B) {
+			b.ReportAllocs()
+			b.SetBytes(int64(len(data)))
+			for i := 0; i < b.N; i++ {
+				_, _ = parse(data, false)
+			}
+		})
+		line++
+	}
+}