From 96b75033c08c4edbfd3b3aa356f3e7d4cce4f2b8 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 25 Oct 2023 14:53:50 +0200 Subject: [PATCH 01/11] trie: use explicit errors in stacktrie (instead of panic) (#28361) This PR removes panics from stacktrie (mostly), and makes the Update return errors instead. While adding tests for this, I also found that one case of possible corruption was not caught, which is now fixed. --- core/state/snapshot/generate.go | 4 +++- trie/stacktrie.go | 9 ++++++--- trie/stacktrie_test.go | 22 ++++++++++++++++++++++ 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go index 204584c956..adeaa1daa0 100644 --- a/core/state/snapshot/generate.go +++ b/core/state/snapshot/generate.go @@ -230,7 +230,9 @@ func (dl *diskLayer) proveRange(ctx *generatorContext, trieId *trie.ID, prefix [ if origin == nil && !diskMore { stackTr := trie.NewStackTrie(nil) for i, key := range keys { - stackTr.Update(key, vals[i]) + if err := stackTr.Update(key, vals[i]); err != nil { + return nil, err + } } if gotRoot := stackTr.Hash(); gotRoot != root { return &proofResult{ diff --git a/trie/stacktrie.go b/trie/stacktrie.go index 423afdec88..f2f5355c49 100644 --- a/trie/stacktrie.go +++ b/trie/stacktrie.go @@ -18,6 +18,7 @@ package trie import ( "bytes" + "errors" "sync" "github.com/ethereum/go-ethereum/common" @@ -92,12 +93,14 @@ func NewStackTrie(options *StackTrieOptions) *StackTrie { // Update inserts a (key, value) pair into the stack trie. func (t *StackTrie) Update(key, value []byte) error { - k := keybytesToHex(key) if len(value) == 0 { - panic("deletion not supported") + return errors.New("trying to insert empty (deletion)") } + k := keybytesToHex(key) k = k[:len(k)-1] // chop the termination flag - + if bytes.Compare(t.last, k) >= 0 { + return errors.New("non-ascending key order") + } // track the first and last inserted entries. if t.first == nil { t.first = append([]byte{}, k...) diff --git a/trie/stacktrie_test.go b/trie/stacktrie_test.go index 629586e2b1..909a77062a 100644 --- a/trie/stacktrie_test.go +++ b/trie/stacktrie_test.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/trie/testutil" + "github.com/stretchr/testify/assert" "golang.org/x/exp/slices" ) @@ -463,3 +464,24 @@ func TestPartialStackTrie(t *testing.T) { } } } + +func TestStackTrieErrors(t *testing.T) { + s := NewStackTrie(nil) + // Deletion + if err := s.Update(nil, nil); err == nil { + t.Fatal("expected error") + } + if err := s.Update(nil, []byte{}); err == nil { + t.Fatal("expected error") + } + if err := s.Update([]byte{0xa}, []byte{}); err == nil { + t.Fatal("expected error") + } + // Non-ascending keys (going backwards or repeating) + assert.Nil(t, s.Update([]byte{0xaa}, []byte{0xa})) + assert.NotNil(t, s.Update([]byte{0xaa}, []byte{0xa}), "repeat insert same key") + assert.NotNil(t, s.Update([]byte{0xaa}, []byte{0xb}), "repeat insert same key") + assert.Nil(t, s.Update([]byte{0xab}, []byte{0xa})) + assert.NotNil(t, s.Update([]byte{0x10}, []byte{0xb}), "out of order insert") + assert.NotNil(t, s.Update([]byte{0xaa}, []byte{0xb}), "repeat insert same key") +} From f7b62e550696c3a7379d17228744c65e2dd29e7b Mon Sep 17 00:00:00 2001 From: Delweng Date: Wed, 25 Oct 2023 21:20:23 +0800 Subject: [PATCH 02/11] graphql: logs return error if from block > to (#28412) As per discussion in ethereum/execution-apis#475 Signed-off-by: jsvisa --- graphql/graphql.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/graphql/graphql.go b/graphql/graphql.go index ec7382f80a..50f0c6ca47 100644 --- a/graphql/graphql.go +++ b/graphql/graphql.go @@ -1417,6 +1417,9 @@ func (r *Resolver) Logs(ctx context.Context, args struct{ Filter FilterCriteria if args.Filter.ToBlock != nil { end = int64(*args.Filter.ToBlock) } + if begin > 0 && end > 0 && begin > end { + return nil, errInvalidBlockRange + } var addresses []common.Address if args.Filter.Addresses != nil { addresses = *args.Filter.Addresses From d8c6ae054c8e120c72d20eeac805f1b95e5802a2 Mon Sep 17 00:00:00 2001 From: Adrian Sutton Date: Wed, 25 Oct 2023 23:52:42 +1000 Subject: [PATCH 03/11] rpc: use correct stringer-method for serializing BlockNumberOrHash (#28358) The String() version of BlockNumberOrHash uses decimal for all block numbers, including negative ones used to indicate labels. Switch to using BlockNumber.String() which encodes it correctly for use in the JSON-RPC API. --- rpc/types.go | 3 +-- rpc/types_test.go | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/rpc/types.go b/rpc/types.go index 34a1451dea..f88c37c59d 100644 --- a/rpc/types.go +++ b/rpc/types.go @@ -21,7 +21,6 @@ import ( "encoding/json" "fmt" "math" - "strconv" "strings" "github.com/ethereum/go-ethereum/common" @@ -221,7 +220,7 @@ func (bnh *BlockNumberOrHash) Number() (BlockNumber, bool) { func (bnh *BlockNumberOrHash) String() string { if bnh.BlockNumber != nil { - return strconv.Itoa(int(*bnh.BlockNumber)) + return bnh.BlockNumber.String() } if bnh.BlockHash != nil { return bnh.BlockHash.String() diff --git a/rpc/types_test.go b/rpc/types_test.go index f110dee7c6..617f441d91 100644 --- a/rpc/types_test.go +++ b/rpc/types_test.go @@ -153,3 +153,24 @@ func TestBlockNumberOrHash_WithNumber_MarshalAndUnmarshal(t *testing.T) { }) } } + +func TestBlockNumberOrHash_StringAndUnmarshal(t *testing.T) { + tests := []BlockNumberOrHash{ + BlockNumberOrHashWithNumber(math.MaxInt64), + BlockNumberOrHashWithNumber(PendingBlockNumber), + BlockNumberOrHashWithNumber(LatestBlockNumber), + BlockNumberOrHashWithNumber(EarliestBlockNumber), + BlockNumberOrHashWithNumber(32), + BlockNumberOrHashWithHash(common.Hash{0xaa}, false), + } + for _, want := range tests { + marshalled, _ := json.Marshal(want.String()) + var have BlockNumberOrHash + if err := json.Unmarshal(marshalled, &have); err != nil { + t.Fatalf("cannot unmarshal (%v): %v", string(marshalled), err) + } + if !reflect.DeepEqual(want, have) { + t.Fatalf("wrong result: have %v, want %v", have, want) + } + } +} From 58ae1df6840e512b263a4fc2e021e1ec5637ca21 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 25 Oct 2023 17:57:12 +0200 Subject: [PATCH 04/11] cmd/geth: test for logging-output (#28373) This PR is a bit in preparation for the slog work in #28187 . Our current test re logging mostly test the internals, but we have no real end-to-end test of the logging output. This PR introduces a simple reexec-based log tester. This also relies upon a special mode in geth, which can be made to eject a set of predefined log messages (only available if the build-tag `integrationtests` is used e.g. go run --tags=integrationtests ./cmd/geth --log.format terminal logtest While working on this, I also noticed a quirk in the setup: when geth was configured to use a file output, then two separate handlers were used (one handler for the file, one handler for the console). Using two separate handlers means that two formatters are used, thus the formatting of any/all records happened twice. This PR changes the mechanism to use two separate io.Writers instead, which is both more optimal and fixes a bug which occurs due to a global statefulness in the formatter. --- build/ci.go | 3 + cmd/geth/logging_test.go | 185 ++++++++++++++++++ cmd/geth/logtestcmd_active.go | 134 +++++++++++++ cmd/geth/logtestcmd_inactive.go | 23 +++ cmd/geth/main.go | 3 + cmd/geth/testdata/logging/logtest-logfmt.txt | 39 ++++ .../testdata/logging/logtest-terminal.txt | 40 ++++ internal/debug/flags.go | 20 +- log/format.go | 8 + log/format_test.go | 139 ------------- 10 files changed, 445 insertions(+), 149 deletions(-) create mode 100644 cmd/geth/logging_test.go create mode 100644 cmd/geth/logtestcmd_active.go create mode 100644 cmd/geth/logtestcmd_inactive.go create mode 100644 cmd/geth/testdata/logging/logtest-logfmt.txt create mode 100644 cmd/geth/testdata/logging/logtest-terminal.txt diff --git a/build/ci.go b/build/ci.go index 46f1ac2812..afe1c332b8 100644 --- a/build/ci.go +++ b/build/ci.go @@ -307,6 +307,9 @@ func doTest(cmdline []string) { // Enable CKZG backend in CI. gotest.Args = append(gotest.Args, "-tags=ckzg") + // Enable integration-tests + gotest.Args = append(gotest.Args, "-tags=integrationtests") + // Test a single package at a time. CI builders are slow // and some tests run into timeouts under load. gotest.Args = append(gotest.Args, "-p", "1") diff --git a/cmd/geth/logging_test.go b/cmd/geth/logging_test.go new file mode 100644 index 0000000000..cc951d6e97 --- /dev/null +++ b/cmd/geth/logging_test.go @@ -0,0 +1,185 @@ +//go:build integrationtests + +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "bufio" + "bytes" + "fmt" + "io" + "math/rand" + "os" + "os/exec" + "strings" + "testing" + + "github.com/docker/docker/pkg/reexec" +) + +func runSelf(args ...string) ([]byte, error) { + cmd := &exec.Cmd{ + Path: reexec.Self(), + Args: append([]string{"geth-test"}, args...), + } + return cmd.CombinedOutput() +} + +func split(input io.Reader) []string { + var output []string + scanner := bufio.NewScanner(input) + scanner.Split(bufio.ScanLines) + for scanner.Scan() { + output = append(output, strings.TrimSpace(scanner.Text())) + } + return output +} + +func censor(input string, start, end int) string { + if len(input) < end { + return input + } + return input[:start] + strings.Repeat("X", end-start) + input[end:] +} + +func TestLogging(t *testing.T) { + testConsoleLogging(t, "terminal", 6, 24) + testConsoleLogging(t, "logfmt", 2, 26) +} + +func testConsoleLogging(t *testing.T, format string, tStart, tEnd int) { + haveB, err := runSelf("--log.format", format, "logtest") + if err != nil { + t.Fatal(err) + } + readFile, err := os.Open(fmt.Sprintf("testdata/logging/logtest-%v.txt", format)) + if err != nil { + t.Fatal(err) + } + wantLines := split(readFile) + haveLines := split(bytes.NewBuffer(haveB)) + for i, want := range wantLines { + if i > len(haveLines)-1 { + t.Fatalf("format %v, line %d missing, want:%v", format, i, want) + } + have := haveLines[i] + for strings.Contains(have, "Unknown config environment variable") { + // This can happen on CI runs. Drop it. + haveLines = append(haveLines[:i], haveLines[i+1:]...) + have = haveLines[i] + } + + // Black out the timestamp + have = censor(have, tStart, tEnd) + want = censor(want, tStart, tEnd) + if have != want { + t.Logf(nicediff([]byte(have), []byte(want))) + t.Fatalf("format %v, line %d\nhave %v\nwant %v", format, i, have, want) + } + } + if len(haveLines) != len(wantLines) { + t.Errorf("format %v, want %d lines, have %d", format, len(haveLines), len(wantLines)) + } +} + +func TestVmodule(t *testing.T) { + checkOutput := func(level int, want, wantNot string) { + t.Helper() + output, err := runSelf("--log.format", "terminal", "--verbosity=0", "--log.vmodule", fmt.Sprintf("logtestcmd_active.go=%d", level), "logtest") + if err != nil { + t.Fatal(err) + } + if len(want) > 0 && !strings.Contains(string(output), want) { // trace should be present at 5 + t.Errorf("failed to find expected string ('%s') in output", want) + } + if len(wantNot) > 0 && strings.Contains(string(output), wantNot) { // trace should be present at 5 + t.Errorf("string ('%s') should not be present in output", wantNot) + } + } + checkOutput(5, "log at level trace", "") // trace should be present at 5 + checkOutput(4, "log at level debug", "log at level trace") // debug should be present at 4, but trace should be missing + checkOutput(3, "log at level info", "log at level debug") // info should be present at 3, but debug should be missing + checkOutput(2, "log at level warn", "log at level info") // warn should be present at 2, but info should be missing + checkOutput(1, "log at level error", "log at level warn") // error should be present at 1, but warn should be missing +} + +func nicediff(have, want []byte) string { + var i = 0 + for ; i < len(have) && i < len(want); i++ { + if want[i] != have[i] { + break + } + } + var end = i + 40 + var start = i - 50 + if start < 0 { + start = 0 + } + var h, w string + if end < len(have) { + h = string(have[start:end]) + } else { + h = string(have[start:]) + } + if end < len(want) { + w = string(want[start:end]) + } else { + w = string(want[start:]) + } + return fmt.Sprintf("have vs want:\n%q\n%q\n", h, w) +} + +func TestFileOut(t *testing.T) { + var ( + have, want []byte + err error + path = fmt.Sprintf("%s/test_file_out-%d", os.TempDir(), rand.Int63()) + ) + t.Cleanup(func() { os.Remove(path) }) + if want, err = runSelf(fmt.Sprintf("--log.file=%s", path), "logtest"); err != nil { + t.Fatal(err) + } + if have, err = os.ReadFile(path); err != nil { + t.Fatal(err) + } + if !bytes.Equal(have, want) { + // show an intelligent diff + t.Logf(nicediff(have, want)) + t.Errorf("file content wrong") + } +} + +func TestRotatingFileOut(t *testing.T) { + var ( + have, want []byte + err error + path = fmt.Sprintf("%s/test_file_out-%d", os.TempDir(), rand.Int63()) + ) + t.Cleanup(func() { os.Remove(path) }) + if want, err = runSelf(fmt.Sprintf("--log.file=%s", path), "--log.rotate", "logtest"); err != nil { + t.Fatal(err) + } + if have, err = os.ReadFile(path); err != nil { + t.Fatal(err) + } + if !bytes.Equal(have, want) { + // show an intelligent diff + t.Logf(nicediff(have, want)) + t.Errorf("file content wrong") + } +} diff --git a/cmd/geth/logtestcmd_active.go b/cmd/geth/logtestcmd_active.go new file mode 100644 index 0000000000..c66013517a --- /dev/null +++ b/cmd/geth/logtestcmd_active.go @@ -0,0 +1,134 @@ +//go:build integrationtests + +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "fmt" + "math" + "math/big" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" + "github.com/holiman/uint256" + "github.com/urfave/cli/v2" +) + +var logTestCommand = &cli.Command{ + Action: logTest, + Name: "logtest", + Usage: "Print some log messages", + ArgsUsage: " ", + Description: ` +This command is only meant for testing. +`} + +// logTest is an entry point which spits out some logs. This is used by testing +// to verify expected outputs +func logTest(ctx *cli.Context) error { + log.ResetGlobalState() + { // big.Int + ba, _ := new(big.Int).SetString("111222333444555678999", 10) // "111,222,333,444,555,678,999" + bb, _ := new(big.Int).SetString("-111222333444555678999", 10) // "-111,222,333,444,555,678,999" + bc, _ := new(big.Int).SetString("11122233344455567899900", 10) // "11,122,233,344,455,567,899,900" + bd, _ := new(big.Int).SetString("-11122233344455567899900", 10) // "-11,122,233,344,455,567,899,900" + log.Info("big.Int", "111,222,333,444,555,678,999", ba) + log.Info("-big.Int", "-111,222,333,444,555,678,999", bb) + log.Info("big.Int", "11,122,233,344,455,567,899,900", bc) + log.Info("-big.Int", "-11,122,233,344,455,567,899,900", bd) + } + { //uint256 + ua, _ := uint256.FromDecimal("111222333444555678999") + ub, _ := uint256.FromDecimal("11122233344455567899900") + log.Info("uint256", "111,222,333,444,555,678,999", ua) + log.Info("uint256", "11,122,233,344,455,567,899,900", ub) + } + { // int64 + log.Info("int64", "1,000,000", int64(1000000)) + log.Info("int64", "-1,000,000", int64(-1000000)) + log.Info("int64", "9,223,372,036,854,775,807", int64(math.MaxInt64)) + log.Info("int64", "-9,223,372,036,854,775,808", int64(math.MinInt64)) + } + { // uint64 + log.Info("uint64", "1,000,000", uint64(1000000)) + log.Info("uint64", "18,446,744,073,709,551,615", uint64(math.MaxUint64)) + } + { // Special characters + log.Info("Special chars in value", "key", "special \r\n\t chars") + log.Info("Special chars in key", "special \n\t chars", "value") + + log.Info("nospace", "nospace", "nospace") + log.Info("with space", "with nospace", "with nospace") + + log.Info("Bash escapes in value", "key", "\u001b[1G\u001b[K\u001b[1A") + log.Info("Bash escapes in key", "\u001b[1G\u001b[K\u001b[1A", "value") + + log.Info("Bash escapes in message \u001b[1G\u001b[K\u001b[1A end", "key", "value") + + colored := fmt.Sprintf("\u001B[%dmColored\u001B[0m[", 35) + log.Info(colored, colored, colored) + } + { // Custom Stringer() - type + log.Info("Custom Stringer value", "2562047h47m16.854s", common.PrettyDuration(time.Duration(9223372036854775807))) + } + { // Lazy eval + log.Info("Lazy evaluation of value", "key", log.Lazy{Fn: func() interface{} { return "lazy value" }}) + } + { // Multi-line message + log.Info("A message with wonky \U0001F4A9 characters") + log.Info("A multiline message \nINFO [10-18|14:11:31.106] with wonky characters \U0001F4A9") + log.Info("A multiline message \nLALA [ZZZZZZZZZZZZZZZZZZ] Actually part of message above") + } + { // Miscellaneous json-quirks + // This will check if the json output uses strings or json-booleans to represent bool values + log.Info("boolean", "true", true, "false", false) + // Handling of duplicate keys. + // This is actually ill-handled by the current handler: the format.go + // uses a global 'fieldPadding' map and mixes up the two keys. If 'alpha' + // is shorter than beta, it sometimes causes erroneous padding -- and what's more + // it causes _different_ padding in multi-handler context, e.g. both file- + // and console output, making the two mismatch. + log.Info("repeated-key 1", "foo", "alpha", "foo", "beta") + log.Info("repeated-key 2", "xx", "short", "xx", "longer") + } + { // loglevels + log.Debug("log at level debug") + log.Trace("log at level trace") + log.Info("log at level info") + log.Warn("log at level warn") + log.Error("log at level error") + } + { + // The current log formatter has a global map of paddings, storing the + // longest seen padding per key in a map. This results in a statefulness + // which has some odd side-effects. Demonstrated here: + log.Info("test", "bar", "short", "a", "aligned left") + log.Info("test", "bar", "a long message", "a", 1) + log.Info("test", "bar", "short", "a", "aligned right") + } + { + // This sequence of logs should be output with alignment, so each field becoems a column. + log.Info("The following logs should align so that the key-fields make 5 columns") + log.Info("Inserted known block", "number", 1_012, "hash", common.HexToHash("0x1234"), "txs", 200, "gas", 1_123_123, "other", "first") + log.Info("Inserted new block", "number", 1, "hash", common.HexToHash("0x1235"), "txs", 2, "gas", 1_123, "other", "second") + log.Info("Inserted known block", "number", 99, "hash", common.HexToHash("0x12322"), "txs", 10, "gas", 1, "other", "third") + log.Warn("Inserted known block", "number", 1_012, "hash", common.HexToHash("0x1234"), "txs", 200, "gas", 99, "other", "fourth") + } + return nil +} diff --git a/cmd/geth/logtestcmd_inactive.go b/cmd/geth/logtestcmd_inactive.go new file mode 100644 index 0000000000..691ab5bcd8 --- /dev/null +++ b/cmd/geth/logtestcmd_inactive.go @@ -0,0 +1,23 @@ +//go:build !integrationtests + +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import "github.com/urfave/cli/v2" + +var logTestCommand *cli.Command diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 4b26de05a9..2d4fe3dc06 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -234,6 +234,9 @@ func init() { // See verkle.go verkleCommand, } + if logTestCommand != nil { + app.Commands = append(app.Commands, logTestCommand) + } sort.Sort(cli.CommandsByName(app.Commands)) app.Flags = flags.Merge( diff --git a/cmd/geth/testdata/logging/logtest-logfmt.txt b/cmd/geth/testdata/logging/logtest-logfmt.txt new file mode 100644 index 0000000000..79f29e1fae --- /dev/null +++ b/cmd/geth/testdata/logging/logtest-logfmt.txt @@ -0,0 +1,39 @@ +t=2023-10-20T12:56:08+0200 lvl=info msg=big.Int 111,222,333,444,555,678,999=111,222,333,444,555,678,999 +t=2023-10-20T12:56:08+0200 lvl=info msg=-big.Int -111,222,333,444,555,678,999=-111,222,333,444,555,678,999 +t=2023-10-20T12:56:08+0200 lvl=info msg=big.Int 11,122,233,344,455,567,899,900=11,122,233,344,455,567,899,900 +t=2023-10-20T12:56:08+0200 lvl=info msg=-big.Int -11,122,233,344,455,567,899,900=-11,122,233,344,455,567,899,900 +t=2023-10-20T12:56:08+0200 lvl=info msg=uint256 111,222,333,444,555,678,999=111,222,333,444,555,678,999 +t=2023-10-20T12:56:08+0200 lvl=info msg=uint256 11,122,233,344,455,567,899,900=11,122,233,344,455,567,899,900 +t=2023-10-20T12:56:08+0200 lvl=info msg=int64 1,000,000=1,000,000 +t=2023-10-20T12:56:08+0200 lvl=info msg=int64 -1,000,000=-1,000,000 +t=2023-10-20T12:56:08+0200 lvl=info msg=int64 9,223,372,036,854,775,807=9,223,372,036,854,775,807 +t=2023-10-20T12:56:08+0200 lvl=info msg=int64 -9,223,372,036,854,775,808=-9,223,372,036,854,775,808 +t=2023-10-20T12:56:08+0200 lvl=info msg=uint64 1,000,000=1,000,000 +t=2023-10-20T12:56:08+0200 lvl=info msg=uint64 18,446,744,073,709,551,615=18,446,744,073,709,551,615 +t=2023-10-20T12:56:08+0200 lvl=info msg="Special chars in value" key="special \r\n\t chars" +t=2023-10-20T12:56:08+0200 lvl=info msg="Special chars in key" "special \n\t chars"=value +t=2023-10-20T12:56:08+0200 lvl=info msg=nospace nospace=nospace +t=2023-10-20T12:56:08+0200 lvl=info msg="with space" "with nospace"="with nospace" +t=2023-10-20T12:56:08+0200 lvl=info msg="Bash escapes in value" key="\x1b[1G\x1b[K\x1b[1A" +t=2023-10-20T12:56:08+0200 lvl=info msg="Bash escapes in key" "\x1b[1G\x1b[K\x1b[1A"=value +t=2023-10-20T12:56:08+0200 lvl=info msg="Bash escapes in message \x1b[1G\x1b[K\x1b[1A end" key=value +t=2023-10-20T12:56:08+0200 lvl=info msg="\x1b[35mColored\x1b[0m[" "\x1b[35mColored\x1b[0m["="\x1b[35mColored\x1b[0m[" +t=2023-10-20T12:56:08+0200 lvl=info msg="Custom Stringer value" 2562047h47m16.854s=2562047h47m16.854s +t=2023-10-20T12:56:08+0200 lvl=info msg="Lazy evaluation of value" key="lazy value" +t=2023-10-20T12:56:08+0200 lvl=info msg="A message with wonky 💩 characters" +t=2023-10-20T12:56:08+0200 lvl=info msg="A multiline message \nINFO [10-18|14:11:31.106] with wonky characters 💩" +t=2023-10-20T12:56:08+0200 lvl=info msg="A multiline message \nLALA [ZZZZZZZZZZZZZZZZZZ] Actually part of message above" +t=2023-10-20T12:56:08+0200 lvl=info msg=boolean true=true false=false +t=2023-10-20T12:56:08+0200 lvl=info msg="repeated-key 1" foo=alpha foo=beta +t=2023-10-20T12:56:08+0200 lvl=info msg="repeated-key 2" xx=short xx=longer +t=2023-10-20T12:56:08+0200 lvl=info msg="log at level info" +t=2023-10-20T12:56:08+0200 lvl=warn msg="log at level warn" +t=2023-10-20T12:56:08+0200 lvl=eror msg="log at level error" +t=2023-10-20T12:56:08+0200 lvl=info msg=test bar=short a="aligned left" +t=2023-10-20T12:56:08+0200 lvl=info msg=test bar="a long message" a=1 +t=2023-10-20T12:56:08+0200 lvl=info msg=test bar=short a="aligned right" +t=2023-10-20T12:56:08+0200 lvl=info msg="The following logs should align so that the key-fields make 5 columns" +t=2023-10-20T12:56:08+0200 lvl=info msg="Inserted known block" number=1012 hash=0x0000000000000000000000000000000000000000000000000000000000001234 txs=200 gas=1,123,123 other=first +t=2023-10-20T12:56:08+0200 lvl=info msg="Inserted new block" number=1 hash=0x0000000000000000000000000000000000000000000000000000000000001235 txs=2 gas=1123 other=second +t=2023-10-20T12:56:08+0200 lvl=info msg="Inserted known block" number=99 hash=0x0000000000000000000000000000000000000000000000000000000000012322 txs=10 gas=1 other=third +t=2023-10-20T12:56:08+0200 lvl=warn msg="Inserted known block" number=1012 hash=0x0000000000000000000000000000000000000000000000000000000000001234 txs=200 gas=99 other=fourth diff --git a/cmd/geth/testdata/logging/logtest-terminal.txt b/cmd/geth/testdata/logging/logtest-terminal.txt new file mode 100644 index 0000000000..ff68b6047a --- /dev/null +++ b/cmd/geth/testdata/logging/logtest-terminal.txt @@ -0,0 +1,40 @@ +INFO [10-20|12:56:42.532] big.Int 111,222,333,444,555,678,999=111,222,333,444,555,678,999 +INFO [10-20|12:56:42.532] -big.Int -111,222,333,444,555,678,999=-111,222,333,444,555,678,999 +INFO [10-20|12:56:42.532] big.Int 11,122,233,344,455,567,899,900=11,122,233,344,455,567,899,900 +INFO [10-20|12:56:42.532] -big.Int -11,122,233,344,455,567,899,900=-11,122,233,344,455,567,899,900 +INFO [10-20|12:56:42.532] uint256 111,222,333,444,555,678,999=111,222,333,444,555,678,999 +INFO [10-20|12:56:42.532] uint256 11,122,233,344,455,567,899,900=11,122,233,344,455,567,899,900 +INFO [10-20|12:56:42.532] int64 1,000,000=1,000,000 +INFO [10-20|12:56:42.532] int64 -1,000,000=-1,000,000 +INFO [10-20|12:56:42.532] int64 9,223,372,036,854,775,807=9,223,372,036,854,775,807 +INFO [10-20|12:56:42.532] int64 -9,223,372,036,854,775,808=-9,223,372,036,854,775,808 +INFO [10-20|12:56:42.532] uint64 1,000,000=1,000,000 +INFO [10-20|12:56:42.532] uint64 18,446,744,073,709,551,615=18,446,744,073,709,551,615 +INFO [10-20|12:56:42.532] Special chars in value key="special \r\n\t chars" +INFO [10-20|12:56:42.532] Special chars in key "special \n\t chars"=value +INFO [10-20|12:56:42.532] nospace nospace=nospace +INFO [10-20|12:56:42.532] with space "with nospace"="with nospace" +INFO [10-20|12:56:42.532] Bash escapes in value key="\x1b[1G\x1b[K\x1b[1A" +INFO [10-20|12:56:42.532] Bash escapes in key "\x1b[1G\x1b[K\x1b[1A"=value +INFO [10-20|12:56:42.532] "Bash escapes in message \x1b[1G\x1b[K\x1b[1A end" key=value +INFO [10-20|12:56:42.532] "\x1b[35mColored\x1b[0m[" "\x1b[35mColored\x1b[0m["="\x1b[35mColored\x1b[0m[" +INFO [10-20|12:56:42.532] Custom Stringer value 2562047h47m16.854s=2562047h47m16.854s +INFO [10-20|12:56:42.532] Lazy evaluation of value key="lazy value" +INFO [10-20|12:56:42.532] "A message with wonky 💩 characters" +INFO [10-20|12:56:42.532] "A multiline message \nINFO [10-18|14:11:31.106] with wonky characters 💩" +INFO [10-20|12:56:42.532] A multiline message +LALA [ZZZZZZZZZZZZZZZZZZ] Actually part of message above +INFO [10-20|12:56:42.532] boolean true=true false=false +INFO [10-20|12:56:42.532] repeated-key 1 foo=alpha foo=beta +INFO [10-20|12:56:42.532] repeated-key 2 xx=short xx=longer +INFO [10-20|12:56:42.532] log at level info +WARN [10-20|12:56:42.532] log at level warn +ERROR[10-20|12:56:42.532] log at level error +INFO [10-20|12:56:42.532] test bar=short a="aligned left" +INFO [10-20|12:56:42.532] test bar="a long message" a=1 +INFO [10-20|12:56:42.532] test bar=short a="aligned right" +INFO [10-20|12:56:42.532] The following logs should align so that the key-fields make 5 columns +INFO [10-20|12:56:42.532] Inserted known block number=1012 hash=000000..001234 txs=200 gas=1,123,123 other=first +INFO [10-20|12:56:42.532] Inserted new block number=1 hash=000000..001235 txs=2 gas=1123 other=second +INFO [10-20|12:56:42.532] Inserted known block number=99 hash=000000..012322 txs=10 gas=1 other=third +WARN [10-20|12:56:42.532] Inserted known block number=1012 hash=000000..001234 txs=200 gas=99 other=fourth diff --git a/internal/debug/flags.go b/internal/debug/flags.go index 736fede943..4f0f5fe860 100644 --- a/internal/debug/flags.go +++ b/internal/debug/flags.go @@ -218,10 +218,9 @@ func Setup(ctx *cli.Context) error { return fmt.Errorf("unknown log format: %v", ctx.String(logFormatFlag.Name)) } var ( - stdHandler = log.StreamHandler(output, logfmt) - ostream = stdHandler - logFile = ctx.String(logFileFlag.Name) - rotation = ctx.Bool(logRotateFlag.Name) + ostream = log.StreamHandler(output, logfmt) + logFile = ctx.String(logFileFlag.Name) + rotation = ctx.Bool(logRotateFlag.Name) ) if len(logFile) > 0 { if err := validateLogLocation(filepath.Dir(logFile)); err != nil { @@ -242,20 +241,21 @@ func Setup(ctx *cli.Context) error { } else { context = append(context, "location", filepath.Join(os.TempDir(), "geth-lumberjack.log")) } - ostream = log.MultiHandler(log.StreamHandler(&lumberjack.Logger{ + lumberWriter := &lumberjack.Logger{ Filename: logFile, MaxSize: ctx.Int(logMaxSizeMBsFlag.Name), MaxBackups: ctx.Int(logMaxBackupsFlag.Name), MaxAge: ctx.Int(logMaxAgeFlag.Name), Compress: ctx.Bool(logCompressFlag.Name), - }, logfmt), stdHandler) + } + ostream = log.StreamHandler(io.MultiWriter(output, lumberWriter), logfmt) } else if logFile != "" { - if logOutputStream, err := log.FileHandler(logFile, logfmt); err != nil { + f, err := os.OpenFile(logFile, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { return err - } else { - ostream = log.MultiHandler(logOutputStream, stdHandler) - context = append(context, "location", logFile) } + ostream = log.StreamHandler(io.MultiWriter(output, f), logfmt) + context = append(context, "location", logFile) } glogger.SetHandler(ostream) diff --git a/log/format.go b/log/format.go index 1adf79c17e..2fd1f28558 100644 --- a/log/format.go +++ b/log/format.go @@ -24,6 +24,14 @@ const ( termCtxMaxPadding = 40 ) +// ResetGlobalState resets the fieldPadding, which is useful for producing +// predictable output. +func ResetGlobalState() { + fieldPaddingLock.Lock() + fieldPadding = make(map[string]int) + fieldPaddingLock.Unlock() +} + // locationTrims are trimmed for display to avoid unwieldy log lines. var locationTrims = []string{ "github.com/ethereum/go-ethereum/", diff --git a/log/format_test.go b/log/format_test.go index e08c1d1a4a..41e1809c38 100644 --- a/log/format_test.go +++ b/log/format_test.go @@ -1,105 +1,10 @@ package log import ( - "fmt" - "math" - "math/big" "math/rand" - "strings" "testing" - - "github.com/holiman/uint256" ) -func TestPrettyInt64(t *testing.T) { - tests := []struct { - n int64 - s string - }{ - {0, "0"}, - {10, "10"}, - {-10, "-10"}, - {100, "100"}, - {-100, "-100"}, - {1000, "1000"}, - {-1000, "-1000"}, - {10000, "10000"}, - {-10000, "-10000"}, - {99999, "99999"}, - {-99999, "-99999"}, - {100000, "100,000"}, - {-100000, "-100,000"}, - {1000000, "1,000,000"}, - {-1000000, "-1,000,000"}, - {math.MaxInt64, "9,223,372,036,854,775,807"}, - {math.MinInt64, "-9,223,372,036,854,775,808"}, - } - for i, tt := range tests { - if have := FormatLogfmtInt64(tt.n); have != tt.s { - t.Errorf("test %d: format mismatch: have %s, want %s", i, have, tt.s) - } - } -} - -func TestPrettyUint64(t *testing.T) { - tests := []struct { - n uint64 - s string - }{ - {0, "0"}, - {10, "10"}, - {100, "100"}, - {1000, "1000"}, - {10000, "10000"}, - {99999, "99999"}, - {100000, "100,000"}, - {1000000, "1,000,000"}, - {math.MaxUint64, "18,446,744,073,709,551,615"}, - } - for i, tt := range tests { - if have := FormatLogfmtUint64(tt.n); have != tt.s { - t.Errorf("test %d: format mismatch: have %s, want %s", i, have, tt.s) - } - } -} - -func TestPrettyBigInt(t *testing.T) { - tests := []struct { - int string - s string - }{ - {"111222333444555678999", "111,222,333,444,555,678,999"}, - {"-111222333444555678999", "-111,222,333,444,555,678,999"}, - {"11122233344455567899900", "11,122,233,344,455,567,899,900"}, - {"-11122233344455567899900", "-11,122,233,344,455,567,899,900"}, - } - - for _, tt := range tests { - v, _ := new(big.Int).SetString(tt.int, 10) - if have := formatLogfmtBigInt(v); have != tt.s { - t.Errorf("invalid output %s, want %s", have, tt.s) - } - } -} - -func TestPrettyUint256(t *testing.T) { - tests := []struct { - int string - s string - }{ - {"111222333444555678999", "111,222,333,444,555,678,999"}, - {"11122233344455567899900", "11,122,233,344,455,567,899,900"}, - } - - for _, tt := range tests { - v := new(uint256.Int) - v.SetFromDecimal(tt.int) - if have := formatLogfmtUint256(v); have != tt.s { - t.Errorf("invalid output %s, want %s", have, tt.s) - } - } -} - var sink string func BenchmarkPrettyInt64Logfmt(b *testing.B) { @@ -115,47 +20,3 @@ func BenchmarkPrettyUint64Logfmt(b *testing.B) { sink = FormatLogfmtUint64(rand.Uint64()) } } - -func TestSanitation(t *testing.T) { - msg := "\u001b[1G\u001b[K\u001b[1A" - msg2 := "\u001b \u0000" - msg3 := "NiceMessage" - msg4 := "Space Message" - msg5 := "Enter\nMessage" - - for i, tt := range []struct { - msg string - want string - }{ - { - msg: msg, - want: fmt.Sprintf("] %q %q=%q\n", msg, msg, msg), - }, - { - msg: msg2, - want: fmt.Sprintf("] %q %q=%q\n", msg2, msg2, msg2), - }, - { - msg: msg3, - want: fmt.Sprintf("] %s %s=%s\n", msg3, msg3, msg3), - }, - { - msg: msg4, - want: fmt.Sprintf("] %s %q=%q\n", msg4, msg4, msg4), - }, - { - msg: msg5, - want: fmt.Sprintf("] %s %q=%q\n", msg5, msg5, msg5), - }, - } { - var ( - logger = New() - out = new(strings.Builder) - ) - logger.SetHandler(LvlFilterHandler(LvlInfo, StreamHandler(out, TerminalFormat(false)))) - logger.Info(tt.msg, tt.msg, tt.msg) - if have := out.String()[24:]; tt.want != have { - t.Fatalf("test %d: want / have: \n%v\n%v", i, tt.want, have) - } - } -} From abe3fca1deb0eca9245cccef3a9c637c57b79f7e Mon Sep 17 00:00:00 2001 From: Delweng Date: Thu, 26 Oct 2023 19:48:51 +0800 Subject: [PATCH 05/11] graphql: fix an issue of nil pointer panic (#28416) Signed-off-by: jsvisa --- graphql/graphql.go | 3 +++ graphql/graphql_test.go | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/graphql/graphql.go b/graphql/graphql.go index 50f0c6ca47..93313d743a 100644 --- a/graphql/graphql.go +++ b/graphql/graphql.go @@ -1325,6 +1325,9 @@ func (r *Resolver) Blocks(ctx context.Context, args struct { From *Long To *Long }) ([]*Block, error) { + if args.From == nil { + return nil, errors.New("from block number must be specified") + } from := rpc.BlockNumber(*args.From) var to rpc.BlockNumber diff --git a/graphql/graphql_test.go b/graphql/graphql_test.go index 4bbfb7251d..540a56778c 100644 --- a/graphql/graphql_test.go +++ b/graphql/graphql_test.go @@ -148,6 +148,11 @@ func TestGraphQLBlockSerialization(t *testing.T) { want: `{"data":{"block":{"number":"0xa","call":{"data":"0x","status":"0x1"}}}}`, code: 200, }, + { + body: `{"query": "{blocks {number}}"}`, + want: `{"errors":[{"message":"from block number must be specified","path":["blocks"]}],"data":null}`, + code: 400, + }, } { resp, err := http.Post(fmt.Sprintf("%s/graphql", stack.HTTPEndpoint()), "application/json", strings.NewReader(tt.body)) if err != nil { From 4cbca5178a2e5f575f1700c0d49a4324b592f09f Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Fri, 27 Oct 2023 04:15:14 +0200 Subject: [PATCH 06/11] core, cmd/geth: add --override.* flags to geth init (#28407) * core, cmd/geth: add --override.* flags to geth init * also apply overrides before genesis commit with new block * review feedback --- cmd/geth/chaincmd.go | 13 ++++++++++++- core/genesis.go | 5 +++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index a6bb2c2d2c..5663963e3c 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -50,6 +50,8 @@ var ( ArgsUsage: "", Flags: flags.Merge([]cli.Flag{ utils.CachePreimagesFlag, + utils.OverrideCancun, + utils.OverrideVerkle, }, utils.DatabaseFlags), Description: ` The init command initializes a new genesis block and definition for the network. @@ -193,6 +195,15 @@ func initGenesis(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() + var overrides core.ChainOverrides + if ctx.IsSet(utils.OverrideCancun.Name) { + v := ctx.Uint64(utils.OverrideCancun.Name) + overrides.OverrideCancun = &v + } + if ctx.IsSet(utils.OverrideVerkle.Name) { + v := ctx.Uint64(utils.OverrideVerkle.Name) + overrides.OverrideVerkle = &v + } for _, name := range []string{"chaindata", "lightchaindata"} { chaindb, err := stack.OpenDatabaseWithFreezer(name, 0, 0, ctx.String(utils.AncientFlag.Name), "", false) if err != nil { @@ -203,7 +214,7 @@ func initGenesis(ctx *cli.Context) error { triedb := utils.MakeTrieDatabase(ctx, chaindb, ctx.Bool(utils.CachePreimagesFlag.Name), false) defer triedb.Close() - _, hash, err := core.SetupGenesisBlock(chaindb, triedb, genesis) + _, hash, err := core.SetupGenesisBlockWithOverride(chaindb, triedb, genesis, &overrides) if err != nil { utils.Fatalf("Failed to write genesis block: %v", err) } diff --git a/core/genesis.go b/core/genesis.go index 0f1e8baf43..1045815fab 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -287,11 +287,11 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *trie.Database, gen } else { log.Info("Writing custom genesis block") } + applyOverrides(genesis.Config) block, err := genesis.Commit(db, triedb) if err != nil { return genesis.Config, common.Hash{}, err } - applyOverrides(genesis.Config) return genesis.Config, block.Hash(), nil } // The genesis block is present(perhaps in ancient database) while the @@ -303,6 +303,7 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *trie.Database, gen if genesis == nil { genesis = DefaultGenesisBlock() } + applyOverrides(genesis.Config) // Ensure the stored genesis matches with the given one. hash := genesis.ToBlock().Hash() if hash != stored { @@ -312,11 +313,11 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *trie.Database, gen if err != nil { return genesis.Config, hash, err } - applyOverrides(genesis.Config) return genesis.Config, block.Hash(), nil } // Check whether the genesis block is already written. if genesis != nil { + applyOverrides(genesis.Config) hash := genesis.ToBlock().Hash() if hash != stored { return genesis.Config, hash, &GenesisMismatchError{stored, hash} From 2d7dba024d76603398907a595da98ad4df81b858 Mon Sep 17 00:00:00 2001 From: Delweng Date: Fri, 27 Oct 2023 22:50:59 +0800 Subject: [PATCH 07/11] graphql: always set content-type to application/json (#28417) --------- Signed-off-by: jsvisa --- graphql/graphql_test.go | 3 +++ graphql/service.go | 8 ++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/graphql/graphql_test.go b/graphql/graphql_test.go index 540a56778c..a83d6bbd46 100644 --- a/graphql/graphql_test.go +++ b/graphql/graphql_test.go @@ -169,6 +169,9 @@ func TestGraphQLBlockSerialization(t *testing.T) { if tt.code != resp.StatusCode { t.Errorf("testcase %d %s,\nwrong statuscode, have: %v, want: %v", i, tt.body, resp.StatusCode, tt.code) } + if ctype := resp.Header.Get("Content-Type"); ctype != "application/json" { + t.Errorf("testcase %d \nwrong Content-Type, have: %v, want: %v", i, ctype, "application/json") + } } } diff --git a/graphql/service.go b/graphql/service.go index f33e763058..584165bdb8 100644 --- a/graphql/service.go +++ b/graphql/service.go @@ -73,12 +73,12 @@ func (h handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } // Setting this disables gzip compression in package node. - w.Header().Set("transfer-encoding", "identity") + w.Header().Set("Transfer-Encoding", "identity") // Flush the response. Since we are writing close to the response timeout, // chunked transfer encoding must be disabled by setting content-length. - w.Header().Set("content-type", "application/json") - w.Header().Set("content-length", strconv.Itoa(len(responseJSON))) + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Length", strconv.Itoa(len(responseJSON))) w.Write(responseJSON) if flush, ok := w.(http.Flusher); ok { flush.Flush() @@ -97,10 +97,10 @@ func (h handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusInternalServerError) return } + w.Header().Set("Content-Type", "application/json") if len(response.Errors) > 0 { w.WriteHeader(http.StatusBadRequest) } - w.Header().Set("Content-Type", "application/json") w.Write(responseJSON) }) } From 233db64cc1d083e6251abe768c97e0454e2ca898 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Sat, 28 Oct 2023 00:14:43 +0200 Subject: [PATCH 08/11] all: make vendored copy of reexec (#28382) a little copying is better than a little dependency -- go proverb We have this dependency on docker, a.k.a moby: a gigantic library, and we only need ~70 LOC, so here I tried moving it inline instead. Co-authored-by: Felix Lange --- build/update-license.go | 3 +-- cmd/clef/run_test.go | 2 +- cmd/ethkey/run_test.go | 2 +- cmd/evm/t8n_test.go | 2 +- cmd/geth/logging_test.go | 2 +- cmd/geth/run_test.go | 2 +- go.mod | 2 -- go.sum | 4 ---- internal/cmdtest/test_cmd.go | 2 +- internal/reexec/reexec.go | 35 +++++++++++++++++++++++++++++++ internal/reexec/self_linux.go | 14 +++++++++++++ internal/reexec/self_others.go | 32 ++++++++++++++++++++++++++++ p2p/simulations/adapters/exec.go | 2 +- p2p/simulations/adapters/types.go | 2 +- 14 files changed, 90 insertions(+), 16 deletions(-) create mode 100644 internal/reexec/reexec.go create mode 100644 internal/reexec/self_linux.go create mode 100644 internal/reexec/self_others.go diff --git a/build/update-license.go b/build/update-license.go index 52a54bf66a..70e2de06c7 100644 --- a/build/update-license.go +++ b/build/update-license.go @@ -65,10 +65,8 @@ var ( "vendor/", "tests/testdata/", "build/", // don't relicense vendored sources - "cmd/internal/browser", "common/bitutil/bitutil", "common/prque/", - "consensus/ethash/xor.go", "crypto/blake2b/", "crypto/bn256/", "crypto/bls12381/", @@ -78,6 +76,7 @@ var ( "log/", "metrics/", "signer/rules/deps", + "internal/reexec", // skip special licenses "crypto/secp256k1", // Relicensed to BSD-3 via https://github.com/ethereum/go-ethereum/pull/17225 diff --git a/cmd/clef/run_test.go b/cmd/clef/run_test.go index fc3145b1e0..5fa6e02e14 100644 --- a/cmd/clef/run_test.go +++ b/cmd/clef/run_test.go @@ -21,8 +21,8 @@ import ( "os" "testing" - "github.com/docker/docker/pkg/reexec" "github.com/ethereum/go-ethereum/internal/cmdtest" + "github.com/ethereum/go-ethereum/internal/reexec" ) const registeredName = "clef-test" diff --git a/cmd/ethkey/run_test.go b/cmd/ethkey/run_test.go index 6006f6b5bb..73506e5da1 100644 --- a/cmd/ethkey/run_test.go +++ b/cmd/ethkey/run_test.go @@ -21,8 +21,8 @@ import ( "os" "testing" - "github.com/docker/docker/pkg/reexec" "github.com/ethereum/go-ethereum/internal/cmdtest" + "github.com/ethereum/go-ethereum/internal/reexec" ) type testEthkey struct { diff --git a/cmd/evm/t8n_test.go b/cmd/evm/t8n_test.go index efacbe95ae..03503d11c3 100644 --- a/cmd/evm/t8n_test.go +++ b/cmd/evm/t8n_test.go @@ -24,9 +24,9 @@ import ( "strings" "testing" - "github.com/docker/docker/pkg/reexec" "github.com/ethereum/go-ethereum/cmd/evm/internal/t8ntool" "github.com/ethereum/go-ethereum/internal/cmdtest" + "github.com/ethereum/go-ethereum/internal/reexec" ) func TestMain(m *testing.M) { diff --git a/cmd/geth/logging_test.go b/cmd/geth/logging_test.go index cc951d6e97..af50e93f94 100644 --- a/cmd/geth/logging_test.go +++ b/cmd/geth/logging_test.go @@ -29,7 +29,7 @@ import ( "strings" "testing" - "github.com/docker/docker/pkg/reexec" + "github.com/ethereum/go-ethereum/internal/reexec" ) func runSelf(args ...string) ([]byte, error) { diff --git a/cmd/geth/run_test.go b/cmd/geth/run_test.go index 0588623acb..2e03dc5eaa 100644 --- a/cmd/geth/run_test.go +++ b/cmd/geth/run_test.go @@ -23,8 +23,8 @@ import ( "testing" "time" - "github.com/docker/docker/pkg/reexec" "github.com/ethereum/go-ethereum/internal/cmdtest" + "github.com/ethereum/go-ethereum/internal/reexec" "github.com/ethereum/go-ethereum/rpc" ) diff --git a/go.mod b/go.mod index 4901030311..385d5afdc8 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,6 @@ require ( github.com/crate-crypto/go-kzg-4844 v0.7.0 github.com/davecgh/go-spew v1.1.1 github.com/deckarep/golang-set/v2 v2.1.0 - github.com/docker/docker v24.0.5+incompatible github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127 github.com/ethereum/c-kzg-4844 v0.4.0 github.com/fatih/color v1.13.0 @@ -141,6 +140,5 @@ require ( golang.org/x/net v0.17.0 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gotest.tools/v3 v3.5.1 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) diff --git a/go.sum b/go.sum index 6017c9f77e..cc38e7975f 100644 --- a/go.sum +++ b/go.sum @@ -169,8 +169,6 @@ github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwu github.com/dlclark/regexp2 v1.7.0 h1:7lJfhqlPssTb1WQx4yvTHN0uElPEv52sbaECrAQxjAo= github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= -github.com/docker/docker v24.0.5+incompatible h1:WmgcE4fxyI6EEXxBRxsHnZXrO1pQ3smi0k/jho4HLeY= -github.com/docker/docker v24.0.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127 h1:qwcF+vdFrvPSEUDSX5RVoRccG8a5DhOdWdQ4zN62zzo= github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= @@ -963,8 +961,6 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= -gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/internal/cmdtest/test_cmd.go b/internal/cmdtest/test_cmd.go index 43137053c1..4890d0b7c6 100644 --- a/internal/cmdtest/test_cmd.go +++ b/internal/cmdtest/test_cmd.go @@ -32,7 +32,7 @@ import ( "text/template" "time" - "github.com/docker/docker/pkg/reexec" + "github.com/ethereum/go-ethereum/internal/reexec" ) func NewTestCmd(t *testing.T, data interface{}) *TestCmd { diff --git a/internal/reexec/reexec.go b/internal/reexec/reexec.go new file mode 100644 index 0000000000..af8d347986 --- /dev/null +++ b/internal/reexec/reexec.go @@ -0,0 +1,35 @@ +// This file originates from Docker/Moby, +// https://github.com/moby/moby/blob/master/pkg/reexec/reexec.go +// Licensed under Apache License 2.0: https://github.com/moby/moby/blob/master/LICENSE +// Copyright 2013-2018 Docker, Inc. +// +// Package reexec facilitates the busybox style reexec of the docker binary that +// we require because of the forking limitations of using Go. Handlers can be +// registered with a name and the argv 0 of the exec of the binary will be used +// to find and execute custom init paths. +package reexec + +import ( + "fmt" + "os" +) + +var registeredInitializers = make(map[string]func()) + +// Register adds an initialization func under the specified name +func Register(name string, initializer func()) { + if _, exists := registeredInitializers[name]; exists { + panic(fmt.Sprintf("reexec func already registered under name %q", name)) + } + registeredInitializers[name] = initializer +} + +// Init is called as the first part of the exec process and returns true if an +// initialization function was called. +func Init() bool { + if initializer, ok := registeredInitializers[os.Args[0]]; ok { + initializer() + return true + } + return false +} diff --git a/internal/reexec/self_linux.go b/internal/reexec/self_linux.go new file mode 100644 index 0000000000..956d09326a --- /dev/null +++ b/internal/reexec/self_linux.go @@ -0,0 +1,14 @@ +// This file originates from Docker/Moby, +// https://github.com/moby/moby/blob/master/pkg/reexec/ +// Licensed under Apache License 2.0: https://github.com/moby/moby/blob/master/LICENSE +// Copyright 2013-2018 Docker, Inc. + +//go:build linux + +package reexec + +// Self returns the path to the current process's binary. +// Returns "/proc/self/exe". +func Self() string { + return "/proc/self/exe" +} diff --git a/internal/reexec/self_others.go b/internal/reexec/self_others.go new file mode 100644 index 0000000000..a9f502ca87 --- /dev/null +++ b/internal/reexec/self_others.go @@ -0,0 +1,32 @@ +// This file originates from Docker/Moby, +// https://github.com/moby/moby/blob/master/pkg/reexec/ +// Licensed under Apache License 2.0: https://github.com/moby/moby/blob/master/LICENSE +// Copyright 2013-2018 Docker, Inc. + +//go:build !linux + +package reexec + +import ( + "os" + "os/exec" + "path/filepath" +) + +// Self returns the path to the current process's binary. +// Uses os.Args[0]. +func Self() string { + name := os.Args[0] + if filepath.Base(name) == name { + if lp, err := exec.LookPath(name); err == nil { + return lp + } + } + // handle conversion of relative paths to absolute + if absName, err := filepath.Abs(name); err == nil { + return absName + } + // if we couldn't get absolute name, return original + // (NOTE: Go only errors on Abs() if os.Getwd fails) + return name +} diff --git a/p2p/simulations/adapters/exec.go b/p2p/simulations/adapters/exec.go index 1d812514de..5ac3379393 100644 --- a/p2p/simulations/adapters/exec.go +++ b/p2p/simulations/adapters/exec.go @@ -34,7 +34,7 @@ import ( "syscall" "time" - "github.com/docker/docker/pkg/reexec" + "github.com/ethereum/go-ethereum/internal/reexec" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" diff --git a/p2p/simulations/adapters/types.go b/p2p/simulations/adapters/types.go index 3b4e05a901..098759599c 100644 --- a/p2p/simulations/adapters/types.go +++ b/p2p/simulations/adapters/types.go @@ -25,8 +25,8 @@ import ( "os" "strconv" - "github.com/docker/docker/pkg/reexec" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/internal/reexec" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" From ea2e66a58e48ef63566d5274c4a875e817a1cd39 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Tue, 31 Oct 2023 17:39:55 +0800 Subject: [PATCH 09/11] trie/triedb/pathdb: improve dirty node flushing trigger (#28426) * trie/triedb/pathdb: improve dirty node flushing trigger * trie/triedb/pathdb: add tests * trie/triedb/pathdb: address comment --- trie/triedb/pathdb/database_test.go | 51 ++++++++++++++++++++----- trie/triedb/pathdb/disklayer.go | 58 +++++++++++++++++++++-------- trie/triedb/pathdb/history.go | 26 ++++--------- 3 files changed, 93 insertions(+), 42 deletions(-) diff --git a/trie/triedb/pathdb/database_test.go b/trie/triedb/pathdb/database_test.go index 912364f7f4..5509682c39 100644 --- a/trie/triedb/pathdb/database_test.go +++ b/trie/triedb/pathdb/database_test.go @@ -96,11 +96,15 @@ type tester struct { snapStorages map[common.Hash]map[common.Hash]map[common.Hash][]byte } -func newTester(t *testing.T) *tester { +func newTester(t *testing.T, historyLimit uint64) *tester { var ( disk, _ = rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false) - db = New(disk, &Config{CleanCacheSize: 256 * 1024, DirtyCacheSize: 256 * 1024}) - obj = &tester{ + db = New(disk, &Config{ + StateHistory: historyLimit, + CleanCacheSize: 256 * 1024, + DirtyCacheSize: 256 * 1024, + }) + obj = &tester{ db: db, preimages: make(map[common.Hash]common.Address), accounts: make(map[common.Hash][]byte), @@ -376,7 +380,7 @@ func (t *tester) bottomIndex() int { func TestDatabaseRollback(t *testing.T) { // Verify state histories - tester := newTester(t) + tester := newTester(t, 0) defer tester.release() if err := tester.verifyHistory(); err != nil { @@ -402,7 +406,7 @@ func TestDatabaseRollback(t *testing.T) { func TestDatabaseRecoverable(t *testing.T) { var ( - tester = newTester(t) + tester = newTester(t, 0) index = tester.bottomIndex() ) defer tester.release() @@ -440,7 +444,7 @@ func TestDatabaseRecoverable(t *testing.T) { } func TestDisable(t *testing.T) { - tester := newTester(t) + tester := newTester(t, 0) defer tester.release() _, stored := rawdb.ReadAccountTrieNode(tester.db.diskdb, nil) @@ -476,7 +480,7 @@ func TestDisable(t *testing.T) { } func TestCommit(t *testing.T) { - tester := newTester(t) + tester := newTester(t, 0) defer tester.release() if err := tester.db.Commit(tester.lastHash(), false); err != nil { @@ -500,7 +504,7 @@ func TestCommit(t *testing.T) { } func TestJournal(t *testing.T) { - tester := newTester(t) + tester := newTester(t, 0) defer tester.release() if err := tester.db.Journal(tester.lastHash()); err != nil { @@ -524,7 +528,7 @@ func TestJournal(t *testing.T) { } func TestCorruptedJournal(t *testing.T) { - tester := newTester(t) + tester := newTester(t, 0) defer tester.release() if err := tester.db.Journal(tester.lastHash()); err != nil { @@ -553,6 +557,35 @@ func TestCorruptedJournal(t *testing.T) { } } +// TestTailTruncateHistory function is designed to test a specific edge case where, +// when history objects are removed from the end, it should trigger a state flush +// if the ID of the new tail object is even higher than the persisted state ID. +// +// For example, let's say the ID of the persistent state is 10, and the current +// history objects range from ID(5) to ID(15). As we accumulate six more objects, +// the history will expand to cover ID(11) to ID(21). ID(11) then becomes the +// oldest history object, and its ID is even higher than the stored state. +// +// In this scenario, it is mandatory to update the persistent state before +// truncating the tail histories. This ensures that the ID of the persistent state +// always falls within the range of [oldest-history-id, latest-history-id]. +func TestTailTruncateHistory(t *testing.T) { + tester := newTester(t, 10) + defer tester.release() + + tester.db.Close() + tester.db = New(tester.db.diskdb, &Config{StateHistory: 10}) + + head, err := tester.db.freezer.Ancients() + if err != nil { + t.Fatalf("Failed to obtain freezer head") + } + stored := rawdb.ReadPersistentStateID(tester.db.diskdb) + if head != stored { + t.Fatalf("Failed to truncate excess history object above, stored: %d, head: %d", stored, head) + } +} + // copyAccounts returns a deep-copied account set of the provided one. func copyAccounts(set map[common.Hash][]byte) map[common.Hash][]byte { copied := make(map[common.Hash][]byte, len(set)) diff --git a/trie/triedb/pathdb/disklayer.go b/trie/triedb/pathdb/disklayer.go index d3b6419cc5..ef697cbce8 100644 --- a/trie/triedb/pathdb/disklayer.go +++ b/trie/triedb/pathdb/disklayer.go @@ -172,37 +172,65 @@ func (dl *diskLayer) commit(bottom *diffLayer, force bool) (*diskLayer, error) { dl.lock.Lock() defer dl.lock.Unlock() - // Construct and store the state history first. If crash happens - // after storing the state history but without flushing the - // corresponding states(journal), the stored state history will - // be truncated in the next restart. + // Construct and store the state history first. If crash happens after storing + // the state history but without flushing the corresponding states(journal), + // the stored state history will be truncated from head in the next restart. + var ( + overflow bool + oldest uint64 + ) if dl.db.freezer != nil { - err := writeHistory(dl.db.diskdb, dl.db.freezer, bottom, dl.db.config.StateHistory) + err := writeHistory(dl.db.freezer, bottom) if err != nil { return nil, err } + // Determine if the persisted history object has exceeded the configured + // limitation, set the overflow as true if so. + tail, err := dl.db.freezer.Tail() + if err != nil { + return nil, err + } + limit := dl.db.config.StateHistory + if limit != 0 && bottom.stateID()-tail > limit { + overflow = true + oldest = bottom.stateID() - limit + 1 // track the id of history **after truncation** + } } // Mark the diskLayer as stale before applying any mutations on top. dl.stale = true - // Store the root->id lookup afterwards. All stored lookups are - // identified by the **unique** state root. It's impossible that - // in the same chain blocks are not adjacent but have the same - // root. + // Store the root->id lookup afterwards. All stored lookups are identified + // by the **unique** state root. It's impossible that in the same chain + // blocks are not adjacent but have the same root. if dl.id == 0 { rawdb.WriteStateID(dl.db.diskdb, dl.root, 0) } rawdb.WriteStateID(dl.db.diskdb, bottom.rootHash(), bottom.stateID()) - // Construct a new disk layer by merging the nodes from the provided - // diff layer, and flush the content in disk layer if there are too - // many nodes cached. The clean cache is inherited from the original - // disk layer for reusing. + // Construct a new disk layer by merging the nodes from the provided diff + // layer, and flush the content in disk layer if there are too many nodes + // cached. The clean cache is inherited from the original disk layer. ndl := newDiskLayer(bottom.root, bottom.stateID(), dl.db, dl.cleans, dl.buffer.commit(bottom.nodes)) - err := ndl.buffer.flush(ndl.db.diskdb, ndl.cleans, ndl.id, force) - if err != nil { + + // In a unique scenario where the ID of the oldest history object (after tail + // truncation) surpasses the persisted state ID, we take the necessary action + // of forcibly committing the cached dirty nodes to ensure that the persisted + // state ID remains higher. + if !force && rawdb.ReadPersistentStateID(dl.db.diskdb) < oldest { + force = true + } + if err := ndl.buffer.flush(ndl.db.diskdb, ndl.cleans, ndl.id, force); err != nil { return nil, err } + // To remove outdated history objects from the end, we set the 'tail' parameter + // to 'oldest-1' due to the offset between the freezer index and the history ID. + if overflow { + pruned, err := truncateFromTail(ndl.db.diskdb, ndl.db.freezer, oldest-1) + if err != nil { + return nil, err + } + log.Debug("Pruned state history", "items", pruned, "tailid", oldest) + } return ndl, nil } diff --git a/trie/triedb/pathdb/history.go b/trie/triedb/pathdb/history.go index 6f33b61859..6e3f3faaed 100644 --- a/trie/triedb/pathdb/history.go +++ b/trie/triedb/pathdb/history.go @@ -512,38 +512,28 @@ func readHistory(freezer *rawdb.ResettableFreezer, id uint64) (*history, error) return &dec, nil } -// writeHistory writes the state history with provided state set. After -// storing the corresponding state history, it will also prune the stale -// histories from the disk with the given threshold. -func writeHistory(db ethdb.KeyValueStore, freezer *rawdb.ResettableFreezer, dl *diffLayer, limit uint64) error { +// writeHistory persists the state history with the provided state set. +func writeHistory(freezer *rawdb.ResettableFreezer, dl *diffLayer) error { // Short circuit if state set is not available. if dl.states == nil { return errors.New("state change set is not available") } var ( - err error - n int - start = time.Now() - h = newHistory(dl.rootHash(), dl.parentLayer().rootHash(), dl.block, dl.states) + start = time.Now() + history = newHistory(dl.rootHash(), dl.parentLayer().rootHash(), dl.block, dl.states) ) - accountData, storageData, accountIndex, storageIndex := h.encode() + accountData, storageData, accountIndex, storageIndex := history.encode() dataSize := common.StorageSize(len(accountData) + len(storageData)) indexSize := common.StorageSize(len(accountIndex) + len(storageIndex)) // Write history data into five freezer table respectively. - rawdb.WriteStateHistory(freezer, dl.stateID(), h.meta.encode(), accountIndex, storageIndex, accountData, storageData) + rawdb.WriteStateHistory(freezer, dl.stateID(), history.meta.encode(), accountIndex, storageIndex, accountData, storageData) - // Prune stale state histories based on the config. - if limit != 0 && dl.stateID() > limit { - n, err = truncateFromTail(db, freezer, dl.stateID()-limit) - if err != nil { - return err - } - } historyDataBytesMeter.Mark(int64(dataSize)) historyIndexBytesMeter.Mark(int64(indexSize)) historyBuildTimeMeter.UpdateSince(start) - log.Debug("Stored state history", "id", dl.stateID(), "block", dl.block, "data", dataSize, "index", indexSize, "pruned", n, "elapsed", common.PrettyDuration(time.Since(start))) + log.Debug("Stored state history", "id", dl.stateID(), "block", dl.block, "data", dataSize, "index", indexSize, "elapsed", common.PrettyDuration(time.Since(start))) + return nil } From 447945e43834fdb24a71885eebc72996ca451f8b Mon Sep 17 00:00:00 2001 From: Jakub Freebit <49676311+jakub-freebit@users.noreply.github.com> Date: Tue, 31 Oct 2023 20:04:45 +0900 Subject: [PATCH 10/11] core/rawdb: add logging and fix comments around AncientRange function. (#28379) This adds warning logs when the read does not match the expected count. We can also remove the size limit since the function documentation explicitly states that callers should limit the count. --- core/rawdb/accessors_chain.go | 19 ++++++++++++------- core/rawdb/freezer_resettable.go | 7 ++++--- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index 97401d283c..d9a89fe90c 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -334,13 +334,18 @@ func ReadHeaderRange(db ethdb.Reader, number uint64, count uint64) []rlp.RawValu return rlpHeaders } // read remaining from ancients - max := count * 700 - data, err := db.AncientRange(ChainFreezerHeaderTable, i+1-count, count, max) - if err == nil && uint64(len(data)) == count { - // the data is on the order [h, h+1, .., n] -- reordering needed - for i := range data { - rlpHeaders = append(rlpHeaders, data[len(data)-1-i]) - } + data, err := db.AncientRange(ChainFreezerHeaderTable, i+1-count, count, 0) + if err != nil { + log.Error("Failed to read headers from freezer", "err", err) + return rlpHeaders + } + if uint64(len(data)) != count { + log.Warn("Incomplete read of headers from freezer", "wanted", count, "read", len(data)) + return rlpHeaders + } + // The data is on the order [h, h+1, .., n] -- reordering needed + for i := range data { + rlpHeaders = append(rlpHeaders, data[len(data)-1-i]) } return rlpHeaders } diff --git a/core/rawdb/freezer_resettable.go b/core/rawdb/freezer_resettable.go index 1df6411a39..7a85489738 100644 --- a/core/rawdb/freezer_resettable.go +++ b/core/rawdb/freezer_resettable.go @@ -119,9 +119,10 @@ func (f *ResettableFreezer) Ancient(kind string, number uint64) ([]byte, error) // AncientRange retrieves multiple items in sequence, starting from the index 'start'. // It will return -// - at most 'max' items, -// - at least 1 item (even if exceeding the maxByteSize), but will otherwise -// return as many items as fit into maxByteSize +// - at most 'count' items, +// - if maxBytes is specified: at least 1 item (even if exceeding the maxByteSize), +// but will otherwise return as many items as fit into maxByteSize. +// - if maxBytes is not specified, 'count' items will be returned if they are present. func (f *ResettableFreezer) AncientRange(kind string, start, count, maxBytes uint64) ([][]byte, error) { f.lock.RLock() defer f.lock.RUnlock() From bc42e88415d36861a173b6f86fe4f0d94a81683b Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 31 Oct 2023 12:39:25 +0100 Subject: [PATCH 11/11] core: add basic chain history support in GenerateChain (#28428) This change improves GenerateChain to support internal chain history access (ChainReader) for the consensus engine and EVM. GenerateChain takes a `parent` block and the number of blocks to create. With my changes, the consensus engine and EVM can now access blocks from `parent` up to the block currently being generated. This is required to make the BLOCKHASH instruction work, and also needed to create real clique chains. Clique uses chain history to figure out if the current signer is in-turn, for example. I've also added some more accessors to BlockGen. These are helpful when creating transactions: - g.Signer returns a signer instance for the current block - g.Difficulty returns the current block difficulty - g.Gas returns the remaining gas amount Another fix in this commit concerns the receipts returned by GenerateChain. The receipts now have properly derived fields (BlockHash, etc.) and should generally match what would be returned by the RPC API. --- core/bench_test.go | 4 +- core/chain_makers.go | 271 +++++++++++++++++++++++------------ core/chain_makers_test.go | 56 ++++++-- core/state_processor.go | 3 +- core/state_processor_test.go | 3 +- eth/filters/filter_test.go | 108 ++++++++------ 6 files changed, 295 insertions(+), 150 deletions(-) diff --git a/core/bench_test.go b/core/bench_test.go index 55fa980a85..c5991f10e8 100644 --- a/core/bench_test.go +++ b/core/bench_test.go @@ -84,7 +84,7 @@ func genValueTx(nbytes int) func(int, *BlockGen) { toaddr := common.Address{} data := make([]byte, nbytes) gas, _ := IntrinsicGas(data, nil, false, false, false, false) - signer := types.MakeSigner(gen.config, big.NewInt(int64(i)), gen.header.Time) + signer := gen.Signer() gasPrice := big.NewInt(0) if gen.header.BaseFee != nil { gasPrice = gen.header.BaseFee @@ -128,7 +128,7 @@ func genTxRing(naccounts int) func(int, *BlockGen) { if gen.header.BaseFee != nil { gasPrice = gen.header.BaseFee } - signer := types.MakeSigner(gen.config, big.NewInt(int64(i)), gen.header.Time) + signer := gen.Signer() for { gas -= params.TxGas if gas < params.TxGas { diff --git a/core/chain_makers.go b/core/chain_makers.go index 0f445ab3f0..31c111b73e 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -38,8 +38,8 @@ import ( // See GenerateChain for a detailed explanation. type BlockGen struct { i int + cm *chainMaker parent *types.Block - chain []*types.Block header *types.Header statedb *state.StateDB @@ -49,7 +49,6 @@ type BlockGen struct { uncles []*types.Header withdrawals []*types.Withdrawal - config *params.ChainConfig engine consensus.Engine } @@ -88,13 +87,18 @@ func (b *BlockGen) SetPoS() { b.header.Difficulty = new(big.Int) } +// Difficulty returns the currently calculated difficulty of the block. +func (b *BlockGen) Difficulty() *big.Int { + return new(big.Int).Set(b.header.Difficulty) +} + // SetParentBeaconRoot sets the parent beacon root field of the generated // block. func (b *BlockGen) SetParentBeaconRoot(root common.Hash) { b.header.ParentBeaconRoot = &root var ( - blockContext = NewEVMBlockContext(b.header, nil, &b.header.Coinbase) - vmenv = vm.NewEVM(blockContext, vm.TxContext{}, b.statedb, b.config, vm.Config{}) + blockContext = NewEVMBlockContext(b.header, b.cm, &b.header.Coinbase) + vmenv = vm.NewEVM(blockContext, vm.TxContext{}, b.statedb, b.cm.config, vm.Config{}) ) ProcessBeaconBlockRoot(root, vmenv, b.statedb) } @@ -111,7 +115,7 @@ func (b *BlockGen) addTx(bc *BlockChain, vmConfig vm.Config, tx *types.Transacti b.SetCoinbase(common.Address{}) } b.statedb.SetTxContext(tx.Hash(), len(b.txs)) - receipt, err := ApplyTransaction(b.config, bc, &b.header.Coinbase, b.gasPool, b.statedb, b.header, tx, &b.header.GasUsed, vmConfig) + receipt, err := ApplyTransaction(b.cm.config, bc, &b.header.Coinbase, b.gasPool, b.statedb, b.header, tx, &b.header.GasUsed, vmConfig) if err != nil { panic(err) } @@ -125,11 +129,11 @@ func (b *BlockGen) addTx(bc *BlockChain, vmConfig vm.Config, tx *types.Transacti // AddTx adds a transaction to the generated block. If no coinbase has // been set, the block's coinbase is set to the zero address. // -// AddTx panics if the transaction cannot be executed. In addition to -// the protocol-imposed limitations (gas limit, etc.), there are some -// further limitations on the content of transactions that can be -// added. Notably, contract code relying on the BLOCKHASH instruction -// will panic during execution. +// AddTx panics if the transaction cannot be executed. In addition to the protocol-imposed +// limitations (gas limit, etc.), there are some further limitations on the content of +// transactions that can be added. Notably, contract code relying on the BLOCKHASH +// instruction will panic during execution if it attempts to access a block number outside +// of the range created by GenerateChain. func (b *BlockGen) AddTx(tx *types.Transaction) { b.addTx(nil, vm.Config{}, tx) } @@ -137,11 +141,10 @@ func (b *BlockGen) AddTx(tx *types.Transaction) { // AddTxWithChain adds a transaction to the generated block. If no coinbase has // been set, the block's coinbase is set to the zero address. // -// AddTxWithChain panics if the transaction cannot be executed. In addition to -// the protocol-imposed limitations (gas limit, etc.), there are some -// further limitations on the content of transactions that can be -// added. If contract code relies on the BLOCKHASH instruction, -// the block in chain will be returned. +// AddTxWithChain panics if the transaction cannot be executed. In addition to the +// protocol-imposed limitations (gas limit, etc.), there are some further limitations on +// the content of transactions that can be added. If contract code relies on the BLOCKHASH +// instruction, the block in chain will be returned. func (b *BlockGen) AddTxWithChain(bc *BlockChain, tx *types.Transaction) { b.addTx(bc, vm.Config{}, tx) } @@ -158,8 +161,7 @@ func (b *BlockGen) GetBalance(addr common.Address) *big.Int { return b.statedb.GetBalance(addr) } -// AddUncheckedTx forcefully adds a transaction to the block without any -// validation. +// AddUncheckedTx forcefully adds a transaction to the block without any validation. // // AddUncheckedTx will cause consensus failures when used during real // chain processing. This is best used in conjunction with raw block insertion. @@ -182,6 +184,16 @@ func (b *BlockGen) BaseFee() *big.Int { return new(big.Int).Set(b.header.BaseFee) } +// Gas returns the amount of gas left in the current block. +func (b *BlockGen) Gas() uint64 { + return b.header.GasLimit - b.header.GasUsed +} + +// Signer returns a valid signer instance for the current block. +func (b *BlockGen) Signer() types.Signer { + return types.MakeSigner(b.cm.config, b.header.Number, b.header.Time) +} + // AddUncheckedReceipt forcefully adds a receipts to the block without a // backing transaction. // @@ -207,20 +219,19 @@ func (b *BlockGen) AddUncle(h *types.Header) { var parent *types.Header for i := b.i - 1; i >= 0; i-- { - if b.chain[i].Hash() == h.ParentHash { - parent = b.chain[i].Header() + if b.cm.chain[i].Hash() == h.ParentHash { + parent = b.cm.chain[i].Header() break } } - chainreader := &fakeChainReader{config: b.config} - h.Difficulty = b.engine.CalcDifficulty(chainreader, b.header.Time, parent) + h.Difficulty = b.engine.CalcDifficulty(b.cm, b.header.Time, parent) // The gas limit and price should be derived from the parent h.GasLimit = parent.GasLimit - if b.config.IsLondon(h.Number) { - h.BaseFee = eip1559.CalcBaseFee(b.config, parent) - if !b.config.IsLondon(parent.Number) { - parentGasLimit := parent.GasLimit * b.config.ElasticityMultiplier() + if b.cm.config.IsLondon(h.Number) { + h.BaseFee = eip1559.CalcBaseFee(b.cm.config, parent) + if !b.cm.config.IsLondon(parent.Number) { + parentGasLimit := parent.GasLimit * b.cm.config.ElasticityMultiplier() h.GasLimit = CalcGasLimit(parentGasLimit, parentGasLimit) } } @@ -242,12 +253,12 @@ func (b *BlockGen) nextWithdrawalIndex() uint64 { return b.withdrawals[len(b.withdrawals)-1].Index + 1 } for i := b.i - 1; i >= 0; i-- { - if wd := b.chain[i].Withdrawals(); len(wd) != 0 { + if wd := b.cm.chain[i].Withdrawals(); len(wd) != 0 { return wd[len(wd)-1].Index + 1 } if i == 0 { // Correctly set the index if no parent had withdrawals. - if wd := b.parent.Withdrawals(); len(wd) != 0 { + if wd := b.cm.bottom.Withdrawals(); len(wd) != 0 { return wd[len(wd)-1].Index + 1 } } @@ -263,9 +274,9 @@ func (b *BlockGen) PrevBlock(index int) *types.Block { panic(fmt.Errorf("block index %d out of range (%d,%d)", index, -1, b.i)) } if index == -1 { - return b.parent + return b.cm.bottom } - return b.chain[index] + return b.cm.chain[index] } // OffsetTime modifies the time instance of a block, implicitly changing its @@ -273,11 +284,10 @@ func (b *BlockGen) PrevBlock(index int) *types.Block { // tied to chain length directly. func (b *BlockGen) OffsetTime(seconds int64) { b.header.Time += uint64(seconds) - if b.header.Time <= b.parent.Header().Time { + if b.header.Time <= b.cm.bottom.Header().Time { panic("block time out of range") } - chainreader := &fakeChainReader{config: b.config} - b.header.Difficulty = b.engine.CalcDifficulty(chainreader, b.header.Time, b.parent.Header()) + b.header.Difficulty = b.engine.CalcDifficulty(b.cm, b.header.Time, b.parent.Header()) } // GenerateChain creates a chain of n blocks. The first block's @@ -296,11 +306,14 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse if config == nil { config = params.TestChainConfig } - blocks, receipts := make(types.Blocks, n), make([]types.Receipts, n) - chainreader := &fakeChainReader{config: config} + if engine == nil { + panic("nil consensus engine") + } + cm := newChainMaker(parent, config, engine) + genblock := func(i int, parent *types.Block, triedb *trie.Database, statedb *state.StateDB) (*types.Block, types.Receipts) { - b := &BlockGen{i: i, chain: blocks, parent: parent, statedb: statedb, config: config, engine: engine} - b.header = makeHeader(chainreader, parent, statedb, b.engine) + b := &BlockGen{i: i, cm: cm, parent: parent, statedb: statedb, engine: engine} + b.header = cm.makeHeader(parent, statedb, b.engine) // Set the difficulty for clique block. The chain maker doesn't have access // to a chain, so the difficulty will be left unset (nil). Set it here to the @@ -330,24 +343,23 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse if gen != nil { gen(i, b) } - if b.engine != nil { - block, err := b.engine.FinalizeAndAssemble(chainreader, b.header, statedb, b.txs, b.uncles, b.receipts, b.withdrawals) - if err != nil { - panic(err) - } - // Write state changes to db - root, err := statedb.Commit(b.header.Number.Uint64(), config.IsEIP158(b.header.Number)) - if err != nil { - panic(fmt.Sprintf("state write error: %v", err)) - } - if err = triedb.Commit(root, false); err != nil { - panic(fmt.Sprintf("trie write error: %v", err)) - } - return block, b.receipts + block, err := b.engine.FinalizeAndAssemble(cm, b.header, statedb, b.txs, b.uncles, b.receipts, b.withdrawals) + if err != nil { + panic(err) } - return nil, nil + + // Write state changes to db + root, err := statedb.Commit(b.header.Number.Uint64(), config.IsEIP158(b.header.Number)) + if err != nil { + panic(fmt.Sprintf("state write error: %v", err)) + } + if err = triedb.Commit(root, false); err != nil { + panic(fmt.Sprintf("trie write error: %v", err)) + } + return block, b.receipts } + // Forcibly use hash-based state scheme for retaining all nodes in disk. triedb := trie.NewDatabase(db, trie.HashDefaults) defer triedb.Close() @@ -357,12 +369,36 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse if err != nil { panic(err) } - block, receipt := genblock(i, parent, triedb, statedb) - blocks[i] = block - receipts[i] = receipt + block, receipts := genblock(i, parent, triedb, statedb) + + // Post-process the receipts. + // Here we assign the final block hash and other info into the receipt. + // In order for DeriveFields to work, the transaction and receipt lists need to be + // of equal length. If AddUncheckedTx or AddUncheckedReceipt are used, there will be + // extra ones, so we just trim the lists here. + receiptsCount := len(receipts) + txs := block.Transactions() + if len(receipts) > len(txs) { + receipts = receipts[:len(txs)] + } else if len(receipts) < len(txs) { + txs = txs[:len(receipts)] + } + var blobGasPrice *big.Int + if block.ExcessBlobGas() != nil { + blobGasPrice = eip4844.CalcBlobFee(*block.ExcessBlobGas()) + } + if err := receipts.DeriveFields(config, block.Hash(), block.NumberU64(), block.Time(), block.BaseFee(), blobGasPrice, txs); err != nil { + panic(err) + } + + // Re-expand to ensure all receipts are returned. + receipts = receipts[:receiptsCount] + + // Advance the chain. + cm.add(block, receipts) parent = block } - return blocks, receipts + return cm.chain, cm.receipts } // GenerateChainWithGenesis is a wrapper of GenerateChain which will initialize @@ -380,35 +416,26 @@ func GenerateChainWithGenesis(genesis *Genesis, engine consensus.Engine, n int, return db, blocks, receipts } -func makeHeader(chain consensus.ChainReader, parent *types.Block, state *state.StateDB, engine consensus.Engine) *types.Header { - var time uint64 - if parent.Time() == 0 { - time = 10 - } else { - time = parent.Time() + 10 // block time is fixed at 10 seconds - } +func (cm *chainMaker) makeHeader(parent *types.Block, state *state.StateDB, engine consensus.Engine) *types.Header { + time := parent.Time() + 10 // block time is fixed at 10 seconds header := &types.Header{ - Root: state.IntermediateRoot(chain.Config().IsEIP158(parent.Number())), + Root: state.IntermediateRoot(cm.config.IsEIP158(parent.Number())), ParentHash: parent.Hash(), Coinbase: parent.Coinbase(), - Difficulty: engine.CalcDifficulty(chain, time, &types.Header{ - Number: parent.Number(), - Time: time - 10, - Difficulty: parent.Difficulty(), - UncleHash: parent.UncleHash(), - }), - GasLimit: parent.GasLimit(), - Number: new(big.Int).Add(parent.Number(), common.Big1), - Time: time, + Difficulty: engine.CalcDifficulty(cm, time, parent.Header()), + GasLimit: parent.GasLimit(), + Number: new(big.Int).Add(parent.Number(), common.Big1), + Time: time, } - if chain.Config().IsLondon(header.Number) { - header.BaseFee = eip1559.CalcBaseFee(chain.Config(), parent.Header()) - if !chain.Config().IsLondon(parent.Number()) { - parentGasLimit := parent.GasLimit() * chain.Config().ElasticityMultiplier() + + if cm.config.IsLondon(header.Number) { + header.BaseFee = eip1559.CalcBaseFee(cm.config, parent.Header()) + if !cm.config.IsLondon(parent.Number()) { + parentGasLimit := parent.GasLimit() * cm.config.ElasticityMultiplier() header.GasLimit = CalcGasLimit(parentGasLimit, parentGasLimit) } } - if chain.Config().IsCancun(header.Number, header.Time) { + if cm.config.IsCancun(header.Number, header.Time) { var ( parentExcessBlobGas uint64 parentBlobGasUsed uint64 @@ -461,18 +488,86 @@ func makeBlockChainWithGenesis(genesis *Genesis, n int, engine consensus.Engine, return db, blocks } -type fakeChainReader struct { - config *params.ChainConfig +// chainMaker contains the state of chain generation. +type chainMaker struct { + bottom *types.Block + engine consensus.Engine + config *params.ChainConfig + chain []*types.Block + chainByHash map[common.Hash]*types.Block + receipts []types.Receipts } -// Config returns the chain configuration. -func (cr *fakeChainReader) Config() *params.ChainConfig { - return cr.config +func newChainMaker(bottom *types.Block, config *params.ChainConfig, engine consensus.Engine) *chainMaker { + return &chainMaker{ + bottom: bottom, + config: config, + engine: engine, + chainByHash: make(map[common.Hash]*types.Block), + } +} + +func (cm *chainMaker) add(b *types.Block, r []*types.Receipt) { + cm.chain = append(cm.chain, b) + cm.chainByHash[b.Hash()] = b + cm.receipts = append(cm.receipts, r) +} + +func (cm *chainMaker) blockByNumber(number uint64) *types.Block { + if number == cm.bottom.NumberU64() { + return cm.bottom + } + cur := cm.CurrentHeader().Number.Uint64() + lowest := cm.bottom.NumberU64() + 1 + if number < lowest || number > cur { + return nil + } + return cm.chain[number-lowest] +} + +// ChainReader/ChainContext implementation + +// Config returns the chain configuration (for consensus.ChainReader). +func (cm *chainMaker) Config() *params.ChainConfig { + return cm.config +} + +// Engine returns the consensus engine (for ChainContext). +func (cm *chainMaker) Engine() consensus.Engine { + return cm.engine +} + +func (cm *chainMaker) CurrentHeader() *types.Header { + if len(cm.chain) == 0 { + return cm.bottom.Header() + } + return cm.chain[len(cm.chain)-1].Header() } -func (cr *fakeChainReader) CurrentHeader() *types.Header { return nil } -func (cr *fakeChainReader) GetHeaderByNumber(number uint64) *types.Header { return nil } -func (cr *fakeChainReader) GetHeaderByHash(hash common.Hash) *types.Header { return nil } -func (cr *fakeChainReader) GetHeader(hash common.Hash, number uint64) *types.Header { return nil } -func (cr *fakeChainReader) GetBlock(hash common.Hash, number uint64) *types.Block { return nil } -func (cr *fakeChainReader) GetTd(hash common.Hash, number uint64) *big.Int { return nil } +func (cm *chainMaker) GetHeaderByNumber(number uint64) *types.Header { + b := cm.blockByNumber(number) + if b == nil { + return nil + } + return b.Header() +} + +func (cm *chainMaker) GetHeaderByHash(hash common.Hash) *types.Header { + b := cm.chainByHash[hash] + if b == nil { + return nil + } + return b.Header() +} + +func (cm *chainMaker) GetHeader(hash common.Hash, number uint64) *types.Header { + return cm.GetHeaderByNumber(number) +} + +func (cm *chainMaker) GetBlock(hash common.Hash, number uint64) *types.Block { + return cm.blockByNumber(number) +} + +func (cm *chainMaker) GetTd(hash common.Hash, number uint64) *big.Int { + return nil // not supported +} diff --git a/core/chain_makers_test.go b/core/chain_makers_test.go index 018151cb0c..84148841f5 100644 --- a/core/chain_makers_test.go +++ b/core/chain_makers_test.go @@ -19,8 +19,10 @@ package core import ( "fmt" "math/big" + "reflect" "testing" + "github.com/davecgh/go-spew/spew" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/beacon" "github.com/ethereum/go-ethereum/consensus/ethash" @@ -52,9 +54,8 @@ func TestGeneratePOSChain(t *testing.T) { Difficulty: common.Big1, GasLimit: 5_000_000, } - gendb = rawdb.NewMemoryDatabase() - signer = types.LatestSigner(gspec.Config) - db = rawdb.NewMemoryDatabase() + gendb = rawdb.NewMemoryDatabase() + db = rawdb.NewMemoryDatabase() ) config.TerminalTotalDifficultyPassed = true @@ -82,10 +83,20 @@ func TestGeneratePOSChain(t *testing.T) { } genesis := gspec.MustCommit(gendb, trie.NewDatabase(gendb, trie.HashDefaults)) - chain, _ := GenerateChain(gspec.Config, genesis, beacon.NewFaker(), gendb, 4, func(i int, gen *BlockGen) { + genchain, genreceipts := GenerateChain(gspec.Config, genesis, beacon.NewFaker(), gendb, 4, func(i int, gen *BlockGen) { gen.SetParentBeaconRoot(common.Hash{byte(i + 1)}) - tx, _ := types.SignTx(types.NewTransaction(gen.TxNonce(address), address, big.NewInt(1000), params.TxGas, new(big.Int).Add(gen.BaseFee(), common.Big1), nil), signer, key) + + // Add value transfer tx. + tx := types.MustSignNewTx(key, gen.Signer(), &types.LegacyTx{ + Nonce: gen.TxNonce(address), + To: &address, + Value: big.NewInt(1000), + Gas: params.TxGas, + GasPrice: new(big.Int).Add(gen.BaseFee(), common.Big1), + }) gen.AddTx(tx) + + // Add withdrawals. if i == 1 { gen.AddWithdrawal(&types.Withdrawal{ Validator: 42, @@ -116,20 +127,39 @@ func TestGeneratePOSChain(t *testing.T) { blockchain, _ := NewBlockChain(db, nil, gspec, nil, beacon.NewFaker(), vm.Config{}, nil, nil) defer blockchain.Stop() - if i, err := blockchain.InsertChain(chain); err != nil { - fmt.Printf("insert error (block %d): %v\n", chain[i].NumberU64(), err) - return + if i, err := blockchain.InsertChain(genchain); err != nil { + t.Fatalf("insert error (block %d): %v\n", genchain[i].NumberU64(), err) } // enforce that withdrawal indexes are monotonically increasing from 0 var ( withdrawalIndex uint64 - head = blockchain.CurrentBlock().Number.Uint64() ) - for i := 0; i < int(head); i++ { - block := blockchain.GetBlockByNumber(uint64(i)) + for i := range genchain { + blocknum := genchain[i].NumberU64() + block := blockchain.GetBlockByNumber(blocknum) if block == nil { - t.Fatalf("block %d not found", i) + t.Fatalf("block %d not found", blocknum) + } + + // Verify receipts. + genBlockReceipts := genreceipts[i] + for _, r := range genBlockReceipts { + if r.BlockNumber.Cmp(block.Number()) != 0 { + t.Errorf("receipt has wrong block number %d, want %d", r.BlockNumber, block.Number()) + } + if r.BlockHash != block.Hash() { + t.Errorf("receipt has wrong block hash %v, want %v", r.BlockHash, block.Hash()) + } + + // patch up empty logs list to make DeepEqual below work + if r.Logs == nil { + r.Logs = []*types.Log{} + } + } + blockchainReceipts := blockchain.GetReceiptsByHash(block.Hash()) + if !reflect.DeepEqual(genBlockReceipts, blockchainReceipts) { + t.Fatalf("receipts mismatch\ngenerated: %s\nblockchain: %s", spew.Sdump(genBlockReceipts), spew.Sdump(blockchainReceipts)) } // Verify withdrawals. @@ -144,7 +174,7 @@ func TestGeneratePOSChain(t *testing.T) { } // Verify parent beacon root. - want := common.Hash{byte(i)} + want := common.Hash{byte(blocknum)} if got := block.BeaconRoot(); *got != want { t.Fatalf("block %d, wrong parent beacon root: got %s, want %s", i, got, want) } diff --git a/core/state_processor.go b/core/state_processor.go index 7dd81487d5..9a4333f723 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -165,7 +165,8 @@ func ApplyTransaction(config *params.ChainConfig, bc ChainContext, author *commo } // Create a new context to be used in the EVM environment blockContext := NewEVMBlockContext(header, bc, author) - vmenv := vm.NewEVM(blockContext, vm.TxContext{BlobHashes: tx.BlobHashes()}, statedb, config, cfg) + txContext := NewEVMTxContext(msg) + vmenv := vm.NewEVM(blockContext, txContext, statedb, config, cfg) return applyTransaction(msg, config, gp, statedb, header.Number, header.Hash(), tx, usedGas, vmenv) } diff --git a/core/state_processor_test.go b/core/state_processor_test.go index 6c6ef5adfd..e80bc288c3 100644 --- a/core/state_processor_test.go +++ b/core/state_processor_test.go @@ -359,7 +359,8 @@ func TestStateProcessorErrors(t *testing.T) { func GenerateBadBlock(parent *types.Block, engine consensus.Engine, txs types.Transactions, config *params.ChainConfig) *types.Block { difficulty := big.NewInt(0) if !config.TerminalTotalDifficultyPassed { - difficulty = engine.CalcDifficulty(&fakeChainReader{config}, parent.Time()+10, &types.Header{ + fakeChainReader := newChainMaker(nil, config, engine) + difficulty = engine.CalcDifficulty(fakeChainReader, parent.Time()+10, &types.Header{ Number: parent.Number(), Time: parent.Time(), Difficulty: parent.Difficulty(), diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index a076fb7377..4e09a9038b 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -123,35 +123,35 @@ func TestFilters(t *testing.T) { pragma solidity >=0.7.0 <0.9.0; contract Logger { - function log0() external { - assembly { - log0(0, 0) - } - } + function log0() external { + assembly { + log0(0, 0) + } + } - function log1(uint t1) external { - assembly { - log1(0, 0, t1) - } - } + function log1(uint t1) external { + assembly { + log1(0, 0, t1) + } + } - function log2(uint t1, uint t2) external { - assembly { - log2(0, 0, t1, t2) - } - } + function log2(uint t1, uint t2) external { + assembly { + log2(0, 0, t1, t2) + } + } - function log3(uint t1, uint t2, uint t3) external { - assembly { - log3(0, 0, t1, t2, t3) - } - } + function log3(uint t1, uint t2, uint t3) external { + assembly { + log3(0, 0, t1, t2, t3) + } + } - function log4(uint t1, uint t2, uint t3, uint t4) external { - assembly { - log4(0, 0, t1, t2, t3, t4) - } - } + function log4(uint t1, uint t2, uint t3, uint t4) external { + assembly { + log4(0, 0, t1, t2, t3, t4) + } + } } */ bytecode = common.FromHex("608060405234801561001057600080fd5b50600436106100575760003560e01c80630aa731851461005c5780632a4c08961461006657806378b9a1f314610082578063c670f8641461009e578063c683d6a3146100ba575b600080fd5b6100646100d6565b005b610080600480360381019061007b9190610143565b6100dc565b005b61009c60048036038101906100979190610196565b6100e8565b005b6100b860048036038101906100b391906101d6565b6100f2565b005b6100d460048036038101906100cf9190610203565b6100fa565b005b600080a0565b808284600080a3505050565b8082600080a25050565b80600080a150565b80828486600080a450505050565b600080fd5b6000819050919050565b6101208161010d565b811461012b57600080fd5b50565b60008135905061013d81610117565b92915050565b60008060006060848603121561015c5761015b610108565b5b600061016a8682870161012e565b935050602061017b8682870161012e565b925050604061018c8682870161012e565b9150509250925092565b600080604083850312156101ad576101ac610108565b5b60006101bb8582860161012e565b92505060206101cc8582860161012e565b9150509250929050565b6000602082840312156101ec576101eb610108565b5b60006101fa8482850161012e565b91505092915050565b6000806000806080858703121561021d5761021c610108565b5b600061022b8782880161012e565b945050602061023c8782880161012e565b935050604061024d8782880161012e565b925050606061025e8782880161012e565b9150509295919450925056fea264697066735822122073a4b156f487e59970dc1ef449cc0d51467268f676033a17188edafcee861f9864736f6c63430008110033") @@ -287,53 +287,71 @@ func TestFilters(t *testing.T) { { f: sys.NewBlockFilter(chain[2].Hash(), []common.Address{contract}, nil), want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696332","0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x3","transactionHash":"0xdefe471992a07a02acdfbe33edaae22fbb86d7d3cec3f1b8e4e77702fb3acc1d","transactionIndex":"0x0","blockHash":"0x7a7556792ca7d37882882e2b001fe14833eaf81c2c7f865c9c771ec37a024f6b","logIndex":"0x0","removed":false}]`, - }, { + }, + { f: sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), []common.Address{contract}, [][]common.Hash{{hash1, hash2, hash3, hash4}}), want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x2","transactionHash":"0xa8028c655b6423204c8edfbc339f57b042d6bec2b6a61145d76b7c08b4cccd42","transactionIndex":"0x0","blockHash":"0x24417bb49ce44cfad65da68f33b510bf2a129c0d89ccf06acb6958b8585ccf34","logIndex":"0x0","removed":false},{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696332","0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x3","transactionHash":"0xdefe471992a07a02acdfbe33edaae22fbb86d7d3cec3f1b8e4e77702fb3acc1d","transactionIndex":"0x0","blockHash":"0x7a7556792ca7d37882882e2b001fe14833eaf81c2c7f865c9c771ec37a024f6b","logIndex":"0x0","removed":false},{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696334"],"data":"0x","blockNumber":"0x3e8","transactionHash":"0x9a87842100a638dfa5da8842b4beda691d2fd77b0c84b57f24ecfa9fb208f747","transactionIndex":"0x0","blockHash":"0xb360bad5265261c075ece02d3bf0e39498a6a76310482cdfd90588748e6c5ee0","logIndex":"0x0","removed":false}]`, - }, { + }, + { f: sys.NewRangeFilter(900, 999, []common.Address{contract}, [][]common.Hash{{hash3}}), - }, { + }, + { f: sys.NewRangeFilter(990, int64(rpc.LatestBlockNumber), []common.Address{contract2}, [][]common.Hash{{hash3}}), want: `[{"address":"0xff00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696333"],"data":"0x","blockNumber":"0x3e7","transactionHash":"0x53e3675800c6908424b61b35a44e51ca4c73ca603e58a65b32c67968b4f42200","transactionIndex":"0x0","blockHash":"0x2e4620a2b426b0612ec6cad9603f466723edaed87f98c9137405dd4f7a2409ff","logIndex":"0x0","removed":false}]`, - }, { + }, + { f: sys.NewRangeFilter(1, 10, []common.Address{contract}, [][]common.Hash{{hash2}, {hash1}}), want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696332","0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x3","transactionHash":"0xdefe471992a07a02acdfbe33edaae22fbb86d7d3cec3f1b8e4e77702fb3acc1d","transactionIndex":"0x0","blockHash":"0x7a7556792ca7d37882882e2b001fe14833eaf81c2c7f865c9c771ec37a024f6b","logIndex":"0x0","removed":false}]`, - }, { + }, + { f: sys.NewRangeFilter(1, 10, nil, [][]common.Hash{{hash1, hash2}}), want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x2","transactionHash":"0xa8028c655b6423204c8edfbc339f57b042d6bec2b6a61145d76b7c08b4cccd42","transactionIndex":"0x0","blockHash":"0x24417bb49ce44cfad65da68f33b510bf2a129c0d89ccf06acb6958b8585ccf34","logIndex":"0x0","removed":false},{"address":"0xff00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x2","transactionHash":"0xdba3e2ea9a7d690b722d70ee605fd67ba4c00d1d3aecd5cf187a7b92ad8eb3df","transactionIndex":"0x1","blockHash":"0x24417bb49ce44cfad65da68f33b510bf2a129c0d89ccf06acb6958b8585ccf34","logIndex":"0x1","removed":false},{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696332","0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x3","transactionHash":"0xdefe471992a07a02acdfbe33edaae22fbb86d7d3cec3f1b8e4e77702fb3acc1d","transactionIndex":"0x0","blockHash":"0x7a7556792ca7d37882882e2b001fe14833eaf81c2c7f865c9c771ec37a024f6b","logIndex":"0x0","removed":false}]`, - }, { + }, + { f: sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), nil, [][]common.Hash{{common.BytesToHash([]byte("fail"))}}), - }, { + }, + { f: sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), []common.Address{common.BytesToAddress([]byte("failmenow"))}, nil), - }, { + }, + { f: sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), nil, [][]common.Hash{{common.BytesToHash([]byte("fail"))}, {hash1}}), - }, { + }, + { f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.LatestBlockNumber), nil, nil), want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696334"],"data":"0x","blockNumber":"0x3e8","transactionHash":"0x9a87842100a638dfa5da8842b4beda691d2fd77b0c84b57f24ecfa9fb208f747","transactionIndex":"0x0","blockHash":"0xb360bad5265261c075ece02d3bf0e39498a6a76310482cdfd90588748e6c5ee0","logIndex":"0x0","removed":false}]`, - }, { + }, + { f: sys.NewRangeFilter(int64(rpc.FinalizedBlockNumber), int64(rpc.LatestBlockNumber), nil, nil), want: `[{"address":"0xff00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696333"],"data":"0x","blockNumber":"0x3e7","transactionHash":"0x53e3675800c6908424b61b35a44e51ca4c73ca603e58a65b32c67968b4f42200","transactionIndex":"0x0","blockHash":"0x2e4620a2b426b0612ec6cad9603f466723edaed87f98c9137405dd4f7a2409ff","logIndex":"0x0","removed":false},{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696334"],"data":"0x","blockNumber":"0x3e8","transactionHash":"0x9a87842100a638dfa5da8842b4beda691d2fd77b0c84b57f24ecfa9fb208f747","transactionIndex":"0x0","blockHash":"0xb360bad5265261c075ece02d3bf0e39498a6a76310482cdfd90588748e6c5ee0","logIndex":"0x0","removed":false}]`, - }, { + }, + { f: sys.NewRangeFilter(int64(rpc.FinalizedBlockNumber), int64(rpc.FinalizedBlockNumber), nil, nil), want: `[{"address":"0xff00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696333"],"data":"0x","blockNumber":"0x3e7","transactionHash":"0x53e3675800c6908424b61b35a44e51ca4c73ca603e58a65b32c67968b4f42200","transactionIndex":"0x0","blockHash":"0x2e4620a2b426b0612ec6cad9603f466723edaed87f98c9137405dd4f7a2409ff","logIndex":"0x0","removed":false}]`, - }, { + }, + { f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.FinalizedBlockNumber), nil, nil), - }, { + }, + { f: sys.NewRangeFilter(int64(rpc.SafeBlockNumber), int64(rpc.LatestBlockNumber), nil, nil), err: "safe header not found", - }, { + }, + { f: sys.NewRangeFilter(int64(rpc.SafeBlockNumber), int64(rpc.SafeBlockNumber), nil, nil), err: "safe header not found", - }, { + }, + { f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.SafeBlockNumber), nil, nil), err: "safe header not found", - }, { + }, + { f: sys.NewRangeFilter(int64(rpc.PendingBlockNumber), int64(rpc.PendingBlockNumber), nil, nil), - want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696335"],"data":"0x","blockNumber":"0x3e9","transactionHash":"0x4110587c1b8d86edc85dce929a34127f1cb8809515a9f177c91c866de3eb0638","transactionIndex":"0x0","blockHash":"0xc7245899e5817f16fa99cf5ad2d9c1e4b98443a565a673ec9c764640443ef037","logIndex":"0x0","removed":false}]`, - }, { + want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696335"],"data":"0x","blockNumber":"0x3e9","transactionHash":"0x4110587c1b8d86edc85dce929a34127f1cb8809515a9f177c91c866de3eb0638","transactionIndex":"0x0","blockHash":"0xd5e8d4e4eb51a2a2a6ec20ef68a4c2801240743c8deb77a6a1d118ac3eefb725","logIndex":"0x0","removed":false}]`, + }, + { f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.PendingBlockNumber), nil, nil), - want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696334"],"data":"0x","blockNumber":"0x3e8","transactionHash":"0x9a87842100a638dfa5da8842b4beda691d2fd77b0c84b57f24ecfa9fb208f747","transactionIndex":"0x0","blockHash":"0xb360bad5265261c075ece02d3bf0e39498a6a76310482cdfd90588748e6c5ee0","logIndex":"0x0","removed":false},{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696335"],"data":"0x","blockNumber":"0x3e9","transactionHash":"0x4110587c1b8d86edc85dce929a34127f1cb8809515a9f177c91c866de3eb0638","transactionIndex":"0x0","blockHash":"0xc7245899e5817f16fa99cf5ad2d9c1e4b98443a565a673ec9c764640443ef037","logIndex":"0x0","removed":false}]`, - }, { + want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696334"],"data":"0x","blockNumber":"0x3e8","transactionHash":"0x9a87842100a638dfa5da8842b4beda691d2fd77b0c84b57f24ecfa9fb208f747","transactionIndex":"0x0","blockHash":"0xb360bad5265261c075ece02d3bf0e39498a6a76310482cdfd90588748e6c5ee0","logIndex":"0x0","removed":false},{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696335"],"data":"0x","blockNumber":"0x3e9","transactionHash":"0x4110587c1b8d86edc85dce929a34127f1cb8809515a9f177c91c866de3eb0638","transactionIndex":"0x0","blockHash":"0xd5e8d4e4eb51a2a2a6ec20ef68a4c2801240743c8deb77a6a1d118ac3eefb725","logIndex":"0x0","removed":false}]`, + }, + { f: sys.NewRangeFilter(int64(rpc.PendingBlockNumber), int64(rpc.LatestBlockNumber), nil, nil), err: "invalid block range", },