Integration tests 1 (erigontech#1793)

* Initial commit

* Add sentry gRPC interface

* p2psentry directory

* Update README.md

* Update README.md

* Update README.md

* Add go package

* Correct syntax

* add external downloader interface (erigontech#2)

* Add txpool (erigontech#3)

* Add private API (erigontech#4)

* Invert control.proto, add PeerMinBlock, Separare incoming Tx message into a separate stream (erigontech#5)

Co-authored-by: Alexey Sharp <alexeysharp@Alexeys-iMac.local>

* Separate upload messages into its own stream (erigontech#6)

Co-authored-by: Alexey Sharp <alexeysharp@Alexeys-iMac.local>

* Only send changed accounts to listeners (erigontech#7)

* Txpool interface doc (erigontech#9)

* Add architecture diagram source and picture (erigontech#10)

* Typed hashes (erigontech#11)

* Typed hashes

* Fix PeerId

* 64-bit tx nonce

* Add proper golang packages, max_block into p2p sentry Status (erigontech#12)

* Add proper golang packages, max_block into p2p sentry Status

* Change EtherReply to address

Co-authored-by: Alexey Sharp <alexeysharp@Alexeys-iMac.local>

* Add Rust infrastructure (erigontech#13)

* DB stats methods removed by erigontech#1665

* more p2p methods (erigontech#15)

* add mining methods (erigontech#16)

* First draft of Consensus gRPC interface (erigontech#14)

* Update Rust build

* Fix interfaces in architecture diagram (erigontech#17)

* Fix KV interface provider

* Fix Consensus interface provider

* drop java attributes (erigontech#18)

* tx pool remove unused import (erigontech#19)

* ethbackend: add protocol version and client version (erigontech#20)

* Add missing ethbackend I/F (erigontech#21)

* Add interface versioning mechanism (erigontech#23)

Add versioning in KV interface

Co-authored-by: Artem Vorotnikov <artem@vorotnikov.me>

* spec of tx pool method (erigontech#24)

* spec of tx pool method (erigontech#25)

* Update version.proto

* Refactor interface versioning

* Refactor interface versioning

* Testing interface

* Remove tree

* Fix

* Build testing protos

* Fix

* Fix

* Update to the newer interfaces

* Add ProtocolVersion and ClientVersion stubs

* Hook up ProtocolVersion and ClientVersion

* Remove service

* Add compatibility checks for RPC daemon

* Fix typos

* Properly update DB schema version

* Fix test

* Add test for KV compatibility|

* Info messages about compability for RPC daemon

* DB schema version to be one key

* Update release intructions

Co-authored-by: Artem Vorotnikov <artem@vorotnikov.me>
Co-authored-by: b00ris <b00ris@mail.ru>
Co-authored-by: Alexey Sharp <alexeysharp@Alexeys-iMac.local>
Co-authored-by: lightclient <14004106+lightclient@users.noreply.github.com>
Co-authored-by: canepat <16927169+canepat@users.noreply.github.com>
Co-authored-by: Alex Sharov <AskAlexSharov@gmail.com>
Co-authored-by: canepat <tullio.canepa@gmail.com>
Co-authored-by: Alex Sharp <alexsharp@Alexs-MacBook-Pro.local>

Makefile

@@ -161,12 +161,15 @@ grpc:
 
 	$(GOBUILD) -o $(GOBIN)/protoc-gen-go google.golang.org/protobuf/cmd/protoc-gen-go # generates proto messages
 	$(GOBUILD) -o $(GOBIN)/protoc-gen-go-grpc google.golang.org/grpc/cmd/protoc-gen-go-grpc # generates grpc services
+	PATH=$(GOBIN):$(PATH) protoc --proto_path=interfaces --go_out=gointerfaces -I=build/include/google \
+		types/types.proto
 	PATH=$(GOBIN):$(PATH) protoc --proto_path=interfaces --go_out=gointerfaces --go-grpc_out=gointerfaces -I=build/include/google \
 		--go_opt=Mtypes/types.proto=github.com/ledgerwatch/turbo-geth/gointerfaces/types \
-		types/types.proto \
+		--go-grpc_opt=Mtypes/types.proto=github.com/ledgerwatch/turbo-geth/gointerfaces/types \
 		p2psentry/sentry.proto \
 		remote/kv.proto remote/ethbackend.proto \
 		snapshot_downloader/external_downloader.proto \
+		testing/testing.proto
 
 prometheus:
 	docker-compose up prometheus grafana

RELEASE_INSTRUCTIONS.md

@@ -20,3 +20,14 @@ utility before it exits, like this:
 ````
 INFO [08-01|15:36:04.282] Checked                                  blocks=10573804 next time specify --block=10573804 duration=36m54.789025062s
 ````
+
+## Update DB Schema version if required
+
+In the file `common/dbutils/bucket.go` there is variable `DBSchemaVersion` that needs to be update if there are any changes in the database schema, leading to data migrations.
+In most cases, it is enough to bump minor version.
+
+## Update remote KV version if required
+
+In the file `ethdb/remote/remotedbserver/server.go` there is variable `KvServiceAPIVersion` that needs to be update if there are any changes in the remote KV interface, or
+database schema, leading to data migrations.
+In most cases, it is enough to bump minor version. It is best to change both DB schema version and remove KV version together.

cmd/rpcdaemon/cli/config.go

@@ -2,15 +2,18 @@ package cli
 
 import (
 	"context"
+	"encoding/binary"
 	"fmt"
 	"net/http"
 	"path"
 	"time"
 
 	"github.com/ledgerwatch/turbo-geth/cmd/utils"
+	"github.com/ledgerwatch/turbo-geth/common/dbutils"
 	"github.com/ledgerwatch/turbo-geth/common/paths"
 	"github.com/ledgerwatch/turbo-geth/core"
 	"github.com/ledgerwatch/turbo-geth/ethdb"
+	"github.com/ledgerwatch/turbo-geth/ethdb/remote/remotedbserver"
 	"github.com/ledgerwatch/turbo-geth/internal/debug"
 	"github.com/ledgerwatch/turbo-geth/log"
 	"github.com/ledgerwatch/turbo-geth/node"
@@ -113,6 +116,42 @@ func RootCommand() (*cobra.Command, *Flags) {
 	return rootCmd, cfg
 }
 
+func checkDbCompatibility(db ethdb.RwKV) error {
+	// DB schema version compatibility check
+	var version []byte
+	var compatErr error
+	var compatTx ethdb.Tx
+	if compatTx, compatErr = db.BeginRo(context.Background()); compatErr != nil {
+		return fmt.Errorf("open Ro Tx for DB schema compability check: %w", compatErr)
+	}
+	defer compatTx.Rollback()
+	if version, compatErr = compatTx.GetOne(dbutils.DatabaseInfoBucket, dbutils.DBSchemaVersionKey); compatErr != nil {
+		return fmt.Errorf("read version for DB schema compability check: %w", compatErr)
+	}
+	if len(version) != 12 {
+		return fmt.Errorf("database does not have major schema version. upgrade and restart turbo-geth core")
+	}
+	major := binary.BigEndian.Uint32(version[:])
+	minor := binary.BigEndian.Uint32(version[4:])
+	patch := binary.BigEndian.Uint32(version[8:])
+	var compatible bool
+	if major != dbutils.DBSchemaVersion.Major {
+		compatible = false
+	} else if minor != dbutils.DBSchemaVersion.Minor {
+		compatible = false
+	} else {
+		compatible = true
+	}
+	if !compatible {
+		return fmt.Errorf("incompatible DB Schema versions: reader %d.%d.%d, database %d.%d.%d",
+			dbutils.DBSchemaVersion.Major, dbutils.DBSchemaVersion.Minor, dbutils.DBSchemaVersion.Patch,
+			major, minor, patch)
+	}
+	log.Info("DB schemas compatible", "reader", fmt.Sprintf("%d.%d.%d", dbutils.DBSchemaVersion.Major, dbutils.DBSchemaVersion.Minor, dbutils.DBSchemaVersion.Patch),
+		"database", fmt.Sprintf("%d.%d.%d", major, minor, patch))
+	return nil
+}
+
 func OpenDB(cfg Flags) (ethdb.RoKV, core.ApiBackend, error) {
 	var db ethdb.RwKV
 	var ethBackend core.ApiBackend
@@ -126,6 +165,9 @@ func OpenDB(cfg Flags) (ethdb.RoKV, core.ApiBackend, error) {
 			err = errOpen
 			_ = err
 		}
+		if compatErr := checkDbCompatibility(db); compatErr != nil {
+			return nil, nil, compatErr
+		}
 		if cfg.SnapshotMode != "" {
 			mode, innerErr := snapshotsync.SnapshotModeFromString(cfg.SnapshotMode)
 			if innerErr != nil {
@@ -140,7 +182,10 @@ func OpenDB(cfg Flags) (ethdb.RoKV, core.ApiBackend, error) {
 	}
 	if cfg.PrivateApiAddr != "" {
 		var remoteKv ethdb.RwKV
-		remoteKv, err = ethdb.NewRemote().Path(cfg.PrivateApiAddr).Open(cfg.TLSCertfile, cfg.TLSKeyFile, cfg.TLSCACert)
+		remoteKv, err = ethdb.NewRemote(
+			remotedbserver.KvServiceAPIVersion.Major,
+			remotedbserver.KvServiceAPIVersion.Minor,
+			remotedbserver.KvServiceAPIVersion.Patch).Path(cfg.PrivateApiAddr).Open(cfg.TLSCertfile, cfg.TLSKeyFile, cfg.TLSCACert)
 		if err != nil {
 			return nil, nil, fmt.Errorf("could not connect to remoteKv: %w", err)
 		}

cmd/rpcdaemon/commands/daemon.go

@@ -21,7 +21,7 @@ func APIList(ctx context.Context, kv ethdb.RoKV, eth core.ApiBackend, filters *f
 	netImpl := NewNetAPIImpl(eth)
 	debugImpl := NewPrivateDebugAPI(kv, cfg.Gascap, pending)
 	traceImpl := NewTraceAPI(kv, pending, &cfg)
-	web3Impl := NewWeb3APIImpl()
+	web3Impl := NewWeb3APIImpl(eth)
 	dbImpl := NewDBAPIImpl()   /* deprecated */
 	shhImpl := NewSHHAPIImpl() /* deprecated */
 

cmd/rpcdaemon/commands/eth_system.go

@@ -9,7 +9,6 @@ import (
 	"github.com/ledgerwatch/turbo-geth/core/types"
 	"github.com/ledgerwatch/turbo-geth/eth/ethconfig"
 	"github.com/ledgerwatch/turbo-geth/eth/gasprice"
-	"github.com/ledgerwatch/turbo-geth/eth/protocols/eth"
 	"github.com/ledgerwatch/turbo-geth/eth/stagedsync/stages"
 	"github.com/ledgerwatch/turbo-geth/ethdb"
 	"github.com/ledgerwatch/turbo-geth/log"
@@ -75,8 +74,12 @@ func (api *APIImpl) ChainId(ctx context.Context) (hexutil.Uint64, error) {
 }
 
 // ProtocolVersion implements eth_protocolVersion. Returns the current ethereum protocol version.
-func (api *APIImpl) ProtocolVersion(_ context.Context) (hexutil.Uint, error) {
-	return hexutil.Uint(eth.ProtocolVersions[0]), nil
+func (api *APIImpl) ProtocolVersion(ctx context.Context) (hexutil.Uint, error) {
+	ver, err := api.ethBackend.ProtocolVersion(ctx)
+	if err != nil {
+		return 0, err
+	}
+	return hexutil.Uint(ver), nil
 }
 
 // GasPrice implements eth_gasPrice. Returns the current price per gas in wei.

cmd/rpcdaemon/commands/web3_api.go

@@ -3,10 +3,9 @@ package commands
 import (
 	"context"
 
-	"github.com/ledgerwatch/turbo-geth/common"
 	"github.com/ledgerwatch/turbo-geth/common/hexutil"
+	"github.com/ledgerwatch/turbo-geth/core"
 	"github.com/ledgerwatch/turbo-geth/crypto"
-	"github.com/ledgerwatch/turbo-geth/params"
 )
 
 // Web3API provides interfaces for the web3_ RPC commands
@@ -17,25 +16,23 @@ type Web3API interface {
 
 type Web3APIImpl struct {
 	*BaseAPI
+	ethBackend core.ApiBackend
 }
 
 // NewWeb3APIImpl returns Web3APIImpl instance
-func NewWeb3APIImpl() *Web3APIImpl {
+func NewWeb3APIImpl(ethBackend core.ApiBackend) *Web3APIImpl {
 	return &Web3APIImpl{
-		BaseAPI: &BaseAPI{},
+		BaseAPI:    &BaseAPI{},
+		ethBackend: ethBackend,
 	}
 }
 
 // ClientVersion implements web3_clientVersion. Returns the current client version.
-func (api *Web3APIImpl) ClientVersion(_ context.Context) (string, error) {
-	return common.MakeName("TurboGeth", params.VersionWithCommit(gitCommit, "")), nil
+func (api *Web3APIImpl) ClientVersion(ctx context.Context) (string, error) {
+	return api.ethBackend.ClientVersion(ctx)
 }
 
 // Sha3 implements web3_sha3. Returns Keccak-256 (not the standardized SHA3-256) of the given data.
 func (api *Web3APIImpl) Sha3(_ context.Context, input hexutil.Bytes) hexutil.Bytes {
 	return crypto.Keccak256(input)
 }
-
-var (
-	gitCommit string
-)

cmd/utils/flags.go

@@ -1264,8 +1264,8 @@ func SetDNSDiscoveryDefaults(cfg *eth.Config, genesis common.Hash) {
 }
 
 // RegisterEthService adds an Ethereum client to the stack.
-func RegisterEthService(stack *node.Node, cfg *eth.Config) *eth.Ethereum {
-	backend, err := eth.New(stack, cfg)
+func RegisterEthService(stack *node.Node, cfg *eth.Config, gitCommit string) *eth.Ethereum {
+	backend, err := eth.New(stack, cfg, gitCommit)
 	if err != nil {
 		Fatalf("Failed to register the Ethereum service: %v", err)
 	}

common/dbutils/bucket.go

@@ -4,8 +4,13 @@ import (
 	"bytes"
 	"sort"
 	"strings"
+
+	"github.com/ledgerwatch/turbo-geth/gointerfaces/types"
 )
 
+// DBSchemaVersion
+var DBSchemaVersion = types.VersionReply{Major: 1, Minor: 0, Patch: 0}
+
 // Buckets
 
 // Dictionary:
@@ -226,6 +231,8 @@ var (
 	//StorageModeCallTraces - does not build index of call traces
 	StorageModeCallTraces = []byte("smCallTraces")
 
+	DBSchemaVersionKey = []byte("dbVersion")
+
 	HeadHeaderKey = "LastHeader"
 
 	SnapshotHeadersHeadNumber = "SnapshotLastHeaderNumber"

core/eth_backend.go

@@ -1,20 +1,17 @@
 package core
 
 import (
-	"bytes"
 	"context"
 	"errors"
 	"io"
 
 	"github.com/ledgerwatch/turbo-geth/common"
 	"github.com/ledgerwatch/turbo-geth/common/hexutil"
-	"github.com/ledgerwatch/turbo-geth/consensus/ethash"
 	"github.com/ledgerwatch/turbo-geth/core/types"
 	"github.com/ledgerwatch/turbo-geth/ethdb"
 	"github.com/ledgerwatch/turbo-geth/gointerfaces"
 	"github.com/ledgerwatch/turbo-geth/gointerfaces/remote"
 	"github.com/ledgerwatch/turbo-geth/log"
-	"github.com/ledgerwatch/turbo-geth/rlp"
 	"google.golang.org/grpc/status"
 )
 
@@ -25,6 +22,8 @@ type ApiBackend interface {
 	AddLocal(context.Context, []byte) ([]byte, error)
 	Etherbase(ctx context.Context) (common.Address, error)
 	NetVersion(ctx context.Context) (uint64, error)
+	ProtocolVersion(ctx context.Context) (uint64, error)
+	ClientVersion(ctx context.Context) (string, error)
 	Subscribe(ctx context.Context, cb func(*remote.SubscribeReply)) error
 
 	Mining(ctx context.Context) (bool, error)
@@ -41,51 +40,6 @@ type EthBackend interface {
 	IsMining() bool
 }
 
-type EthBackendImpl struct {
-	eth    EthBackend
-	ethash *ethash.API
-}
-
-func NewEthBackend(eth EthBackend, ethashApi *ethash.API) *EthBackendImpl {
-	return &EthBackendImpl{eth: eth, ethash: ethashApi}
-}
-
-func (back *EthBackendImpl) AddLocal(_ context.Context, signedtx []byte) ([]byte, error) {
-	s := rlp.NewStream(bytes.NewReader(signedtx), 0)
-	tx, err := types.DecodeTransaction(s)
-	if err != nil {
-		return nil, err
-	}
-	return tx.Hash().Bytes(), back.eth.TxPool().AddLocal(tx)
-}
-
-func (back *EthBackendImpl) Etherbase(_ context.Context) (common.Address, error) {
-	return back.eth.Etherbase()
-}
-func (back *EthBackendImpl) NetVersion(_ context.Context) (uint64, error) {
-	return back.eth.NetVersion()
-}
-func (back *EthBackendImpl) Subscribe(_ context.Context, cb func(*remote.SubscribeReply)) error {
-	// do nothing
-	return nil
-}
-
-func (back *EthBackendImpl) GetWork(ctx context.Context) ([4]string, error) {
-	return back.ethash.GetWork()
-}
-func (back *EthBackendImpl) SubmitWork(ctx context.Context, nonce types.BlockNonce, hash, digest common.Hash) (bool, error) {
-	return back.ethash.SubmitWork(nonce, hash, digest), nil
-}
-func (back *EthBackendImpl) SubmitHashRate(ctx context.Context, rate hexutil.Uint64, id common.Hash) (bool, error) {
-	return back.ethash.SubmitHashRate(rate, id), nil
-}
-func (back *EthBackendImpl) GetHashRate(ctx context.Context) (uint64, error) {
-	return back.ethash.GetHashrate(), nil
-}
-func (back *EthBackendImpl) Mining(ctx context.Context) (bool, error) {
-	return back.eth.IsMining(), nil
-}
-
 type RemoteBackend struct {
 	remoteEthBackend remote.ETHBACKENDClient
 	log              log.Logger
@@ -133,6 +87,30 @@ func (back *RemoteBackend) NetVersion(ctx context.Context) (uint64, error) {
 	return res.Id, nil
 }
 
+func (back *RemoteBackend) ProtocolVersion(ctx context.Context) (uint64, error) {
+	res, err := back.remoteEthBackend.ProtocolVersion(ctx, &remote.ProtocolVersionRequest{})
+	if err != nil {
+		if s, ok := status.FromError(err); ok {
+			return 0, errors.New(s.Message())
+		}
+		return 0, err
+	}
+
+	return res.Id, nil
+}
+
+func (back *RemoteBackend) ClientVersion(ctx context.Context) (string, error) {
+	res, err := back.remoteEthBackend.ClientVersion(ctx, &remote.ClientVersionRequest{})
+	if err != nil {
+		if s, ok := status.FromError(err); ok {
+			return "", errors.New(s.Message())
+		}
+		return "", err
+	}
+
+	return res.NodeName, nil
+}
+
 func (back *RemoteBackend) Subscribe(ctx context.Context, onNewEvent func(*remote.SubscribeReply)) error {
 	subscription, err := back.remoteEthBackend.Subscribe(ctx, &remote.SubscribeRequest{})
 	if err != nil {