Merge branch 'develop' into bugfix/ip-validation

Signed-off-by: Nikolay Nedkov <nikolai_nedkov@yahoo.com>

.github/workflows/release.yml

@@ -46,12 +46,12 @@ jobs:
           sed -i -e "s/image: zkevm-node/image: hermeznetwork\/zkevm-node:$GIT_TAG_NAME/g" testnet/docker-compose.yml
           zip -r testnet.zip testnet
           # MAINNET
-          mkdir -p mainnet/config/environments/testnet
+          mkdir -p mainnet/config/environments/mainnet
           mkdir -p mainnet/db/scripts
-          cp config/environments/mainnet/* mainnet/config/environments/testnet
+          cp config/environments/mainnet/* mainnet/config/environments/mainnet
           cp docker-compose.yml mainnet
           cp db/scripts/init_prover_db.sql mainnet/db/scripts
-          mv mainnet/config/environments/testnet/example.env mainnet
+          mv mainnet/config/environments/mainnet/example.env mainnet
           sed -i -e "s/image: zkevm-node/image: hermeznetwork\/zkevm-node:$GIT_TAG_NAME/g" mainnet/docker-compose.yml
           zip -r mainnet.zip mainnet
 
@@ -61,4 +61,4 @@ jobs:
           files: 'testnet.zip;mainnet.zip'
           repo-token: ${{ secrets.TOKEN_RELEASE }}
           release-tag: ${{ steps.tagName.outputs.tag }}
-
+

CONTRIBUTING.md

@@ -2,22 +2,22 @@
 
 This document addresses how we should create PRs, give and receive reviews. The motivation is to have better code, reduce the time from creation to merge while sharing knowledge and insights that help everyone becoming better developers.
 
-Note that non of this is a hard rule, but suggestions / guidelines. Although everyone is encouraged to stick to this points as much as posible. Use your common sense if some of this do not apply well on a particular PR
+Note that non of this is a hard rule, but suggestions / guidelines. Although everyone is encouraged to stick to this points as much as possible. Use your common sense if some of this do not apply well on a particular PR
 
 ## How to create a good PR
 
 - Follow the template, unless for some reason it doesn't fit the content of the PR
 - Try hard on doing small PRs (> ~400 lines), in general is better to have 2 small PRs rather than a big one
 - Indicate clearly who should review it, ideally 2 team mates
-- Author of the PR is responsible for merging. Never do it until you have the aproval of the specified reviewers unless you have their explicit permision
+- Author of the PR is responsible for merging. Never do it until you have the approval of the specified reviewers unless you have their explicit permission
 - Introduce the purpose of the PR, for example: `Fixes the handle of ...`
 - Give brief context on why this is being done and link it to any relevant issue
 - Feel free to ask to specific team mates to review specific parts of the PR
 
 ## How to do a good review
 
-- In general it's hard to set a quality treshold for changes. A good measure for when to approve is to accept changes once the overall quality of the code has been improved (compared to the code base before the PR)
-- Try hard to avoid taking things personaly. For instance avoid using `I`, `you`, `I (don't) like`, ...
+- In general it's hard to set a quality threshold for changes. A good measure for when to approve is to accept changes once the overall quality of the code has been improved (compared to the code base before the PR)
+- Try hard to avoid taking things personally. For instance avoid using `I`, `you`, `I (don't) like`, ...
 - Ask, don’t tell. ("What about trying...?" rather than "Don’t do...")
 - Try to use positive language. You can even use emoji to clarify tone.
 - Be super clear on how confident you are when requesting changes. One way to do it is by starting the message like this:

Dockerfile

@@ -14,7 +14,7 @@ RUN cd /src && make build
 # CONTAINER FOR RUNNING BINARY
 FROM alpine:3.18.0
 COPY --from=build /src/dist/zkevm-node /app/zkevm-node
-COPY --from=build /src/config/environments/testnet/testnet.node.config.toml /app/example.config.toml
+COPY --from=build /src/config/environments/testnet/node.config.toml /app/example.config.toml
 RUN apk update && apk add postgresql15-client
 EXPOSE 8123
 CMD ["/bin/sh", "-c", "/app/zkevm-node run"]

Makefile

@@ -77,7 +77,7 @@ $(GENERATE_SCHEMA_DOC): $(VENV_PYTHON)
 	$(PYTHON) -m pip install json-schema-for-humans
 
 PHONY: config-doc-gen
-config-doc-gen: config-doc-node config-doc-custom_network ## Generate config file's json-schema for node and custom_network  and documentation
+config-doc-gen: config-doc-node config-doc-custom_network ## Generate config file's json-schema for node and custom_network and documentation
 	#
 
 .PHONY: config-doc-node
@@ -119,9 +119,9 @@ install-git-hooks: ## Moves hook files to the .git/hooks directory
 
 .PHONY: generate-code-from-proto
 generate-code-from-proto: ## Generates code from proto files
-	cd proto/src/proto/hashdb/v1 && protoc --proto_path=. --proto_path=../../../../include --go_out=../../../../../merkletree/pb --go-grpc_out=../../../../../merkletree/pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative hashdb.proto
+	cd proto/src/proto/hashdb/v1 && protoc --proto_path=. --proto_path=../../../../include --go_out=../../../../../merkletree/hashdb --go-grpc_out=../../../../../merkletree/hashdb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative hashdb.proto
 	cd proto/src/proto/executor/v1 && protoc --proto_path=. --go_out=../../../../../state/runtime/executor --go-grpc_out=../../../../../state/runtime/executor --go-grpc_opt=paths=source_relative --go_opt=paths=source_relative executor.proto
-	cd proto/src/proto/aggregator/v1 && protoc --proto_path=. --proto_path=../../../../include --go_out=../../../../../aggregator/pb --go-grpc_out=../../../../../aggregator/pb --go-grpc_opt=paths=source_relative --go_opt=paths=source_relative aggregator.proto
+	cd proto/src/proto/aggregator/v1 && protoc --proto_path=. --proto_path=../../../../include --go_out=../../../../../aggregator/prover --go-grpc_out=../../../../../aggregator/prover --go-grpc_opt=paths=source_relative --go_opt=paths=source_relative aggregator.proto
 
 ## Help display.
 ## Pulls comments from beside commands and prints a nicely formatted

README.md

@@ -22,7 +22,6 @@ Glossary:
 - Consolidated state: state that is proven on-chain by submitting a ZKP (Zero Knowledge Proof) that proves the execution of a sequence of the last virtual batch.
 - Invalid transaction: a transaction that can't be processed and doesn't affect the state. Note that such a transaction could be included in a virtual batch. The reason for a transaction to be invalid could be related to the Ethereum protocol (invalid nonce, not enough balance, ...) or due to limitations introduced by the zkEVM (each batch can make use of a limited amount of resources such as the total amount of keccak hashes that can be computed)
 - Reverted transaction: a transaction that is executed, but is reverted (because of smart contract logic). The main difference with *invalid transaction* is that this transaction modifies the state, at least to increment nonce of the sender.
-- Proof of Efficiency (PoE): name of the protocol used by the network, it's enforced by the [smart contracts](https://github.com/0xPolygonHermez/zkevm-contracts)
 
 ## Architecture
 
@@ -32,19 +31,21 @@ Glossary:
 
 The diagram represents the main components of the software and how they interact between them. Note that this reflects a single entity running a node, in particular a node that acts as the trusted sequencer. But there are many entities running nodes in the network, and each of these entities can perform different roles. More on this later.
 
-- (JSON) RPC: an interface that allows users (metamask, etherscan, ...) to interact with the node. Fully compatible with Ethereum RPC + some extra endpoints specifics of the network. It interacts with the `state` to get data and process transactions and with the `pool` to store transactions
+- (JSON) RPC: an HTTP interface that allows users (dApps, metamask, etherscan, ...) to interact with the node. Fully compatible with Ethereum RPC + some extra [custom endpoints](./docs/zkEVM-custom-endpoints.md) specifics of the network. It interacts with the `state` (to get data and process transactions) as well as the `pool` (to store transactions).
+- L2GasPricer: it fetches the L1 gas price and applies some formula to calculate the gas price that will be suggested for the users to use for paying fees on L2. The suggestions are stored on the `pool`, and will be consumed by the `rpc`
 - Pool: DB that stores transactions by the `RPC` to be selected/discarded by the `sequencer` later on
-- Trusted Sequencer: get transactions from the `pool`, check if they are valid by processing them using the `state`, and create sequences. Once transactions are added into the state, they are immediately available through the `rpc`. Sequences are sent to L1 using the `etherman`
-- Permissionless Sequencer: *coming soon*
+- Sequencer: responsible for building the trusted state. To do so, it gets transactions from the pool and puts them in a specific order. It needs to take care of opening and closing batches while trying to make them as full as possible. To achieve this it needs to use the executor to actually process the transaction not only to execute the state transition (and update the hashDB) but also to check the consumed resources by the transactions and the remaining resources of the batch. After executing a transaction that fits into a batch, it gets stored on the `state`. Once transactions are added into the state, they are immediately available through the `rpc`.
+- SequenceSender: gets closed batches from the `state`, tries to aggregate as many of them as possible, and at some point, decides that it's time to send those batches to L1, turning the state from trusted to virtualized. In order to send the L1 tx, it uses the `ethtxmanager`
+- EthTxManager: handles requests to send L1 transactions from `sequencesender` and `aggregator`. It takes care of dealing with the nonce of the accounts, increasing the gas price, and other actions that may be needed to ensure that L1 transactions get mined
 - Etherman: abstraction that implements the needed methods to interact with the Ethereum network and the relevant smart contracts.
-- Synchronizer: Updates the `state` by fetching data from Ethereum through the `etherman`. If the node is not a `trusted sequencer` it also updates the state with the data fetched from the `rpc` of the `trusted sequencer`. It also detects and handles reorgs that can happen if the `trusted sequencer` sends different data in the rpc vs the sequences sent to L1 (trusted vs virtual state)
+- Synchronizer: Updates the `state` (virtual batches, verified batches, forced batches, ...) by fetching data from L1 through the `etherman`. If the node is not a `trusted sequencer` it also updates the state with the data fetched from the `rpc` of the `trusted sequencer`. It also detects and handles reorgs that can happen if the `trusted sequencer` sends different data in the rpc vs the sequences sent to L1 (trusted reorg aka L2 reorg). Also handles L1 reorgs (reorgs that happen on the L1 network)
 - State: Responsible for managing the state data (batches, blocks, transactions, ...) that is stored on the `state SB`. It also handles the integration with the `executor` and the `Merkletree` service
-- State DB: persistence layer for the state data (except the Merkletree that is handled by the `Merkletree` service)
-- Aggregator: consolidates batches by generating ZKPs (Zero Knowledge proofs). To do so it gathers the necessary data that the `prover` needs as input through the `state` and sends a request to it. Once the proof is generated it's sent to Ethereum through the `etherman`
-- Prover/Executor: service that generates ZK proofs. Note that this component is not implemented in this repository, and it's treated as a "black box" from the perspective of the node. The prover/executor has two implementations: [JS reference implementation](https://github.com/0xPolygonHermez/zkevm-proverjs) and [C production-ready implementation](https://github.com/0xPolygonHermez/zkevm-prover). Although it's the same software/service, it has two very different purposes:
-  - Provide an EVM implementation that allows processing transactions and getting all needed results metadata (state root, receipts, logs, ...)
-  - Generate ZKPs
-- Merkletree: service that stores the Merkletree, containing all the account information (balances, nonces, smart contract code, and smart contract storage). This component is also not implemented in this repo and is consumed as an external service by the node. The implementation can be found [here](https://github.com/0xPolygonHermez/zkevm-prover)
+- State DB: persistence layer for the state data (except the Merkletree that is handled by the `HashDB` service), it stores informationrelated to L1 (blocks, global exit root updates, ...) and L2 (batches, L2 blocks, transactions, ...)
+- Aggregator: consolidates batches by generating ZKPs (Zero Knowledge proofs). To do so it gathers the necessary data that the `prover` needs as input through the `state` and sends a request to it. Once the proof is generated it sends a request to send an L1 tx to verify the proof and move the state from virtual to verified to the `ethtxmanager`. Note that provers connect to the aggregator and not the other way arround. The aggregator can handle multiple connected provers at once and make them work concurrently in the generation of different proofs
+- Prover/Executor/hashDB: service that generates ZK proofs. Note that this component is not implemented in this repository, and it's treated as a "black box" from the perspective of the node. The prover/executor has two implementations: [JS reference implementation](https://github.com/0xPolygonHermez/zkevm-proverjs) and [C production-ready implementation](https://github.com/0xPolygonHermez/zkevm-prover). Although it's the same software/binary, it implements three services:
+  - Executor: Provides an EVM implementation that allows processing batches as well as getting metadata (state root, transaction receipts, logs, ...) of all the needed results.
+  - Prover: Generates ZKPs for batches, batches aggregation, and final proofs.
+  - HashDB: service that stores the Merkletree, containing all the account information (balances, nonces, smart contract code, and smart contract storage)
 
 ## Roles of the network
 
@@ -80,10 +81,6 @@ Required services and components:
 
 Note that the JSON RPC is required to receive transactions. It's recommended that the JSON RPC runs on separated instances, and potentially more than one (depending on the load of the network). It's also recommended that the JSON RPC and the Sequencer don't share the same executor instance, to make sure that the sequencer has exclusive access to an executor
 
-### Permissionless sequencer
-
-TBD
-
 ### Aggregator
 
 This role can be performed by anyone.
@@ -118,6 +115,6 @@ It's recommended to use `make` for building, and testing the code, ... Run `make
 
 ## Contribute
 
-Before opening a pull request, please read this [guide](CONTRIBUTING.md)
+Before opening a pull request, please read this [guide](CONTRIBUTING.md).
 
 

aggregator/aggregator.go

@@ -14,7 +14,6 @@ import (
 	"unicode"
 
 	"github.com/0xPolygonHermez/zkevm-node/aggregator/metrics"
-	"github.com/0xPolygonHermez/zkevm-node/aggregator/pb"
 	"github.com/0xPolygonHermez/zkevm-node/aggregator/prover"
 	"github.com/0xPolygonHermez/zkevm-node/config/types"
 	"github.com/0xPolygonHermez/zkevm-node/encoding"
@@ -41,12 +40,12 @@ type finalProofMsg struct {
 	proverName     string
 	proverID       string
 	recursiveProof *state.Proof
-	finalProof     *pb.FinalProof
+	finalProof     *prover.FinalProof
 }
 
 // Aggregator represents an aggregator
 type Aggregator struct {
-	pb.UnimplementedAggregatorServiceServer
+	prover.UnimplementedAggregatorServiceServer
 
 	cfg Config
 
@@ -129,7 +128,7 @@ func (a *Aggregator) Start(ctx context.Context) error {
 	}
 
 	a.srv = grpc.NewServer()
-	pb.RegisterAggregatorServiceServer(a.srv, a)
+	prover.RegisterAggregatorServiceServer(a.srv, a)
 
 	healthService := newHealthChecker()
 	grpchealth.RegisterHealthServer(a.srv, healthService)
@@ -159,7 +158,7 @@ func (a *Aggregator) Stop() {
 
 // Channel implements the bi-directional communication channel between the
 // Prover client and the Aggregator server.
-func (a *Aggregator) Channel(stream pb.AggregatorService_ChannelServer) error {
+func (a *Aggregator) Channel(stream prover.AggregatorService_ChannelServer) error {
 	metrics.ConnectedProver()
 	defer metrics.DisconnectedProver()
 
@@ -306,7 +305,7 @@ func (a *Aggregator) handleFailureToAddVerifyBatchToBeMonitored(ctx context.Cont
 }
 
 // buildFinalProof builds and return the final proof for an aggregated/batch proof.
-func (a *Aggregator) buildFinalProof(ctx context.Context, prover proverInterface, proof *state.Proof) (*pb.FinalProof, error) {
+func (a *Aggregator) buildFinalProof(ctx context.Context, prover proverInterface, proof *state.Proof) (*prover.FinalProof, error) {
 	log := log.WithFields(
 		"prover", prover.Name(),
 		"proverId", prover.ID(),
@@ -972,14 +971,14 @@ func (a *Aggregator) isSynced(ctx context.Context, batchNum *uint64) bool {
 	return true
 }
 
-func (a *Aggregator) buildInputProver(ctx context.Context, batchToVerify *state.Batch) (*pb.InputProver, error) {
+func (a *Aggregator) buildInputProver(ctx context.Context, batchToVerify *state.Batch) (*prover.InputProver, error) {
 	previousBatch, err := a.State.GetBatchByNumber(ctx, batchToVerify.BatchNumber-1, nil)
 	if err != nil && err != state.ErrStateNotSynchronized {
 		return nil, fmt.Errorf("failed to get previous batch, err: %v", err)
 	}
 
-	inputProver := &pb.InputProver{
-		PublicInputs: &pb.PublicInputs{
+	inputProver := &prover.InputProver{
+		PublicInputs: &prover.PublicInputs{
 			OldStateRoot:    previousBatch.StateRoot.Bytes(),
 			OldAccInputHash: previousBatch.AccInputHash.Bytes(),
 			OldBatchNum:     previousBatch.BatchNumber,

aggregator/aggregator_test.go

@@ -10,7 +10,7 @@ import (
 	"time"
 
 	"github.com/0xPolygonHermez/zkevm-node/aggregator/mocks"
-	"github.com/0xPolygonHermez/zkevm-node/aggregator/pb"
+	"github.com/0xPolygonHermez/zkevm-node/aggregator/prover"
 	configTypes "github.com/0xPolygonHermez/zkevm-node/config/types"
 	ethmanTypes "github.com/0xPolygonHermez/zkevm-node/etherman/types"
 	"github.com/0xPolygonHermez/zkevm-node/ethtxmanager"
@@ -53,7 +53,7 @@ func TestSendFinalProof(t *testing.T) {
 		BatchNumber:      batchNum,
 		BatchNumberFinal: batchNumFinal,
 	}
-	finalProof := &pb.FinalProof{}
+	finalProof := &prover.FinalProof{}
 	cfg := Config{SenderAddress: from.Hex()}
 
 	testCases := []struct {
@@ -1000,9 +1000,9 @@ func TestTryBuildFinalProof(t *testing.T) {
 	proverName := "proverName"
 	proverID := "proverID"
 	finalProofID := "finalProofID"
-	finalProof := pb.FinalProof{
+	finalProof := prover.FinalProof{
 		Proof: "",
-		Public: &pb.PublicInputsExtended{
+		Public: &prover.PublicInputsExtended{
 			NewStateRoot:     []byte("newStateRoot"),
 			NewLocalExitRoot: []byte("newLocalExitRoot"),
 		},

aggregator/interfaces.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"math/big"
 
-	"github.com/0xPolygonHermez/zkevm-node/aggregator/pb"
+	"github.com/0xPolygonHermez/zkevm-node/aggregator/prover"
 	ethmanTypes "github.com/0xPolygonHermez/zkevm-node/etherman/types"
 	"github.com/0xPolygonHermez/zkevm-node/ethtxmanager"
 	"github.com/0xPolygonHermez/zkevm-node/state"
@@ -19,11 +19,11 @@ type proverInterface interface {
 	ID() string
 	Addr() string
 	IsIdle() (bool, error)
-	BatchProof(input *pb.InputProver) (*string, error)
+	BatchProof(input *prover.InputProver) (*string, error)
 	AggregatedProof(inputProof1, inputProof2 string) (*string, error)
 	FinalProof(inputProof string, aggregatorAddr string) (*string, error)
 	WaitRecursiveProof(ctx context.Context, proofID string) (string, error)
-	WaitFinalProof(ctx context.Context, proofID string) (*pb.FinalProof, error)
+	WaitFinalProof(ctx context.Context, proofID string) (*prover.FinalProof, error)
 }
 
 // ethTxManager contains the methods required to send txs to