Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Validium mode #1017

Closed
wants to merge 11 commits into from
2,732 changes: 1,963 additions & 769 deletions Cargo.lock

Large diffs are not rendered by default.

2 changes: 2 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ members = [

# SDK section
"sdk/zksync-rs",

"validium_mode_example",
]
resolver = "2"

Expand Down
2 changes: 2 additions & 0 deletions core/lib/config/src/configs/eth_sender.rs
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ impl ETHSenderConfig {
internal_enforced_l1_gas_price: None,
poll_period: 5,
max_l1_gas_price: None,
l1_gas_per_pubdata_byte: 17,
},
}
}
Expand Down Expand Up @@ -135,6 +136,7 @@ pub struct GasAdjusterConfig {
pub poll_period: u64,
/// Max number of l1 gas price that is allowed to be used in state keeper.
pub max_l1_gas_price: Option<u64>,
pub l1_gas_per_pubdata_byte: u64,
}

impl GasAdjusterConfig {
Expand Down
1 change: 1 addition & 0 deletions core/lib/env_config/src/eth_sender.rs
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ mod tests {
internal_enforced_l1_gas_price: None,
poll_period: 15,
max_l1_gas_price: Some(100000000),
l1_gas_per_pubdata_byte: 17,
},
}
}
Expand Down
41 changes: 22 additions & 19 deletions core/lib/types/src/commitment.rs
Original file line number Diff line number Diff line change
Expand Up @@ -135,29 +135,32 @@ impl L1BatchWithMetadata {
/// This data is currently part of calldata but will be submitted as part of the blob section post EIP-4844.
pub fn construct_pubdata(&self) -> Vec<u8> {
let mut res: Vec<u8> = vec![];
let validium_mode = std::env::var("VALIDIUM_MODE") == Ok("true".to_owned());

// Process and Pack Logs
res.extend((self.header.l2_to_l1_logs.len() as u32).to_be_bytes());
for l2_to_l1_log in &self.header.l2_to_l1_logs {
res.extend(l2_to_l1_log.0.to_bytes());
}
if !validium_mode {
// Process and Pack Logs
res.extend((self.header.l2_to_l1_logs.len() as u32).to_be_bytes());
for l2_to_l1_log in &self.header.l2_to_l1_logs {
res.extend(l2_to_l1_log.0.to_bytes());
}

// Process and Pack Messages
res.extend((self.header.l2_to_l1_messages.len() as u32).to_be_bytes());
for msg in &self.header.l2_to_l1_messages {
res.extend((msg.len() as u32).to_be_bytes());
res.extend(msg);
}
// Process and Pack Messages
res.extend((self.header.l2_to_l1_messages.len() as u32).to_be_bytes());
for msg in &self.header.l2_to_l1_messages {
res.extend((msg.len() as u32).to_be_bytes());
res.extend(msg);
}

// Process and Pack Bytecodes
res.extend((self.factory_deps.len() as u32).to_be_bytes());
for bytecode in &self.factory_deps {
res.extend((bytecode.len() as u32).to_be_bytes());
res.extend(bytecode);
}
// Process and Pack Bytecodes
res.extend((self.factory_deps.len() as u32).to_be_bytes());
for bytecode in &self.factory_deps {
res.extend((bytecode.len() as u32).to_be_bytes());
res.extend(bytecode);
}

// Extend with Compressed StateDiffs
res.extend(&self.metadata.state_diffs_compressed);
// Extend with Compressed StateDiffs
res.extend(&self.metadata.state_diffs_compressed);
}

res
}
Expand Down
3 changes: 1 addition & 2 deletions core/lib/zksync_core/src/l1_gas_price/gas_adjuster/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ use std::{
use tokio::sync::watch;
use zksync_config::GasAdjusterConfig;
use zksync_eth_client::{Error, EthInterface};
use zksync_system_constants::L1_GAS_PER_PUBDATA_BYTE;

use self::metrics::METRICS;
use super::{L1GasPriceProvider, L1TxParamsProvider};
Expand Down Expand Up @@ -130,7 +129,7 @@ impl<E: EthInterface> L1GasPriceProvider for GasAdjuster<E> {

fn estimate_effective_pubdata_price(&self) -> u64 {
// For now, pubdata is only sent via calldata, so its price is pegged to the L1 gas price.
self.estimate_effective_gas_price() * L1_GAS_PER_PUBDATA_BYTE as u64
self.estimate_effective_gas_price() * self.config.l1_gas_per_pubdata_byte
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The L1_GAS_PER_PUBDATA_BYTE variable is used in many other parts of the code, so we risk inconsistency (if config value changes, this component will consider a different value than others). Additionally, it doesn't feel right that in config when we set the mode (rollup/validium) we also need to set this gas per pubdata byte to zero.

What if we have a trait GasAdjuster with all its logic identical to this file, and two structs that implement it:

struct RollupGasAdjuster {
    l1_gas_per_pubdata_byte: L1_GAS_PER_PUBDATA_BYTE
}

struct ValidiumGasAdjuster {
    l1_gas_per_pubdata_byte: 0
}

This also allows for different DA modes to have slightly different behaviour when predicting L1/DA cost of a transaction

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please also consider this comment for other places where you add/change configs. Would the same reasoning apply?

Copy link
Contributor Author

@ilitteri ilitteri Feb 7, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The L1_GAS_PER_PUBDATA_BYTE variable is used in many other parts of the code, so we risk inconsistency (if config value changes, this component will consider a different value than others). Additionally, it doesn't feel right that in config when we set the mode (rollup/validium) we also need to set this gas per pubdata byte to zero.

What if we have a trait GasAdjuster with all its logic identical to this file, and two structs that implement it:

struct RollupGasAdjuster {
    l1_gas_per_pubdata_byte: L1_GAS_PER_PUBDATA_BYTE
}

struct ValidiumGasAdjuster {
    l1_gas_per_pubdata_byte: 0
}

This also allows for different DA modes to have slightly different behaviour when predicting L1/DA cost of a transaction

I completely agree with this (actually, this design passed through my mind but I was not sure because of the complexity it implies and because of it comes with big changes), we'll move forward with this request.

Please also consider this comment for other places where you add/change configs. Would the same reasoning apply?

For sure! We'll take a look at the configs we've added/changed and take a similar approach for that (in the case the conclusion is that it'd be right for us as it is, I'll also let you know so you can see if it's ok or if you have some other idea).

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The L1_GAS_PER_PUBDATA_BYTE variable is used in many other parts of the code, so we risk inconsistency (if config value changes, this component will consider a different value than others). Additionally, it doesn't feel right that in config when we set the mode (rollup/validium) we also need to set this gas per pubdata byte to zero.

What if we have a trait GasAdjuster with all its logic identical to this file, and two structs that implement it:

struct RollupGasAdjuster {
    l1_gas_per_pubdata_byte: L1_GAS_PER_PUBDATA_BYTE
}

struct ValidiumGasAdjuster {
    l1_gas_per_pubdata_byte: 0
}

This also allows for different DA modes to have slightly different behaviour when predicting L1/DA cost of a transaction

A note on this, making this change will imply changing the actual [eth_sender.gas_adjuster] section for [eth_sender.rollup_gas_adjuster] and adding a [eth_sender.validium_gas_adjuster] one (in addition with another match on the l1_bach_commit_data_generator_mode config to instantiate either RollupGasAdjuster or ValidiumGasAdjuster (I think that now that this config is being used in more than one case we should think in another name for it, if you have any ideas on generalizing Rollup and Validium modes it'd be helpful).

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A note on this, making this change will imply changing the actual [eth_sender.gas_adjuster] section for [eth_sender.rollup_gas_adjuster] and adding a [eth_sender.validium_gas_adjuster] one

Hmmm why is it needed though?

Copy link
Contributor Author

@ilitteri ilitteri Feb 7, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd also suggest setting another constant/config for the Validium's l1_pubdata_price given that as said here: https://github.com/matter-labs/zksync-era/blob/2fa6a7369b0f4fbd94e0dda3cec06ceeda090d15/docs/guides/advanced/fee_model.md#validium--data-availability-configurations

If you're running an alternative DA, you should adjust the l1_pubdata_price to roughly cover the cost of writing one byte to the DA, and set

In the mean time I'll come back with something.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd also suggest setting another constant/config for the Validium's l1_pubdata_price given that as said here:

ah, makes sense. It's indeed possible, but given we don't have alternative DAs (and we can run the first testnet version without one I think), we can add this config later

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Got it! I'll add a comment in the line and file an issue internally (and in this repo after the code is merged) to track this TODO.

Copy link
Contributor Author

@ilitteri ilitteri Feb 16, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hi @RomanBrodetski, we created a PR to test how this change would look, and we think it adds too much complexity, making the codebase messier just to avoid adding one more configuration parameter. We suggest keeping it simpler, but if there's no way for the current solution to fit (or the new one), we can think of other solutions.

Copy link
Contributor Author

@ilitteri ilitteri Feb 16, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As an addition to the above comment, feel free to comment and ask any questions on the linked PR.

}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ async fn kept_updated() {
internal_enforced_l1_gas_price: None,
poll_period: 5,
max_l1_gas_price: None,
l1_gas_per_pubdata_byte: 17,
},
)
.await
Expand Down
1 change: 1 addition & 0 deletions core/lib/zksync_core/src/state_keeper/io/tests/tester.rs
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ impl Tester {
internal_enforced_l1_gas_price: None,
poll_period: 10,
max_l1_gas_price: None,
l1_gas_per_pubdata_byte: 17,
};

GasAdjuster::new(eth_client, gas_adjuster_config)
Expand Down
14 changes: 9 additions & 5 deletions etc/env/base/chain.toml
Original file line number Diff line number Diff line change
Expand Up @@ -52,21 +52,25 @@ minimal_l2_gas_price=100000000
# The constant that represents the possibility that a batch can be sealed because of overuse of computation resources.
# It has range from 0 to 1. If it is 0, the compute will not depend on the cost for closing the batch.
# If it is 1, the gas limit per batch will have to cover the entire cost of closing the batch.
compute_overhead_part=0.0
# compute_overhead_part=0.0
compute_overhead_part=1.0

# The constant that represents the possibility that a batch can be sealed because of overuse of pubdata.
# It has range from 0 to 1. If it is 0, the pubdata will not depend on the cost for closing the batch.
# If it is 1, the pubdata limit per batch will have to cover the entire cost of closing the batch.
pubdata_overhead_part=1.0
# pubdata_overhead_part=1.0
pubdata_overhead_part=0

# The constant amount of L1 gas that is used as the overhead for the batch. It includes the price for batch verification, etc.
batch_overhead_l1_gas=800000
# batch_overhead_l1_gas=800000
batch_overhead_l1_gas=1000000

# The maximum amount of gas that can be used by the batch. This value is derived from the circuits limitation per batch.
max_gas_per_batch=200000000

# The maximum amount of pubdata that can be used by the batch. Note that if the calldata is used as pubdata, this variable should not exceed 128kb.
max_pubdata_per_batch=100000
# max_pubdata_per_batch=100000
max_pubdata_per_batch=1000000000000

# The version of the fee model to use.
# - `V1`, the first model that was used in zkSync Era. In this fee model, the pubdata price must be pegged to the L1 gas price.
Expand All @@ -75,7 +79,7 @@ max_pubdata_per_batch=100000
# - `V2`, the second model that was used in zkSync Era. There the pubdata price might be independent from the L1 gas price. Also,
# The fair L2 gas price is expected to both the proving/computation price for the operator and the costs that come from
# processing the batch on L1.
fee_model_version="V1"
fee_model_version="V2"

# Max number of computational gas that validation step is allowed to take.
validation_computational_gas_limit=300000
Expand Down
2 changes: 2 additions & 0 deletions etc/env/base/eth_sender.toml
Original file line number Diff line number Diff line change
Expand Up @@ -62,3 +62,5 @@ pricing_formula_parameter_b=1.0005
internal_l1_pricing_multiplier=0.8
# Node polling period in seconds.
poll_period=5
internal_enforced_l1_gas_price=45_000_000_000
l1_gas_per_pubdata_byte=0
6 changes: 4 additions & 2 deletions infrastructure/zk/src/hyperchain_wizard.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,8 @@ async function initHyperchain() {
testTokens: {
deploy: deployTestTokens,
args: ['--private-key', deployerPrivateKey, '--envFile', process.env.CHAIN_ETH_NETWORK!]
}
},
validiumMode: false
};

await init(initArgs);
Expand Down Expand Up @@ -788,7 +789,8 @@ async function configDemoHyperchain(cmd: Command) {
testTokens: {
deploy: deployTestTokens,
args: ['--private-key', deployerPrivateKey, '--envFile', process.env.CHAIN_ETH_NETWORK!]
}
},
validiumMode: false
};

if (!cmd.skipEnvSetup) {
Expand Down
44 changes: 36 additions & 8 deletions infrastructure/zk/src/init.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,27 @@ import * as run from './run/run';
import * as server from './server';
import { up } from './up';

import * as fs from 'fs';

const entry = chalk.bold.yellow;
const announce = chalk.yellow;
const success = chalk.green;
const timestamp = chalk.grey;

export async function init(initArgs: InitArgs = DEFAULT_ARGS) {
const { skipSubmodulesCheckout, skipEnvSetup, testTokens, governorPrivateKeyArgs, deployerL2ContractInput } =
initArgs;
const {
skipSubmodulesCheckout,
skipEnvSetup,
testTokens,
governorPrivateKeyArgs,
deployerL2ContractInput,
validiumMode
} = initArgs;

let envFileContent = fs.readFileSync(process.env.ENV_FILE!).toString();
envFileContent += `VALIDIUM_MODE=${validiumMode}\n`;
fs.writeFileSync(process.env.ENV_FILE!, envFileContent);
await announced(`Initializing in ${validiumMode ? 'Validium mode' : 'Roll-up mode'}`);

if (!process.env.CI && !skipEnvSetup) {
await announced('Pulling images', docker.pull());
Expand Down Expand Up @@ -63,7 +76,10 @@ export async function init(initArgs: InitArgs = DEFAULT_ARGS) {

// A smaller version of `init` that "resets" the localhost environment, for which `init` was already called before.
// It does less and runs much faster.
export async function reinit() {
export async function reinit(validiumMode: boolean) {
process.env.VALIDIUM_MODE = validiumMode.toString();
await announced(`Initializing in ${validiumMode ? 'Validium mode' : 'Roll-up mode'}`);

await announced('Setting up containers', up());
await announced('Compiling JS packages', run.yarn());
await announced('Compile l2 contracts', compiler.compileAll());
Expand All @@ -83,7 +99,11 @@ export async function reinit() {
}

// A lightweight version of `init` that sets up local databases, generates genesis and deploys precompiled contracts
export async function lightweightInit() {
export async function lightweightInit(validiumMode: boolean) {
process.env.VALIDIUM_MODE = validiumMode.toString();
await announced(`Initializing in ${validiumMode ? 'Validium mode' : 'Roll-up mode'}`);

await announced(`Setting up containers`, up());
await announced('Clean rocksdb', clean('db'));
await announced('Clean backups', clean('backups'));
await announced('Deploying L1 verifier', contract.deployVerifier([]));
Expand Down Expand Up @@ -144,33 +164,41 @@ export interface InitArgs {
deploy: boolean;
args: any[];
};
validiumMode: boolean;
}

const DEFAULT_ARGS: InitArgs = {
skipSubmodulesCheckout: false,
skipEnvSetup: false,
governorPrivateKeyArgs: [],
deployerL2ContractInput: { args: [], includePaymaster: true, includeL2WETH: true },
testTokens: { deploy: true, args: [] }
testTokens: { deploy: true, args: [] },
validiumMode: false
};

export const initCommand = new Command('init')
.option('--skip-submodules-checkout')
.option('--skip-env-setup')
.option('--validium-mode')
.description('perform zksync network initialization for development')
.action(async (cmd: Command) => {
const initArgs: InitArgs = {
skipSubmodulesCheckout: cmd.skipSubmodulesCheckout,
skipEnvSetup: cmd.skipEnvSetup,
governorPrivateKeyArgs: [],
deployerL2ContractInput: { args: [], includePaymaster: true, includeL2WETH: true },
testTokens: { deploy: true, args: [] }
testTokens: { deploy: true, args: [] },
validiumMode: cmd.validiumMode !== undefined ? cmd.validiumMode : false
};
await init(initArgs);
});
export const reinitCommand = new Command('reinit')
.description('"reinitializes" network. Runs faster than `init`, but requires `init` to be executed prior')
.action(reinit);
.action(async (cmd: Command) => {
await reinit(cmd.validiumMode);
});
export const lightweightInitCommand = new Command('lightweight-init')
.description('perform lightweight zksync network initialization for development')
.action(lightweightInit);
.action(async (cmd: Command) => {
await lightweightInit(cmd.validiumMode);
});
1 change: 1 addition & 0 deletions validium_mode_example/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
/target
39 changes: 39 additions & 0 deletions validium_mode_example/BytesWriter.abi
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
[
{
"inputs": [
{
"internalType": "bytes",
"name": "_message",
"type": "bytes"
}
],
"stateMutability": "nonpayable",
"type": "constructor"
},
{
"inputs": [],
"name": "readBytes",
"outputs": [
{
"internalType": "bytes",
"name": "",
"type": "bytes"
}
],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [
{
"internalType": "bytes",
"name": "_message",
"type": "bytes"
}
],
"name": "writeBytes",
"outputs": [],
"stateMutability": "nonpayable",
"type": "function"
}
]
Loading
Loading