feat(vm): Allow running new VM in state keeper (#2616)

## What ❔ - Allows running the new VM in the state keeper. - Adds a CI load test with the new VM. ## Why ❔ Allows to monitor new VM performance regressions.
matter-labs · Aug 9, 2024 · 7da372f · 7da372f
1 parent 9a83049
commit 7da372f
Show file tree

Hide file tree

Showing 16 changed files with 145 additions and 62 deletions.
diff --git a/.github/workflows/ci-core-reusable.yml b/.github/workflows/ci-core-reusable.yml
@@ -67,6 +67,10 @@ jobs:
 
   loadtest:
     runs-on: [matterlabs-ci-runner]
+    strategy:
+      fail-fast: false
+      matrix:
+        vm_mode: ["old", "new"]
 
     steps:
       - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
@@ -82,7 +86,8 @@ jobs:
 
       - name: Loadtest configuration
         run: |
-          echo EXPECTED_TX_COUNT="16000" >> .env
+          echo EXPECTED_TX_COUNT=${{ matrix.vm_mode == 'new' && 24000 || 18000 }} >> .env
+          echo ACCOUNTS_AMOUNT="150" >> .env
           echo FAIL_FAST=true >> .env
           echo IN_DOCKER=1 >> .env
           echo DATABASE_MERKLE_TREE_MODE=lightweight >> .env
@@ -105,7 +110,9 @@ jobs:
       # `sleep 60` because we need to wait until server added all the tokens
       - name: Run server
         run: |
-          ci_run zk server --uring --components api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads &>server.log &
+          EXPERIMENTAL_VM_STATE_KEEPER_FAST_VM_MODE=${{ matrix.vm_mode }} \
+          PASSED_ENV_VARS="EXPERIMENTAL_VM_STATE_KEEPER_FAST_VM_MODE" \
+            ci_run zk server --uring --components api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads &>server.log &
           ci_run sleep 60
 
       - name: Deploy legacy era contracts

diff --git a/core/bin/zksync_server/src/main.rs b/core/bin/zksync_server/src/main.rs
@@ -11,11 +11,11 @@ use zksync_config::{
         },
         fri_prover_group::FriProverGroupConfig,
         house_keeper::HouseKeeperConfig,
-        BasicWitnessInputProducerConfig, ContractsConfig, DatabaseSecrets,
-        ExperimentalVmPlaygroundConfig, ExternalPriceApiClientConfig, FriProofCompressorConfig,
-        FriProverConfig, FriProverGatewayConfig, FriWitnessGeneratorConfig,
-        FriWitnessVectorGeneratorConfig, L1Secrets, ObservabilityConfig, PrometheusConfig,
-        ProofDataHandlerConfig, ProtectiveReadsWriterConfig, Secrets,
+        BasicWitnessInputProducerConfig, ContractsConfig, DatabaseSecrets, ExperimentalVmConfig,
+        ExternalPriceApiClientConfig, FriProofCompressorConfig, FriProverConfig,
+        FriProverGatewayConfig, FriWitnessGeneratorConfig, FriWitnessVectorGeneratorConfig,
+        L1Secrets, ObservabilityConfig, PrometheusConfig, ProofDataHandlerConfig,
+        ProtectiveReadsWriterConfig, Secrets,
     },
     ApiConfig, BaseTokenAdjusterConfig, ContractVerifierConfig, DADispatcherConfig, DBConfig,
     EthConfig, EthWatchConfig, ExternalProofIntegrationApiConfig, GasAdjusterConfig, GenesisConfig,
@@ -202,13 +202,13 @@ fn load_env_config() -> anyhow::Result<TempConfigStore> {
         da_dispatcher_config: DADispatcherConfig::from_env().ok(),
         protective_reads_writer_config: ProtectiveReadsWriterConfig::from_env().ok(),
         basic_witness_input_producer_config: BasicWitnessInputProducerConfig::from_env().ok(),
-        vm_playground_config: ExperimentalVmPlaygroundConfig::from_env().ok(),
         core_object_store: ObjectStoreConfig::from_env().ok(),
         base_token_adjuster_config: BaseTokenAdjusterConfig::from_env().ok(),
         commitment_generator: None,
         pruning: None,
         snapshot_recovery: None,
         external_price_api_client_config: ExternalPriceApiClientConfig::from_env().ok(),
         external_proof_integration_api_config: ExternalProofIntegrationApiConfig::from_env().ok(),
+        experimental_vm_config: ExperimentalVmConfig::from_env().ok(),
     })
 }
diff --git a/core/bin/zksync_server/src/node_builder.rs b/core/bin/zksync_server/src/node_builder.rs
@@ -242,8 +242,10 @@ impl MainNodeBuilder {
             try_load_config!(wallets.state_keeper),
         );
         let db_config = try_load_config!(self.configs.db_config);
+        let experimental_vm_config = try_load_config!(self.configs.experimental_vm_config);
         let main_node_batch_executor_builder_layer =
-            MainBatchExecutorLayer::new(sk_config.save_call_traces, OPTIONAL_BYTECODE_COMPRESSION);
+            MainBatchExecutorLayer::new(sk_config.save_call_traces, OPTIONAL_BYTECODE_COMPRESSION)
+                .with_fast_vm_mode(experimental_vm_config.state_keeper_fast_vm_mode);
 
         let rocksdb_options = RocksdbStorageOptions {
             block_cache_capacity: db_config
@@ -568,9 +570,9 @@ impl MainNodeBuilder {
     }
 
     fn add_vm_playground_layer(mut self) -> anyhow::Result<Self> {
-        let vm_playground_config = try_load_config!(self.configs.vm_playground_config);
+        let vm_config = try_load_config!(self.configs.experimental_vm_config);
         self.node.add_layer(VmPlaygroundLayer::new(
-            vm_playground_config,
+            vm_config.playground,
             self.genesis_config.l2_chain_id,
         ));
 

diff --git a/core/lib/config/src/configs/experimental.rs b/core/lib/config/src/configs/experimental.rs
@@ -62,7 +62,8 @@ impl ExperimentalDBConfig {
     }
 }
 
-#[derive(Debug, Deserialize, Clone, PartialEq)]
+/// Configuration for the VM playground (an experimental component that's unlikely to ever be stabilized).
+#[derive(Debug, Clone, PartialEq, Deserialize)]
 pub struct ExperimentalVmPlaygroundConfig {
     /// Mode in which to run the fast VM implementation. Note that for it to actually be used, L1 batches should have a recent version.
     #[serde(default)]
@@ -95,3 +96,15 @@ impl ExperimentalVmPlaygroundConfig {
         "./db/vm_playground".to_owned()
     }
 }
+
+/// Experimental VM configuration options.
+#[derive(Debug, Clone, Default, PartialEq, Deserialize)]
+pub struct ExperimentalVmConfig {
+    #[serde(skip)] // Isn't properly deserialized by `envy`
+    pub playground: ExperimentalVmPlaygroundConfig,
+
+    /// Mode in which to run the fast VM implementation in the state keeper. Should not be set in production;
+    /// the new VM doesn't produce call traces and can diverge from the old VM!
+    #[serde(default)]
+    pub state_keeper_fast_vm_mode: FastVmMode,
+}
diff --git a/core/lib/config/src/configs/general.rs b/core/lib/config/src/configs/general.rs
@@ -9,7 +9,7 @@ use crate::{
         pruning::PruningConfig,
         snapshot_recovery::SnapshotRecoveryConfig,
         vm_runner::{BasicWitnessInputProducerConfig, ProtectiveReadsWriterConfig},
-        CommitmentGeneratorConfig, ExperimentalVmPlaygroundConfig, ExternalPriceApiClientConfig,
+        CommitmentGeneratorConfig, ExperimentalVmConfig, ExternalPriceApiClientConfig,
         FriProofCompressorConfig, FriProverConfig, FriProverGatewayConfig,
         FriWitnessGeneratorConfig, FriWitnessVectorGeneratorConfig, ObservabilityConfig,
         PrometheusConfig, ProofDataHandlerConfig,
@@ -43,7 +43,6 @@ pub struct GeneralConfig {
     pub da_dispatcher_config: Option<DADispatcherConfig>,
     pub protective_reads_writer_config: Option<ProtectiveReadsWriterConfig>,
     pub basic_witness_input_producer_config: Option<BasicWitnessInputProducerConfig>,
-    pub vm_playground_config: Option<ExperimentalVmPlaygroundConfig>,
     pub commitment_generator: Option<CommitmentGeneratorConfig>,
     pub snapshot_recovery: Option<SnapshotRecoveryConfig>,
     pub pruning: Option<PruningConfig>,
@@ -52,4 +51,5 @@ pub struct GeneralConfig {
     pub external_price_api_client_config: Option<ExternalPriceApiClientConfig>,
     pub consensus_config: Option<ConsensusConfig>,
     pub external_proof_integration_api_config: Option<ExternalProofIntegrationApiConfig>,
+    pub experimental_vm_config: Option<ExperimentalVmConfig>,
 }
diff --git a/core/lib/config/src/configs/mod.rs b/core/lib/config/src/configs/mod.rs
@@ -9,7 +9,7 @@ pub use self::{
     database::{DBConfig, PostgresConfig},
     eth_sender::{EthConfig, GasAdjusterConfig},
     eth_watch::EthWatchConfig,
-    experimental::{ExperimentalDBConfig, ExperimentalVmPlaygroundConfig},
+    experimental::{ExperimentalDBConfig, ExperimentalVmConfig, ExperimentalVmPlaygroundConfig},
     external_price_api_client::ExternalPriceApiClientConfig,
     external_proof_integration_api::ExternalProofIntegrationApiConfig,
     fri_proof_compressor::FriProofCompressorConfig,

diff --git a/core/lib/config/src/testonly.rs b/core/lib/config/src/testonly.rs
@@ -295,18 +295,31 @@ impl Distribution<configs::ExperimentalDBConfig> for EncodeDist {
 impl Distribution<configs::ExperimentalVmPlaygroundConfig> for EncodeDist {
     fn sample<R: Rng + ?Sized>(&self, rng: &mut R) -> configs::ExperimentalVmPlaygroundConfig {
         configs::ExperimentalVmPlaygroundConfig {
-            fast_vm_mode: match rng.gen_range(0..3) {
-                0 => FastVmMode::Old,
-                1 => FastVmMode::New,
-                _ => FastVmMode::Shadow,
-            },
+            fast_vm_mode: gen_fast_vm_mode(rng),
             db_path: self.sample(rng),
             first_processed_batch: L1BatchNumber(rng.gen()),
             reset: self.sample(rng),
         }
     }
 }
 
+fn gen_fast_vm_mode<R: Rng + ?Sized>(rng: &mut R) -> FastVmMode {
+    match rng.gen_range(0..3) {
+        0 => FastVmMode::Old,
+        1 => FastVmMode::New,
+        _ => FastVmMode::Shadow,
+    }
+}
+
+impl Distribution<configs::ExperimentalVmConfig> for EncodeDist {
+    fn sample<R: Rng + ?Sized>(&self, rng: &mut R) -> configs::ExperimentalVmConfig {
+        configs::ExperimentalVmConfig {
+            playground: self.sample(rng),
+            state_keeper_fast_vm_mode: gen_fast_vm_mode(rng),
+        }
+    }
+}
+
 impl Distribution<configs::database::DBConfig> for EncodeDist {
     fn sample<R: Rng + ?Sized>(&self, rng: &mut R) -> configs::database::DBConfig {
         configs::database::DBConfig {
@@ -1066,7 +1079,6 @@ impl Distribution<configs::GeneralConfig> for EncodeDist {
             da_dispatcher_config: self.sample(rng),
             protective_reads_writer_config: self.sample(rng),
             basic_witness_input_producer_config: self.sample(rng),
-            vm_playground_config: self.sample(rng),
             commitment_generator: self.sample(rng),
             snapshot_recovery: self.sample(rng),
             pruning: self.sample(rng),
@@ -1075,6 +1087,7 @@ impl Distribution<configs::GeneralConfig> for EncodeDist {
             external_price_api_client_config: self.sample(rng),
             consensus_config: self.sample(rng),
             external_proof_integration_api_config: self.sample(rng),
+            experimental_vm_config: self.sample(rng),
         }
     }
 }
diff --git a/core/lib/env_config/src/vm_runner.rs b/core/lib/env_config/src/vm_runner.rs
@@ -1,5 +1,5 @@
 use zksync_config::configs::{
-    BasicWitnessInputProducerConfig, ExperimentalVmPlaygroundConfig, ProtectiveReadsWriterConfig,
+    BasicWitnessInputProducerConfig, ExperimentalVmConfig, ProtectiveReadsWriterConfig,
 };
 
 use crate::{envy_load, FromEnv};
@@ -16,9 +16,12 @@ impl FromEnv for BasicWitnessInputProducerConfig {
     }
 }
 
-impl FromEnv for ExperimentalVmPlaygroundConfig {
+impl FromEnv for ExperimentalVmConfig {
     fn from_env() -> anyhow::Result<Self> {
-        envy_load("vm_runner.playground", "VM_RUNNER_PLAYGROUND_")
+        Ok(Self {
+            playground: envy_load("experimental_vm.playground", "EXPERIMENTAL_VM_PLAYGROUND_")?,
+            ..envy_load("experimental_vm", "EXPERIMENTAL_VM_")?
+        })
     }
 }
 
@@ -48,36 +51,38 @@ mod tests {
     }
 
     #[test]
-    fn playground_config_from_env() {
+    fn experimental_vm_config_from_env() {
         let mut lock = MUTEX.lock();
         let config = r#"
-            VM_RUNNER_PLAYGROUND_FAST_VM_MODE=shadow
-            VM_RUNNER_PLAYGROUND_DB_PATH=/db/vm_playground
-            VM_RUNNER_PLAYGROUND_FIRST_PROCESSED_BATCH=123
-            VM_RUNNER_PLAYGROUND_RESET=true
+            EXPERIMENTAL_VM_STATE_KEEPER_FAST_VM_MODE=new
+            EXPERIMENTAL_VM_PLAYGROUND_FAST_VM_MODE=shadow
+            EXPERIMENTAL_VM_PLAYGROUND_DB_PATH=/db/vm_playground
+            EXPERIMENTAL_VM_PLAYGROUND_FIRST_PROCESSED_BATCH=123
+            EXPERIMENTAL_VM_PLAYGROUND_RESET=true
         "#;
         lock.set_env(config);
 
-        let config = ExperimentalVmPlaygroundConfig::from_env().unwrap();
-        assert_eq!(config.fast_vm_mode, FastVmMode::Shadow);
-        assert_eq!(config.db_path, "/db/vm_playground");
-        assert_eq!(config.first_processed_batch, L1BatchNumber(123));
-        assert!(config.reset);
+        let config = ExperimentalVmConfig::from_env().unwrap();
+        assert_eq!(config.state_keeper_fast_vm_mode, FastVmMode::New);
+        assert_eq!(config.playground.fast_vm_mode, FastVmMode::Shadow);
+        assert_eq!(config.playground.db_path, "/db/vm_playground");
+        assert_eq!(config.playground.first_processed_batch, L1BatchNumber(123));
+        assert!(config.playground.reset);
 
-        lock.remove_env(&["VM_RUNNER_PLAYGROUND_RESET"]);
-        let config = ExperimentalVmPlaygroundConfig::from_env().unwrap();
-        assert!(!config.reset);
+        lock.remove_env(&["EXPERIMENTAL_VM_PLAYGROUND_RESET"]);
+        let config = ExperimentalVmConfig::from_env().unwrap();
+        assert!(!config.playground.reset);
 
-        lock.remove_env(&["VM_RUNNER_PLAYGROUND_FIRST_PROCESSED_BATCH"]);
-        let config = ExperimentalVmPlaygroundConfig::from_env().unwrap();
-        assert_eq!(config.first_processed_batch, L1BatchNumber(0));
+        lock.remove_env(&["EXPERIMENTAL_VM_PLAYGROUND_FIRST_PROCESSED_BATCH"]);
+        let config = ExperimentalVmConfig::from_env().unwrap();
+        assert_eq!(config.playground.first_processed_batch, L1BatchNumber(0));
 
-        lock.remove_env(&["VM_RUNNER_PLAYGROUND_FAST_VM_MODE"]);
-        let config = ExperimentalVmPlaygroundConfig::from_env().unwrap();
-        assert_eq!(config.fast_vm_mode, FastVmMode::Old);
+        lock.remove_env(&["EXPERIMENTAL_VM_PLAYGROUND_FAST_VM_MODE"]);
+        let config = ExperimentalVmConfig::from_env().unwrap();
+        assert_eq!(config.playground.fast_vm_mode, FastVmMode::Old);
 
-        lock.remove_env(&["VM_RUNNER_PLAYGROUND_DB_PATH"]);
-        let config = ExperimentalVmPlaygroundConfig::from_env().unwrap();
-        assert!(!config.db_path.is_empty());
+        lock.remove_env(&["EXPERIMENTAL_VM_PLAYGROUND_DB_PATH"]);
+        let config = ExperimentalVmConfig::from_env().unwrap();
+        assert!(!config.playground.db_path.is_empty());
     }
 }
diff --git a/core/lib/protobuf_config/src/experimental.rs b/core/lib/protobuf_config/src/experimental.rs
@@ -5,7 +5,7 @@ use zksync_basic_types::{vm::FastVmMode, L1BatchNumber};
 use zksync_config::configs;
 use zksync_protobuf::{repr::ProtoRepr, required};
 
-use crate::proto::experimental as proto;
+use crate::{proto::experimental as proto, read_optional_repr};
 
 impl ProtoRepr for proto::Db {
     type Type = configs::ExperimentalDBConfig;
@@ -98,3 +98,28 @@ impl ProtoRepr for proto::VmPlayground {
         }
     }
 }
+
+impl ProtoRepr for proto::Vm {
+    type Type = configs::ExperimentalVmConfig;
+
+    fn read(&self) -> anyhow::Result<Self::Type> {
+        Ok(Self::Type {
+            playground: read_optional_repr(&self.playground).unwrap_or_default(),
+            state_keeper_fast_vm_mode: self
+                .state_keeper_fast_vm_mode
+                .map(proto::FastVmMode::try_from)
+                .transpose()
+                .context("fast_vm_mode")?
+                .map_or_else(FastVmMode::default, |mode| mode.parse()),
+        })
+    }
+
+    fn build(this: &Self::Type) -> Self {
+        Self {
+            playground: Some(ProtoRepr::build(&this.playground)),
+            state_keeper_fast_vm_mode: Some(
+                proto::FastVmMode::new(this.state_keeper_fast_vm_mode).into(),
+            ),
+        }
+    }
+}
diff --git a/core/lib/protobuf_config/src/general.rs b/core/lib/protobuf_config/src/general.rs
@@ -43,7 +43,7 @@ impl ProtoRepr for proto::GeneralConfig {
             external_proof_integration_api_config: read_optional_repr(
                 &self.external_proof_integration_api,
             ),
-            vm_playground_config: read_optional_repr(&self.vm_playground),
+            experimental_vm_config: read_optional_repr(&self.experimental_vm),
         })
     }
 
@@ -98,7 +98,7 @@ impl ProtoRepr for proto::GeneralConfig {
                 .external_proof_integration_api_config
                 .as_ref()
                 .map(ProtoRepr::build),
-            vm_playground: this.vm_playground_config.as_ref().map(ProtoRepr::build),
+            experimental_vm: this.experimental_vm_config.as_ref().map(ProtoRepr::build),
         }
     }
 }
diff --git a/core/lib/protobuf_config/src/proto/config/experimental.proto b/core/lib/protobuf_config/src/proto/config/experimental.proto
@@ -32,3 +32,8 @@ message VmPlayground {
   optional uint32 first_processed_batch = 3; // optional; defaults to 0
   optional bool reset = 4; // optional; defaults to false
 }
+
+message Vm {
+  optional VmPlayground playground = 1; // optional
+  optional FastVmMode state_keeper_fast_vm_mode = 2; // optional; if not set, fast VM is not used
+}
diff --git a/core/lib/protobuf_config/src/proto/config/general.proto b/core/lib/protobuf_config/src/proto/config/general.proto
@@ -57,5 +57,5 @@ message GeneralConfig {
     optional external_price_api_client.ExternalPriceApiClient external_price_api_client = 41;
     optional core.consensus.Config consensus = 42;
     optional external_proof_integration_api.ExternalProofIntegrationApi external_proof_integration_api = 43;
-    optional experimental.VmPlayground vm_playground = 44;
+    optional experimental.Vm experimental_vm = 44;
 }