fix: ignore unknown fields in rpc json response (#2962)

In some previous pr, I've added a field to json RPC response (consensusGlobalConfig) and it turned out to be incompatible due to the default parsing behavior: an error was returned when unknown fields were present. In general, for programmatic communications the recommended behaviour is to silently ignore unknown fields, so that adding new fields is compatible. I've also weakened the restriction on accepting new consensus genesis by external nodes - when the unknown fields are silently dropped, it is possible that the node will perceive genesis with the same fork number differently after a binary update (because it receives more fields).
matter-labs · Sep 26, 2024 · 692ea73 · 692ea73
1 parent c3d4bc1
commit 692ea73
Show file tree

Hide file tree

Showing 19 changed files with 380 additions and 327 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -230,16 +230,16 @@ zk_evm_1_5_0 = { package = "zk_evm", version = "=0.150.5" }
 zksync_vm2 = { git = "https://github.com/matter-labs/vm2.git", rev = "74577d9be13b1bff9d1a712389731f669b179e47" }
 
 # Consensus dependencies.
-zksync_concurrency = "=0.1.1"
-zksync_consensus_bft = "=0.1.1"
-zksync_consensus_crypto = "=0.1.1"
-zksync_consensus_executor = "=0.1.1"
-zksync_consensus_network = "=0.1.1"
-zksync_consensus_roles = "=0.1.1"
-zksync_consensus_storage = "=0.1.1"
-zksync_consensus_utils = "=0.1.1"
-zksync_protobuf = "=0.1.1"
-zksync_protobuf_build = "=0.1.1"
+zksync_concurrency = "=0.3.0"
+zksync_consensus_bft = "=0.3.0"
+zksync_consensus_crypto = "=0.3.0"
+zksync_consensus_executor = "=0.3.0"
+zksync_consensus_network = "=0.3.0"
+zksync_consensus_roles = "=0.3.0"
+zksync_consensus_storage = "=0.3.0"
+zksync_consensus_utils = "=0.3.0"
+zksync_protobuf = "=0.3.0"
+zksync_protobuf_build = "=0.3.0"
 
 # "Local" dependencies
 zksync_multivm = { version = "0.1.0", path = "core/lib/multivm" }

diff --git a/core/bin/block_reverter/src/main.rs b/core/bin/block_reverter/src/main.rs
@@ -20,7 +20,7 @@ use zksync_config::{
     },
     ContractsConfig, DBConfig, EthConfig, GenesisConfig, PostgresConfig,
 };
-use zksync_core_leftovers::temp_config_store::decode_yaml_repr;
+use zksync_core_leftovers::temp_config_store::read_yaml_repr;
 use zksync_dal::{ConnectionPool, Core};
 use zksync_env_config::{object_store::SnapshotsObjectStoreConfig, FromEnv};
 use zksync_object_store::ObjectStoreFactory;
@@ -127,27 +127,26 @@ async fn main() -> anyhow::Result<()> {
         .build();
 
     let general_config: Option<GeneralConfig> = if let Some(path) = opts.config_path {
-        let yaml = std::fs::read_to_string(&path).with_context(|| path.display().to_string())?;
-        let config =
-            decode_yaml_repr::<zksync_protobuf_config::proto::general::GeneralConfig>(&yaml)
-                .context("failed decoding general YAML config")?;
-        Some(config)
+        Some(
+            read_yaml_repr::<zksync_protobuf_config::proto::general::GeneralConfig>(&path)
+                .context("failed decoding general YAML config")?,
+        )
     } else {
         None
     };
     let wallets_config: Option<Wallets> = if let Some(path) = opts.wallets_path {
-        let yaml = std::fs::read_to_string(&path).with_context(|| path.display().to_string())?;
-        let config = decode_yaml_repr::<zksync_protobuf_config::proto::wallets::Wallets>(&yaml)
-            .context("failed decoding wallets YAML config")?;
-        Some(config)
+        Some(
+            read_yaml_repr::<zksync_protobuf_config::proto::wallets::Wallets>(&path)
+                .context("failed decoding wallets YAML config")?,
+        )
     } else {
         None
     };
     let genesis_config: Option<GenesisConfig> = if let Some(path) = opts.genesis_path {
-        let yaml = std::fs::read_to_string(&path).with_context(|| path.display().to_string())?;
-        let config = decode_yaml_repr::<zksync_protobuf_config::proto::genesis::Genesis>(&yaml)
-            .context("failed decoding genesis YAML config")?;
-        Some(config)
+        Some(
+            read_yaml_repr::<zksync_protobuf_config::proto::genesis::Genesis>(&path)
+                .context("failed decoding genesis YAML config")?,
+        )
     } else {
         None
     };
@@ -183,19 +182,15 @@ async fn main() -> anyhow::Result<()> {
             .context("BasicWitnessInputProducerConfig::from_env()")?,
     };
     let contracts = match opts.contracts_config_path {
-        Some(path) => {
-            let yaml =
-                std::fs::read_to_string(&path).with_context(|| path.display().to_string())?;
-            decode_yaml_repr::<zksync_protobuf_config::proto::contracts::Contracts>(&yaml)
-                .context("failed decoding contracts YAML config")?
-        }
+        Some(path) => read_yaml_repr::<zksync_protobuf_config::proto::contracts::Contracts>(&path)
+            .context("failed decoding contracts YAML config")?,
         None => ContractsConfig::from_env().context("ContractsConfig::from_env()")?,
     };
     let secrets_config = if let Some(path) = opts.secrets_path {
-        let yaml = std::fs::read_to_string(&path).with_context(|| path.display().to_string())?;
-        let config = decode_yaml_repr::<zksync_protobuf_config::proto::secrets::Secrets>(&yaml)
-            .context("failed decoding secrets YAML config")?;
-        Some(config)
+        Some(
+            read_yaml_repr::<zksync_protobuf_config::proto::secrets::Secrets>(&path)
+                .context("failed decoding secrets YAML config")?,
+        )
     } else {
         None
     };

diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs
@@ -19,7 +19,7 @@ use zksync_config::{
 };
 use zksync_consensus_crypto::TextFmt;
 use zksync_consensus_roles as roles;
-use zksync_core_leftovers::temp_config_store::{decode_yaml_repr, read_yaml_repr};
+use zksync_core_leftovers::temp_config_store::read_yaml_repr;
 #[cfg(test)]
 use zksync_dal::{ConnectionPool, Core};
 use zksync_metadata_calculator::MetadataCalculatorRecoveryConfig;
@@ -1149,9 +1149,8 @@ pub(crate) fn read_consensus_secrets() -> anyhow::Result<Option<ConsensusSecrets
     let Ok(path) = env::var("EN_CONSENSUS_SECRETS_PATH") else {
         return Ok(None);
     };
-    let cfg = std::fs::read_to_string(&path).context(path)?;
     Ok(Some(
-        decode_yaml_repr::<proto::secrets::ConsensusSecrets>(&cfg)
+        read_yaml_repr::<proto::secrets::ConsensusSecrets>(&path.into())
             .context("failed decoding YAML")?,
     ))
 }
@@ -1160,9 +1159,8 @@ pub(crate) fn read_consensus_config() -> anyhow::Result<Option<ConsensusConfig>>
     let Ok(path) = env::var("EN_CONSENSUS_CONFIG_PATH") else {
         return Ok(None);
     };
-    let cfg = std::fs::read_to_string(&path).context(path)?;
     Ok(Some(
-        decode_yaml_repr::<proto::consensus::Config>(&cfg).context("failed decoding YAML")?,
+        read_yaml_repr::<proto::consensus::Config>(&path.into()).context("failed decoding YAML")?,
     ))
 }
 
@@ -1253,16 +1251,16 @@ impl ExternalNodeConfig<()> {
         secrets_configs_path: PathBuf,
         consensus_config_path: Option<PathBuf>,
     ) -> anyhow::Result<Self> {
-        let general_config = read_yaml_repr::<proto::general::GeneralConfig>(general_config_path)
+        let general_config = read_yaml_repr::<proto::general::GeneralConfig>(&general_config_path)
             .context("failed decoding general YAML config")?;
         let external_node_config =
-            read_yaml_repr::<proto::en::ExternalNode>(external_node_config_path)
+            read_yaml_repr::<proto::en::ExternalNode>(&external_node_config_path)
                 .context("failed decoding external node YAML config")?;
-        let secrets_config = read_yaml_repr::<proto::secrets::Secrets>(secrets_configs_path)
+        let secrets_config = read_yaml_repr::<proto::secrets::Secrets>(&secrets_configs_path)
             .context("failed decoding secrets YAML config")?;
 
         let consensus = consensus_config_path
-            .map(read_yaml_repr::<proto::consensus::Config>)
+            .map(|path| read_yaml_repr::<proto::consensus::Config>(&path))
             .transpose()
             .context("failed decoding consensus YAML config")?;
         let consensus_secrets = secrets_config.consensus.clone();

diff --git a/core/bin/genesis_generator/src/main.rs b/core/bin/genesis_generator/src/main.rs
@@ -9,7 +9,7 @@ use clap::Parser;
 use serde_yaml::Serializer;
 use zksync_config::{configs::DatabaseSecrets, GenesisConfig};
 use zksync_contracts::BaseSystemContracts;
-use zksync_core_leftovers::temp_config_store::decode_yaml_repr;
+use zksync_core_leftovers::temp_config_store::read_yaml_repr;
 use zksync_dal::{ConnectionPool, Core, CoreDal};
 use zksync_env_config::FromEnv;
 use zksync_node_genesis::{insert_genesis_batch, GenesisParams};
@@ -40,17 +40,13 @@ async fn main() -> anyhow::Result<()> {
     let database_secrets = match opt.config_path {
         None => DatabaseSecrets::from_env()?,
         Some(path) => {
-            let yaml =
-                std::fs::read_to_string(&path).with_context(|| path.display().to_string())?;
-            let config = decode_yaml_repr::<zksync_protobuf_config::proto::secrets::Secrets>(&yaml)
-                .context("failed decoding general YAML config")?;
+            let config = read_yaml_repr::<zksync_protobuf_config::proto::secrets::Secrets>(&path)
+                .context("failed decoding secrets YAML")?;
             config.database.context("Database secrets must exist")?
         }
     };
 
-    let yaml = std::fs::read_to_string(DEFAULT_GENESIS_FILE_PATH)
-        .with_context(|| DEFAULT_GENESIS_FILE_PATH.to_string())?;
-    let original_genesis = decode_yaml_repr::<Genesis>(&yaml)?;
+    let original_genesis = read_yaml_repr::<Genesis>(&DEFAULT_GENESIS_FILE_PATH.into())?;
     let db_url = database_secrets.master_url()?;
     let new_genesis = generate_new_config(db_url, original_genesis.clone()).await?;
     if opt.check {

diff --git a/core/bin/zksync_server/src/config.rs b/core/bin/zksync_server/src/config.rs
@@ -1,16 +1,15 @@
 use anyhow::Context as _;
 use zksync_config::configs::consensus::{ConsensusConfig, ConsensusSecrets};
-use zksync_core_leftovers::temp_config_store::decode_yaml_repr;
+use zksync_core_leftovers::temp_config_store::read_yaml_repr;
 use zksync_protobuf_config::proto;
 
 pub(crate) fn read_consensus_secrets() -> anyhow::Result<Option<ConsensusSecrets>> {
     // Read public config.
     let Ok(path) = std::env::var("CONSENSUS_SECRETS_PATH") else {
         return Ok(None);
     };
-    let secrets = std::fs::read_to_string(&path).context(path)?;
     Ok(Some(
-        decode_yaml_repr::<proto::secrets::ConsensusSecrets>(&secrets)
+        read_yaml_repr::<proto::secrets::ConsensusSecrets>(&path.into())
             .context("failed decoding YAML")?,
     ))
 }
@@ -20,8 +19,7 @@ pub(crate) fn read_consensus_config() -> anyhow::Result<Option<ConsensusConfig>>
     let Ok(path) = std::env::var("CONSENSUS_CONFIG_PATH") else {
         return Ok(None);
     };
-    let cfg = std::fs::read_to_string(&path).context(path)?;
     Ok(Some(
-        decode_yaml_repr::<proto::consensus::Config>(&cfg).context("failed decoding YAML")?,
+        read_yaml_repr::<proto::consensus::Config>(&path.into()).context("failed decoding YAML")?,
     ))
 }