feat(prover): file based configs for witness generator

prover/witness_generator/Cargo.toml

@@ -28,6 +28,8 @@ zksync_utils.workspace = true
 vk_setup_data_generator_server_fri.workspace = true
 zksync_prover_fri_types.workspace = true
 zksync_prover_fri_utils.workspace = true
+zksync_core_leftovers.workspace = true
+zksync_protobuf_config.workspace = true
 
 zkevm_test_harness = { workspace = true }
 circuit_definitions = { workspace = true, features = [ "log_tracing" ] }

prover/witness_generator/src/main.rs

@@ -9,14 +9,13 @@ use prover_dal::{ConnectionPool, Prover, ProverDal};
 use structopt::StructOpt;
 use tokio::sync::watch;
 use zksync_config::{
-    configs::{
-        DatabaseSecrets, FriWitnessGeneratorConfig, ObservabilityConfig, PostgresConfig,
-        PrometheusConfig,
-    },
+    configs::{DatabaseSecrets, FriWitnessGeneratorConfig, PostgresConfig, PrometheusConfig},
     ObjectStoreConfig,
 };
+use zksync_core_leftovers::temp_config_store::{decode_yaml_repr, TempConfigStore};
 use zksync_env_config::{object_store::ProverObjectStoreConfig, FromEnv};
 use zksync_object_store::ObjectStoreFactory;
+use zksync_protobuf_config::proto::secrets::Secrets;
 use zksync_queued_job_processor::JobProcessor;
 use zksync_types::basic_fri_types::AggregationRound;
 use zksync_utils::wait_for_tasks::ManagedTasks;
@@ -64,12 +63,41 @@ struct Opt {
     /// Start all aggregation rounds for the witness generator.
     #[structopt(short = "a", long = "all_rounds")]
     all_rounds: bool,
+    /// Path to the configuration file.
+    #[structopt(long)]
+    config_path: Option<std::path::PathBuf>,
+    /// Path to the secrets file.
+    #[structopt(long)]
+    secrets_path: Option<std::path::PathBuf>,
 }
 
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
-    let observability_config =
-        ObservabilityConfig::from_env().context("ObservabilityConfig::from_env()")?;
+    let opt = Opt::from_args();
+
+    let general_config = match opt.config_path {
+        Some(path) => {
+            let yaml = std::fs::read_to_string(path).context("Failed to read general config")?;
+            decode_yaml_repr::<zksync_protobuf_config::proto::general::GeneralConfig>(&yaml)
+                .context("Failed to parse general config")?
+        }
+        None => load_env_config()?.general(),
+    };
+
+    let database_secrets = match opt.secrets_path {
+        Some(path) => {
+            let yaml = std::fs::read_to_string(path).context("Failed to read secrets")?;
+            let secrets = decode_yaml_repr::<Secrets>(&yaml).context("Failed to parse secrets")?;
+            secrets
+                .database
+                .context("failed to parse database secrets")?
+        }
+        None => DatabaseSecrets::from_env().context("database secrets")?,
+    };
+
+    let observability_config = general_config
+        .observability
+        .context("observability config")?;
     let log_format: vlog::LogFormat = observability_config
         .log_format
         .parse()
@@ -100,18 +128,24 @@ async fn main() -> anyhow::Result<()> {
         tracing::info!("No sentry URL was provided");
     }
 
-    let opt = Opt::from_args();
     let started_at = Instant::now();
     let use_push_gateway = opt.batch_size.is_some();
 
-    let object_store_config =
-        ProverObjectStoreConfig::from_env().context("ProverObjectStoreConfig::from_env()")?;
+    let object_store_config = ProverObjectStoreConfig(
+        general_config
+            .prover_config
+            .context("prover config")?
+            .object_store
+            .context("object store")?,
+    );
     let store_factory = ObjectStoreFactory::new(object_store_config.0);
-    let config =
-        FriWitnessGeneratorConfig::from_env().context("FriWitnessGeneratorConfig::from_env()")?;
-    let prometheus_config = PrometheusConfig::from_env().context("PrometheusConfig::from_env()")?;
-    let postgres_config = PostgresConfig::from_env().context("PostgresConfig::from_env()")?;
-    let database_secrets = DatabaseSecrets::from_env().context("DatabaseSecrets::from_env()")?;
+    let config = general_config
+        .witness_generator
+        .context("witness generator config")?;
+    let prometheus_config = general_config
+        .prometheus_config
+        .context("prometheus config")?;
+    let postgres_config = general_config.postgres_config.context("postgres config")?;
     let connection_pool = ConnectionPool::<Core>::builder(
         database_secrets.master_url()?,
         postgres_config.max_connections()?,
@@ -283,3 +317,12 @@ async fn main() -> anyhow::Result<()> {
     tracing::info!("Finished witness generation");
     Ok(())
 }
+
+fn load_env_config() -> anyhow::Result<TempConfigStore> {
+    Ok(TempConfigStore {
+        postgres_config: PostgresConfig::from_env().ok(),
+        fri_witness_generator_config: FriWitnessGeneratorConfig::from_env().ok(),
+        prometheus_config: PrometheusConfig::from_env().ok(),
+        ..Default::default()
+    })
+}