diff --git a/testsuite/forge-cli/src/main.rs b/testsuite/forge-cli/src/main.rs index 5ed4df1edc7d13..ec95b6fa1b2c68 100644 --- a/testsuite/forge-cli/src/main.rs +++ b/testsuite/forge-cli/src/main.rs @@ -432,16 +432,12 @@ fn main() -> Result<()> { OperatorCommand::Create(create) => { let kube_client = runtime.block_on(create_k8s_client())?; let era = generate_new_era(); - let values = ForgeDeployerValues { - profile: DEFAULT_FORGE_DEPLOYER_PROFILE.to_string(), + let config = ForgeDeployerConfig::new( + DEFAULT_FORGE_DEPLOYER_PROFILE.to_string(), era, - namespace: create.namespace, - indexer_grpc_values: None, - indexer_processor_values: None, - }; - let forge_deployer_manager = - ForgeDeployerManager::from_k8s_client(kube_client, values); - runtime.block_on(forge_deployer_manager.ensure_namespace_prepared())?; + create.namespace, + ); + let forge_deployer_manager = ForgeDeployerManager::new(kube_client, config); // NOTE: this is generally not going to run from within the cluster, do not perform any operations // that might require internal DNS resolution to work, such as txn emission directly against the node service IPs. runtime.block_on(forge_deployer_manager.start(ForgeDeployerType::Testnet))?; diff --git a/testsuite/forge/src/backend/k8s/mod.rs b/testsuite/forge/src/backend/k8s/mod.rs index b2496b89d14426..45aea1ba0eaa98 100644 --- a/testsuite/forge/src/backend/k8s/mod.rs +++ b/testsuite/forge/src/backend/k8s/mod.rs @@ -20,7 +20,7 @@ mod stateful_set; mod swarm; use super::{ - ForgeDeployerManager, ForgeDeployerType, ForgeDeployerValues, DEFAULT_FORGE_DEPLOYER_PROFILE, + ForgeDeployerConfig, ForgeDeployerManager, ForgeDeployerType, DEFAULT_FORGE_DEPLOYER_PROFILE, }; use aptos_sdk::crypto::ed25519::ED25519_PRIVATE_KEY_LENGTH; pub use cluster_helper::*; @@ -185,18 +185,14 @@ impl Factory for K8sFactory { // add an indexer too! if self.enable_indexer { // NOTE: by default, use a deploy profile and no additional configuration values - let values = ForgeDeployerValues { - profile: DEFAULT_FORGE_DEPLOYER_PROFILE.to_string(), - era: new_era.clone().expect("Era not set in created testnet"), - namespace: self.kube_namespace.clone(), - indexer_grpc_values: None, - indexer_processor_values: None, - }; - - let forge_deployer_manager = - ForgeDeployerManager::from_k8s_client(kube_client.clone(), values); - - forge_deployer_manager.ensure_namespace_prepared().await?; + let config = ForgeDeployerConfig::new( + DEFAULT_FORGE_DEPLOYER_PROFILE.to_string(), + new_era.clone().expect("Era not set in created testnet"), + self.kube_namespace.clone(), + ); + + let forge_deployer_manager = ForgeDeployerManager::new(kube_client.clone(), config); + forge_deployer_manager .start(ForgeDeployerType::Indexer) .await?; diff --git a/testsuite/forge/src/backend/k8s_deployer/deployer.rs b/testsuite/forge/src/backend/k8s_deployer/deployer.rs index 0be557cf6d8a2a..3c934fd8648a86 100644 --- a/testsuite/forge/src/backend/k8s_deployer/deployer.rs +++ b/testsuite/forge/src/backend/k8s_deployer/deployer.rs @@ -19,32 +19,39 @@ use kube::{ use serde::{Deserialize, Serialize}; use std::{collections::BTreeMap, fmt, sync::Arc}; -/// These are the values that the forge deployer needs to deploy the forge components to the k8s cluster. -/// There are global values such as profile, era, and namespace as well as component-specific values +/// This is the configuration that is consumed by a Forge Deployer to deploy resources to a k8s cluster. +/// There are global values such as profile, era, and namespace as well as component-specific values. +/// This entire struct is serialized to a JSON string and stored in a ConfigMap that is consumed by the deployer. For reference, see https://github.com/aptos-labs/internal-ops/blob/main/infra/cli/commands/forge/types.ts +/// Depending on the Deployer, only a subset of the component-specific values will be used. #[derive(Serialize, Deserialize, Clone, Debug)] -pub struct ForgeDeployerValues { +pub struct ForgeDeployerConfig { pub profile: String, pub era: String, pub namespace: String, - // component specific values - // TODO: add an options reference. Ideally this customization is almost always optional and instead handled by the profiles + #[serde(rename = "indexer-grpc-values")] pub indexer_grpc_values: Option, + #[serde(rename = "indexer-processor-values")] pub indexer_processor_values: Option, + #[serde(rename = "testnet-values")] + pub testnet_values: Option, + #[serde(rename = "genesis-values")] + pub genesis_values: Option, } -/// The ForgeDeployerManager is responsible for managing the lifecycle of forge deployers, wihch deploy the -/// forge components to the k8s cluster. -pub struct ForgeDeployerManager { - // all the k8s APIs we need. Specifying each API separately allows for easier testing - pub jobs_api: Arc>, - pub config_maps_api: Arc>, - pub namespace_api: Arc>, - pub serviceaccount_api: Arc>, - pub rolebinding_api: Arc>, - - // the values to use for the deployer, including namespace, era, etc - pub values: ForgeDeployerValues, +impl ForgeDeployerConfig { + /// Create a new ForgeDeployerValues with all the required values + pub fn new(profile: String, era: String, namespace: String) -> Self { + Self { + profile, + era, + namespace, + indexer_grpc_values: None, + indexer_processor_values: None, + testnet_values: None, + genesis_values: None, + } + } } #[derive(Clone, Copy)] @@ -62,24 +69,38 @@ impl fmt::Display for ForgeDeployerType { } } +/// The ForgeDeployerManager is responsible for managing the lifecycle of forge deployers, which deploy the +/// forge components to the k8s cluster. +pub struct ForgeDeployerManager { + // all the k8s APIs we need. Specifying each API separately allows for easier testing + pub jobs_api: Arc>, + pub config_maps_api: Arc>, + pub namespace_api: Arc>, + pub serviceaccount_api: Arc>, + pub rolebinding_api: Arc>, + + // the values to use for the deployer, including namespace, era, etc + pub config: ForgeDeployerConfig, +} + impl ForgeDeployerManager { - pub fn from_k8s_client(kube_client: kube::Client, values: ForgeDeployerValues) -> Self { + pub fn new(kube_client: kube::Client, config: ForgeDeployerConfig) -> Self { let jobs_api = Arc::new(K8sApi::from_client( kube_client.clone(), - Some(values.namespace.clone()), + Some(config.namespace.clone()), )); let config_maps_api = Arc::new(K8sApi::from_client( kube_client.clone(), - Some(values.namespace.clone()), + Some(config.namespace.clone()), )); let namespace_api = Arc::new(K8sApi::from_client(kube_client.clone(), None)); let serviceaccount_api = Arc::new(K8sApi::from_client( kube_client.clone(), - Some(values.namespace.clone()), + Some(config.namespace.clone()), )); let rolebinding_api = Arc::new(K8sApi::from_client( kube_client.clone(), - Some(values.namespace.clone()), + Some(config.namespace.clone()), )); // ensure it lives long enough between async @@ -89,30 +110,30 @@ impl ForgeDeployerManager { namespace_api, serviceaccount_api, rolebinding_api, - values, + config, } } /// Given a deployer type return the name to use for k8s components /// This is the canonical name for the deployer and each of its components pub(crate) fn get_name(&self, deployer_type: ForgeDeployerType) -> String { - format!("deploy-forge-{}-e{}", deployer_type, &self.values.era) + format!("deploy-forge-{}-e{}", deployer_type, &self.config.era) } - /// Gets a k8s configmap for the forge deployer that contains the values needed to deploy the forge components + /// Builds a k8s configmap for the forge deployer that contains the values needed to deploy the forge components /// Does not actually create the configmap in k8s - fn get_forge_deployer_k8s_config_map( + fn build_forge_deployer_k8s_config_map( &self, deployer_type: ForgeDeployerType, ) -> Result { let configmap_name = self.get_name(deployer_type); - let deploy_values_json = serde_json::to_string(&self.values)?; + let deploy_values_json = serde_json::to_string(&self.config)?; // create the configmap with values let config_map = ConfigMap { metadata: ObjectMeta { name: Some(configmap_name.clone()), - namespace: Some(self.values.namespace.clone()), + namespace: Some(self.config.namespace.clone()), ..Default::default() }, data: Some(BTreeMap::from([( @@ -125,11 +146,11 @@ impl ForgeDeployerManager { Ok(config_map) } - /// Gets a k8s job for the forge deployer that implements the particular interface that it expects: + /// Builds a k8s job for the forge deployer that implements the particular interface that it expects: /// - Runs the corresponding forge--deployer image /// - Sets the FORGE_DEPLOY_VALUES_JSON environment variable to the configmap that contains the values /// Does not actually create the job in k8s - fn get_forge_deployer_k8s_job( + fn build_forge_deployer_k8s_job( &self, deployer_type: ForgeDeployerType, configmap_name: String, @@ -144,7 +165,7 @@ impl ForgeDeployerManager { let job = Job { metadata: ObjectMeta { name: Some(job_name.clone()), - namespace: Some(self.values.namespace.clone()), + namespace: Some(self.config.namespace.clone()), ..Default::default() }, spec: Some(k8s_openapi::api::batch::v1::JobSpec { @@ -185,8 +206,9 @@ impl ForgeDeployerManager { } pub async fn start(&self, deployer_type: ForgeDeployerType) -> Result<()> { - let config_map = self.get_forge_deployer_k8s_config_map(deployer_type)?; - let job = self.get_forge_deployer_k8s_job(deployer_type, config_map.name())?; + self.ensure_namespace_prepared().await?; + let config_map = self.build_forge_deployer_k8s_config_map(deployer_type)?; + let job = self.build_forge_deployer_k8s_job(deployer_type, config_map.name())?; self.config_maps_api .create(&PostParams::default(), &config_map) .await?; @@ -194,32 +216,32 @@ impl ForgeDeployerManager { Ok(()) } - pub async fn ensure_namespace_prepared(&self) -> Result<()> { - let namespace = Namespace { + fn build_namespace(&self) -> Namespace { + Namespace { metadata: ObjectMeta { - name: Some(self.values.namespace.clone()), + name: Some(self.config.namespace.clone()), ..Default::default() }, ..Default::default() - }; - maybe_create_k8s_resource(self.namespace_api.clone(), namespace.clone()).await?; + } + } - // create a serviceaccount FORGE_DEPLOYER_SERVICE_ACCOUNT_NAME - let service_account = ServiceAccount { + fn build_service_account(&self) -> ServiceAccount { + ServiceAccount { metadata: ObjectMeta { name: Some(FORGE_DEPLOYER_SERVICE_ACCOUNT_NAME.to_string()), - namespace: Some(namespace.name()), + namespace: Some(self.config.namespace.clone()), ..Default::default() }, ..Default::default() - }; - maybe_create_k8s_resource(self.serviceaccount_api.clone(), service_account).await?; + } + } - // create a rolebinding for the service account to the clusterrole cluster-admin - let role_binding = RoleBinding { + fn build_role_binding(&self) -> RoleBinding { + RoleBinding { metadata: ObjectMeta { name: Some("forge-admin".to_string()), - namespace: Some(namespace.name()), + namespace: Some(self.config.namespace.clone()), ..Default::default() }, role_ref: k8s_openapi::api::rbac::v1::RoleRef { @@ -230,10 +252,18 @@ impl ForgeDeployerManager { subjects: Some(vec![k8s_openapi::api::rbac::v1::Subject { kind: "ServiceAccount".to_string(), name: FORGE_DEPLOYER_SERVICE_ACCOUNT_NAME.to_string(), - namespace: Some(namespace.name()), + namespace: Some(self.config.namespace.clone()), ..Default::default() }]), - }; + } + } + + async fn ensure_namespace_prepared(&self) -> Result<()> { + let namespace = self.build_namespace(); + maybe_create_k8s_resource(self.namespace_api.clone(), namespace.clone()).await?; + let service_account = self.build_service_account(); + maybe_create_k8s_resource(self.serviceaccount_api.clone(), service_account).await?; + let role_binding = self.build_role_binding(); maybe_create_k8s_resource(self.rolebinding_api.clone(), role_binding).await?; Ok(()) } @@ -259,20 +289,18 @@ mod tests { /// exists in the namespace yet #[tokio::test] async fn test_start_deployer_fresh_environment() { - let values = ForgeDeployerValues { - profile: "large-banana".to_string(), - era: "1".to_string(), - namespace: "forge-large-banana".to_string(), - indexer_grpc_values: None, - indexer_processor_values: None, - }; + let config = ForgeDeployerConfig::new( + "large-banana".to_string(), + "1".to_string(), + "forge-large-banana".to_string(), + ); let manager = ForgeDeployerManager { jobs_api: Arc::new(MockK8sResourceApi::new()), config_maps_api: Arc::new(MockK8sResourceApi::new()), namespace_api: Arc::new(MockK8sResourceApi::new()), serviceaccount_api: Arc::new(MockK8sResourceApi::new()), rolebinding_api: Arc::new(MockK8sResourceApi::new()), - values, + config, }; manager.start(ForgeDeployerType::Indexer).await.unwrap(); let indexer_deployer_name = manager.get_name(ForgeDeployerType::Indexer); @@ -292,13 +320,11 @@ mod tests { /// and we cannot override/mutate it. #[tokio::test] async fn test_start_deployer_existing_job() { - let values = ForgeDeployerValues { - profile: "large-banana".to_string(), - era: "1".to_string(), - namespace: "forge-large-banana".to_string(), - indexer_grpc_values: None, - indexer_processor_values: None, - }; + let config = ForgeDeployerConfig::new( + "large-banana".to_string(), + "1".to_string(), + "forge-large-banana".to_string(), + ); let manager = ForgeDeployerManager { jobs_api: Arc::new(MockK8sResourceApi::from_resource(Job { metadata: ObjectMeta { @@ -312,7 +338,7 @@ mod tests { namespace_api: Arc::new(MockK8sResourceApi::new()), serviceaccount_api: Arc::new(MockK8sResourceApi::new()), rolebinding_api: Arc::new(MockK8sResourceApi::new()), - values, + config, }; let result = manager.start(ForgeDeployerType::Indexer).await; assert!(result.is_err()); @@ -322,13 +348,11 @@ mod tests { /// as the new job/deployment will be in a different era and unrelated to the existing job #[tokio::test] async fn test_start_deployer_existing_job_different_era() { - let values = ForgeDeployerValues { - profile: "large-banana".to_string(), - era: "2".to_string(), - namespace: "forge-large-banana".to_string(), - indexer_grpc_values: None, - indexer_processor_values: None, - }; + let config = ForgeDeployerConfig::new( + "large-banana".to_string(), + "2".to_string(), + "forge-large-banana".to_string(), + ); let manager = ForgeDeployerManager { jobs_api: Arc::new(MockK8sResourceApi::from_resource(Job { metadata: ObjectMeta { @@ -342,7 +366,7 @@ mod tests { namespace_api: Arc::new(MockK8sResourceApi::new()), serviceaccount_api: Arc::new(MockK8sResourceApi::new()), rolebinding_api: Arc::new(MockK8sResourceApi::new()), - values, + config, }; manager.start(ForgeDeployerType::Indexer).await.unwrap(); } @@ -351,20 +375,18 @@ mod tests { /// Collisions should be OK to ensure idempotency #[tokio::test] async fn test_ensure_namespace_prepared_fresh_namespace() { - let values = ForgeDeployerValues { - profile: "large-banana".to_string(), - era: "1".to_string(), - namespace: "forge-large-banana".to_string(), - indexer_grpc_values: None, - indexer_processor_values: None, - }; + let config = ForgeDeployerConfig::new( + "large-banana".to_string(), + "1".to_string(), + "forge-large-banana".to_string(), + ); let manager = ForgeDeployerManager { jobs_api: Arc::new(MockK8sResourceApi::new()), config_maps_api: Arc::new(MockK8sResourceApi::new()), namespace_api: Arc::new(MockK8sResourceApi::new()), serviceaccount_api: Arc::new(MockK8sResourceApi::new()), rolebinding_api: Arc::new(MockK8sResourceApi::new()), - values, + config, }; manager .ensure_namespace_prepared() @@ -401,13 +423,11 @@ mod tests { /// Test the same thing but with existing resources. This should not error out and should be idempotent #[tokio::test] async fn test_ensure_namespace_prepared_existing_resources() { - let values = ForgeDeployerValues { - profile: "large-banana".to_string(), - era: "1".to_string(), - namespace: "forge-large-banana".to_string(), - indexer_grpc_values: None, - indexer_processor_values: None, - }; + let config = ForgeDeployerConfig::new( + "large-banana".to_string(), + "1".to_string(), + "forge-large-banana".to_string(), + ); let manager = ForgeDeployerManager { jobs_api: Arc::new(MockK8sResourceApi::new()), config_maps_api: Arc::new(MockK8sResourceApi::new()), @@ -434,7 +454,7 @@ mod tests { }, ..Default::default() })), - values, + config, }; manager .ensure_namespace_prepared()