Move Gateway configuration out of Sled Agent, into RSS

docs/how-to-run.adoc

@@ -78,7 +78,7 @@ Transformation Engine (OPTE, the kernel module which provides the private
 virtual networking to guests). To make this work, OPTE needs to know about the
 local networking configuration, in this case the MAC address of the local
 internet gateway. To be able to get into your instances, you _must_ specify this
-in the `gateway_mac` field of the config file `smf/sled-agent/config.toml`.
+in the `gateway.mac` field of the config file `smf/sled-agent/config-rss.toml`.
 
 The value there is correct for the lab environment, so if you're running there,
 no changes are needed. If you're running elsewhere, you find the value with:
@@ -174,7 +174,7 @@ be set as a default route for the Nexus zone.
 | Crucible Downstairs 3      | `[fd00:1122:3344:0101::8]:32345`
 | Internal DNS Service       | `[fd00:1122:3344:0001::1]:5353`
 | Nexus: External API        | `192.168.1.20:80`
-| Internet Gateway           | None, but can be set in `smf/sled-agent/config.toml`
+| Internet Gateway           | None, but can be set in `smf/sled-agent/config-rss.toml`
 |===================================================================================================
 
 Note that Sled Agent runs in the global zone and is the one responsible for bringing up all the other

sled-agent/src/bootstrap/agent.rs

@@ -277,7 +277,7 @@ impl Agent {
             self.parent_log.clone(),
             sled_address,
             is_scrimlet,
-            request.rack_id,
+            request.clone(),
         )
         .await
         .map_err(|e| {
@@ -553,8 +553,12 @@ mod tests {
         let request = PersistentSledAgentRequest {
             request: Cow::Owned(SledAgentRequest {
                 id: Uuid::new_v4(),
-                subnet: Ipv6Subnet::new(Ipv6Addr::LOCALHOST),
                 rack_id: Uuid::new_v4(),
+                gateway: crate::bootstrap::params::Gateway {
+                    address: None,
+                    mac: MacAddr6::nil(),
+                },
+                subnet: Ipv6Subnet::new(Ipv6Addr::LOCALHOST),
             }),
             trust_quorum_share: Some(
                 ShareDistribution {

sled-agent/src/bootstrap/params.rs

@@ -5,11 +5,35 @@
 //! Request types for the bootstrap agent
 
 use super::trust_quorum::SerializableShareDistribution;
+use macaddr::MacAddr6;
 use omicron_common::address::{Ipv6Subnet, SLED_PREFIX};
 use serde::{Deserialize, Serialize};
+use serde_with::serde_as;
+use serde_with::DisplayFromStr;
+use serde_with::PickFirst;
 use std::borrow::Cow;
+use std::net::Ipv4Addr;
 use uuid::Uuid;
 
+/// Information about the internet gateway used for externally-facing services.
+#[serde_as]
+#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
+pub struct Gateway {
+    /// IP address of the Internet gateway, which is particularly
+    /// relevant for external-facing services (such as Nexus).
+    pub address: Option<Ipv4Addr>,
+
+    /// MAC address of the internet gateway above. This is used to provide
+    /// external connectivity into guests, by allowing OPTE to forward traffic
+    /// destined for the broader network to the gateway.
+    // This uses the `serde_with` crate's `serde_as` attribute, which tries
+    // each of the listed serialization types (starting with the default) until
+    // one succeeds. This supports deserialization from either an array of u8,
+    // or the display-string representation.
+    #[serde_as(as = "PickFirst<(_, DisplayFromStr)>")]
+    pub mac: MacAddr6,
+}
+
 /// Configuration information for launching a Sled Agent.
 #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
 pub struct SledAgentRequest {
@@ -19,6 +43,15 @@ pub struct SledAgentRequest {
     /// Uuid of the rack to which this sled agent belongs.
     pub rack_id: Uuid,
 
+    /// Information about internet gateway to use
+    // NOTE: This information is currently being configured and sent from RSS,
+    // but it contains dynamic information that could plausibly change during
+    // the duration of the sled's lifetime.
+    //
+    // Longer-term, it probably makes sense to store this in CRDB and transfer
+    // it to Sled Agent as part of the request to launch Nexus.
+    pub gateway: Gateway,
+
     // Note: The order of these fields is load bearing, because we serialize
     // `SledAgentRequest`s as toml. `subnet` serializes as a TOML table, so it
     // must come after non-table fields.
@@ -109,8 +142,9 @@ mod tests {
             request: Request::SledAgentRequest(
                 Cow::Owned(SledAgentRequest {
                     id: Uuid::new_v4(),
-                    subnet: Ipv6Subnet::new(Ipv6Addr::LOCALHOST),
                     rack_id: Uuid::new_v4(),
+                    gateway: Gateway { address: None, mac: MacAddr6::nil() },
+                    subnet: Ipv6Subnet::new(Ipv6Addr::LOCALHOST),
                 }),
                 Some(
                     ShareDistribution {

sled-agent/src/config.rs

@@ -8,17 +8,11 @@ use crate::common::vlan::VlanID;
 use crate::illumos::dladm::{self, Dladm, PhysicalLink};
 use crate::illumos::zpool::ZpoolName;
 use dropshot::ConfigLogging;
-use macaddr::MacAddr6;
 use serde::Deserialize;
-use serde_with::serde_as;
-use serde_with::DisplayFromStr;
-use serde_with::PickFirst;
-use std::net::Ipv4Addr;
 use std::path::{Path, PathBuf};
 use uuid::Uuid;
 
 /// Configuration for a sled agent
-#[serde_as]
 #[derive(Clone, Debug, Deserialize)]
 pub struct Config {
     /// Unique id for the sled
@@ -30,20 +24,6 @@ pub struct Config {
     /// Optional list of zpools to be used as "discovered disks".
     pub zpools: Option<Vec<ZpoolName>>,
 
-    /// IP address of the Internet gateway, which is particularly
-    /// relevant for external-facing services (such as Nexus).
-    pub gateway_address: Option<Ipv4Addr>,
-
-    /// MAC address of the internet gateway above. This is used to provide
-    /// external connectivity into guests, by allowing OPTE to forward traffic
-    /// destined for the broader network to the gateway.
-    // This uses the `serde_with` crate's `serde_as` attribute, which tries
-    // each of the listed serialization types (starting with the default) until
-    // one succeeds. This supports deserialization from either an array of u8,
-    // or the display-string representation.
-    #[serde_as(as = "PickFirst<(_, DisplayFromStr)>")]
-    pub gateway_mac: MacAddr6,
-
     /// The data link on which we infer the bootstrap address.
     ///
     /// If unsupplied, we default to the first physical device.

sled-agent/src/rack_setup/config.rs

@@ -4,6 +4,7 @@
 
 //! Interfaces for working with RSS config.
 
+use crate::bootstrap::params::Gateway;
 use crate::config::ConfigError;
 use crate::params::{DatasetEnsureBody, ServiceRequest};
 use omicron_common::address::{
@@ -36,6 +37,9 @@ pub struct SetupServiceConfig {
     /// If this value is less than 2, no rack secret will be created on startup;
     /// this is the typical case for single-server test/development.
     pub rack_secret_threshold: usize,
+
+    /// Internet gateway information.
+    pub gateway: Gateway,
 }
 
 /// A request to initialize a sled.
@@ -89,6 +93,7 @@ mod test {
             rack_subnet: "fd00:1122:3344:0100::".parse().unwrap(),
             requests: vec![],
             rack_secret_threshold: 0,
+            gateway: Gateway { address: None, mac: macaddr::MacAddr6::nil() },
         };
 
         assert_eq!(

sled-agent/src/rack_setup/service.rs

@@ -356,6 +356,7 @@ impl ServiceInner {
                         id: Uuid::new_v4(),
                         subnet,
                         rack_id,
+                        gateway: config.gateway.clone(),
                     },
                     services_request: request,
                 },

sled-agent/src/server.rs

@@ -7,6 +7,7 @@
 use super::config::Config;
 use super::http_entrypoints::api as http_api;
 use super::sled_agent::SledAgent;
+use crate::bootstrap::params::SledAgentRequest;
 use crate::nexus::LazyNexusClient;
 use omicron_common::backoff::{
     internal_service_policy_with_max, retry_notify, BackoffError,
@@ -38,7 +39,7 @@ impl Server {
         log: Logger,
         addr: SocketAddrV6,
         is_scrimlet: bool,
-        rack_id: Uuid,
+        request: SledAgentRequest,
     ) -> Result<Server, String> {
         info!(log, "setting up sled agent server");
 
@@ -52,7 +53,7 @@ impl Server {
             log.clone(),
             lazy_nexus_client.clone(),
             addr,
-            rack_id,
+            request,
         )
         .await
         .map_err(|e| e.to_string())?;

sled-agent/src/sled_agent.rs

@@ -4,6 +4,7 @@
 
 //! Sled agent implementation
 
+use crate::bootstrap::params::SledAgentRequest;
 use crate::config::Config;
 use crate::illumos::vnic::VnicKind;
 use crate::illumos::zfs::{
@@ -126,7 +127,7 @@ impl SledAgent {
         log: Logger,
         lazy_nexus_client: LazyNexusClient,
         sled_address: SocketAddrV6,
-        rack_id: Uuid,
+        request: SledAgentRequest,
     ) -> Result<SledAgent, Error> {
         let id = config.id;
 
@@ -254,11 +255,11 @@ impl SledAgent {
             lazy_nexus_client.clone(),
             etherstub.clone(),
             *sled_address.ip(),
-            config.gateway_mac,
+            request.gateway.mac,
         );
 
         let svc_config = services::Config {
-            gateway_address: config.gateway_address,
+            gateway_address: request.gateway.address,
             ..Default::default()
         };
         let services = ServiceManager::new(
@@ -268,7 +269,7 @@ impl SledAgent {
             *sled_address.ip(),
             svc_config,
             config.get_link()?,
-            rack_id,
+            request.rack_id,
         )
         .await?;
 

smf/sled-agent/config-rss.toml

@@ -11,6 +11,22 @@ rack_subnet = "fd00:1122:3344:0100::"
 # For values less than 2, no rack secret will be generated.
 rack_secret_threshold = 1
 
+[gateway]
+
+# IP address of Internet gateway
+#
+# NOTE: In the lab, use "172.20.15.225"
+# address = "192.168.1.1"
+
+# MAC address of the internet gateway in the local network, i.e., of the above
+# IP address.
+#
+# NOTE: This is currently configured for the lab. Developers should override
+# this with whatever value they wish to provide inbound connectivity to guests
+# in their local network, using the current workaround methods in OPTE. See
+# how-to-run.adoc for details on how to determine the value for your network.
+mac = "00:0d:b9:54:fe:e4"
+
 [[request]]
 
 # TODO(https://github.com/oxidecomputer/omicron/issues/732): Nexus

smf/sled-agent/config.toml

@@ -21,20 +21,6 @@ zpools = [
 # $ dladm show-phys -p -o LINK
 # data_link = "igb0"
 
-# IP address of Internet gateway
-#
-# NOTE: In the lab, use "172.20.15.225"
-# gateway_address = "192.168.1.1"
-
-# MAC address of the internet gateway in the local network, i.e., of the above
-# IP address.
-#
-# NOTE: This is currently configured for the lab. Developers should override
-# this with whatever value they wish to provide inbound connectivity to guests
-# in their local network, using the current workaround methods in OPTE. See
-# how-to-run.adoc for details on how to determine the value for your network.
-gateway_mac = "00:0d:b9:54:fe:e4"
-
 [log]
 level = "info"
 mode = "file"