Push probe zones to sled-agent instead of pulling them from Nexus (#9353)

- Add APIs in the sled-agent for creating / deleting probes, and have
Nexus use them when managing probes from the external API, especially
replacing the entire set of probes with a PUT.
- Rework the probe manager to accept the list of expected probes from
Nexus, and drive the state toward that, rather than periodically
pollling Nexus.
- Add background task for periodically pushing probes to sleds, and omdb
innards for reporting its state.
- Closes #9157

Author: bnaecker

Cargo.lock

@@ -60,6 +60,7 @@ use nexus_types::internal_api::background::InstanceReincarnationStatus;
 use nexus_types::internal_api::background::InstanceUpdaterStatus;
 use nexus_types::internal_api::background::InventoryLoadStatus;
 use nexus_types::internal_api::background::LookupRegionPortStatus;
+use nexus_types::internal_api::background::ProbeDistributorStatus;
 use nexus_types::internal_api::background::ReadOnlyRegionReplacementStartStatus;
 use nexus_types::internal_api::background::RegionReplacementDriverStatus;
 use nexus_types::internal_api::background::RegionReplacementStatus;
@@ -1193,6 +1194,9 @@ fn print_task_details(bgtask: &BackgroundTask, details: &serde_json::Value) {
         "phantom_disks" => {
             print_task_phantom_disks(details);
         }
+        "probe_distributor" => {
+            print_task_probe_distributor(details);
+        }
         "read_only_region_replacement_start" => {
             print_task_read_only_region_replacement_start(details);
         }
@@ -2131,6 +2135,32 @@ fn print_task_phantom_disks(details: &serde_json::Value) {
     };
 }
 
+fn print_task_probe_distributor(details: &serde_json::Value) {
+    match serde_json::from_value::<ProbeDistributorStatus>(details.clone()) {
+        Err(error) => eprintln!(
+            "warning: failed to interpret task details: {:?}: {:?}",
+            error, details
+        ),
+        Ok(ProbeDistributorStatus { probes_by_sled, errors }) => {
+            let n_total_probes: usize = probes_by_sled.values().sum();
+            println!("    succesfully-pushed probes: {} total", n_total_probes);
+            for (sled_id, count) in probes_by_sled {
+                if count == 0 {
+                    continue;
+                }
+                println!("      sled_id={} n_probes={}", sled_id, count);
+            }
+            println!("    errors while pushing probes: {} total", errors.len());
+            for err in errors {
+                println!(
+                    "      sled_id={} sled_ip={} error={}",
+                    err.sled_id, err.sled_ip, err.error,
+                );
+            }
+        }
+    };
+}
+
 fn print_task_read_only_region_replacement_start(details: &serde_json::Value) {
     match serde_json::from_value::<ReadOnlyRegionReplacementStartStatus>(
         details.clone(),

dev-tools/omdb/src/bin/omdb/nexus.rs

@@ -149,6 +149,10 @@ task: "physical_disk_adoption"
     ensure new physical disks are automatically marked in-service
 
 
+task: "probe_distributor"
+    distributes networking probe zones to sleds
+
+
 task: "read_only_region_replacement_start"
     detect if read-only regions need replacement and begin the process
 
@@ -373,6 +377,10 @@ task: "physical_disk_adoption"
     ensure new physical disks are automatically marked in-service
 
 
+task: "probe_distributor"
+    distributes networking probe zones to sleds
+
+
 task: "read_only_region_replacement_start"
     detect if read-only regions need replacement and begin the process
 
@@ -584,6 +592,10 @@ task: "physical_disk_adoption"
     ensure new physical disks are automatically marked in-service
 
 
+task: "probe_distributor"
+    distributes networking probe zones to sleds
+
+
 task: "read_only_region_replacement_start"
     detect if read-only regions need replacement and begin the process
 

dev-tools/omdb/tests/env.out

@@ -384,6 +384,10 @@ task: "physical_disk_adoption"
     ensure new physical disks are automatically marked in-service
 
 
+task: "probe_distributor"
+    distributes networking probe zones to sleds
+
+
 task: "read_only_region_replacement_start"
     detect if read-only regions need replacement and begin the process
 
@@ -713,6 +717,13 @@ task: "physical_disk_adoption"
     started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
     last completion reported error: task disabled
 
+task: "probe_distributor"
+  configured period: every <REDACTED_DURATION>m
+  last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
+    started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
+    succesfully-pushed probes: 0 total
+    errors while pushing probes: 0 total
+
 task: "read_only_region_replacement_start"
   configured period: every <REDACTED_DURATION>days <REDACTED_DURATION>h <REDACTED_DURATION>m <REDACTED_DURATION>s
   last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
@@ -1267,6 +1278,13 @@ task: "physical_disk_adoption"
     started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
     last completion reported error: task disabled
 
+task: "probe_distributor"
+  configured period: every <REDACTED_DURATION>m
+  last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
+    started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
+    succesfully-pushed probes: 0 total
+    errors while pushing probes: 0 total
+
 task: "read_only_region_replacement_start"
   configured period: every <REDACTED_DURATION>days <REDACTED_DURATION>h <REDACTED_DURATION>m <REDACTED_DURATION>s
   last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>

dev-tools/omdb/tests/successes.out

@@ -57,7 +57,7 @@ impl Drop for PortInner {
             Err(e) => e,
         };
         eprintln!(
-            "WARNING: Failed to delete the xde device. It must be deleted
+            "WARNING: Failed to delete the xde device. It must be deleted \
             out of band, and it will not be possible to recreate the xde \
             device until then. Error: {:?}",
             err,

illumos-utils/src/opte/port.rs

@@ -443,6 +443,8 @@ pub struct BackgroundTaskConfig {
     pub sp_ereport_ingester: SpEreportIngesterConfig,
     /// configuration for fault management background tasks
     pub fm: FmTasksConfig,
+    /// configuration for networking probe distributor
+    pub probe_distributor: ProbeDistributorConfig,
 }
 
 #[serde_as]
@@ -897,6 +899,15 @@ impl Default for FmTasksConfig {
     }
 }
 
+#[serde_as]
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
+pub struct ProbeDistributorConfig {
+    /// period (in seconds) for periodic activations of the background task that
+    /// distributes networking probe zones to sled-agents.
+    #[serde_as(as = "DurationSeconds<u64>")]
+    pub period_secs: Duration,
+}
+
 /// Configuration for a nexus server
 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
 pub struct PackageConfig {
@@ -1201,6 +1212,7 @@ mod test {
             sp_ereport_ingester.period_secs = 47
             fm.sitrep_load_period_secs = 48
             fm.sitrep_gc_period_secs = 49
+            probe_distributor.period_secs = 50
             [default_region_allocation_strategy]
             type = "random"
             seed = 0
@@ -1448,7 +1460,10 @@ mod test {
                         fm: FmTasksConfig {
                             sitrep_load_period_secs: Duration::from_secs(48),
                             sitrep_gc_period_secs: Duration::from_secs(49),
-                        }
+                        },
+                        probe_distributor: ProbeDistributorConfig {
+                            period_secs: Duration::from_secs(50),
+                        },
                     },
                     default_region_allocation_strategy:
                         crate::nexus_config::RegionAllocationStrategy::Random {
@@ -1549,6 +1564,7 @@ mod test {
             sp_ereport_ingester.period_secs = 44
             fm.sitrep_load_period_secs = 45
             fm.sitrep_gc_period_secs = 46
+            probe_distributor.period_secs = 47
 
             [default_region_allocation_strategy]
             type = "random"

nexus-config/src/nexus_config.rs

@@ -53,6 +53,7 @@ pub struct BackgroundTasks {
     pub task_reconfigurator_config_loader: Activator,
     pub task_fm_sitrep_loader: Activator,
     pub task_fm_sitrep_gc: Activator,
+    pub task_probe_distributor: Activator,
 
     // Handles to activate background tasks that do not get used by Nexus
     // at-large.  These background tasks are implementation details as far as