[sled-agent] Remove chicken switch and always destroy orphaned Omicron datasets (#8895)

The main point of this PR is that the sled-agent config reconciler now
always attempts to destroy datasets it believes have been expunged
instead of defaulting to reporting them unless an operator has toggled a
chicken switch. (See the comments on `datasets_destroy_orphans()` for
details; this is pretty specific to only destroy datasets whose names we
understand.) We wanted to ship "report only" in R16 and then confirm
during upgrades that this wasn't showing any surprises; now that R16 is
out the door we can turn this on all the time.

This also (starts to? attempts to?) removes the sled-agent API endpoints
for interacting with the chicken switches. The openapi xtask yelled at
me for this, so instead I followed the guide for removing endpoints.
This was almost entirely painless, except that I'd already (locally)
committed the change to remove the endpoints mixed in with other things,
so I had to dig back out the endpoints and the types they used. Kudos to
all involved for the tooling and instructions here!

Closes #6177. Closes #7312 (no longer needed). Closes #7313.

Author: jgallagher

clients/sled-agent-client/src/lib.rs

@@ -331,6 +331,19 @@ impl From<omicron_common::api::internal::shared::NetworkInterfaceKind>
     }
 }
 
+// TODO-cleanup This is icky; can we move these methods to a separate client so
+// we don't need to add this header by hand?
+// https://github.com/oxidecomputer/omicron/issues/8900
+trait ApiVersionHeader {
+    fn api_version_header(self, api_version: &'static str) -> Self;
+}
+
+impl ApiVersionHeader for reqwest::RequestBuilder {
+    fn api_version_header(self, api_version: &'static str) -> Self {
+        self.header("api-version", api_version)
+    }
+}
+
 /// Exposes additional [`Client`] interfaces for use by the test suite. These
 /// are bonus endpoints, not generated in the real client.
 #[async_trait]
@@ -353,6 +366,7 @@ impl TestInterfaces for Client {
         let url = format!("{}/vmms/{}/poke-single-step", baseurl, id);
         client
             .post(url)
+            .api_version_header(self.api_version())
             .send()
             .await
             .expect("instance_single_step() failed unexpectedly");
@@ -364,6 +378,7 @@ impl TestInterfaces for Client {
         let url = format!("{}/vmms/{}/poke", baseurl, id);
         client
             .post(url)
+            .api_version_header(self.api_version())
             .send()
             .await
             .expect("instance_finish_transition() failed unexpectedly");
@@ -375,6 +390,7 @@ impl TestInterfaces for Client {
         let url = format!("{}/disks/{}/poke", baseurl, id);
         client
             .post(url)
+            .api_version_header(self.api_version())
             .send()
             .await
             .expect("disk_finish_transition() failed unexpectedly");
@@ -390,6 +406,7 @@ impl TestInterfaces for Client {
         let url = format!("{baseurl}/vmms/{id}/sim-migration-source");
         client
             .post(url)
+            .api_version_header(self.api_version())
             .json(&params)
             .send()
             .await

dev-tools/omdb/src/bin/omdb/sled_agent.rs

@@ -5,13 +5,11 @@
 //! omdb commands that query or update specific Sleds
 
 use crate::Omdb;
-use crate::check_allow_destructive::DestructiveOperationToken;
 use crate::helpers::CONNECTION_OPTIONS_HEADING;
 use anyhow::Context;
 use anyhow::bail;
 use clap::Args;
 use clap::Subcommand;
-use sled_agent_client::types::ChickenSwitchDestroyOrphanedDatasets;
 
 /// Arguments to the "omdb sled-agent" subcommand
 #[derive(Debug, Args)]
@@ -39,11 +37,6 @@ enum SledAgentCommands {
     /// print information about the local bootstore node
     #[clap(subcommand)]
     Bootstore(BootstoreCommands),
-
-    /// control "chicken switches" (potentially-destructive sled-agent behavior
-    /// that can be toggled on or off via `omdb`)
-    #[clap(subcommand)]
-    ChickenSwitch(ChickenSwitchCommands),
 }
 
 #[derive(Debug, Subcommand)]
@@ -58,12 +51,6 @@ enum BootstoreCommands {
     Status,
 }
 
-#[derive(Debug, Subcommand)]
-enum ChickenSwitchCommands {
-    /// interact with the "destroy orphaned datasets" chicken switch
-    DestroyOrphans(DestroyOrphansArgs),
-}
-
 #[derive(Debug, Args)]
 struct DestroyOrphansArgs {
     #[command(subcommand)]
@@ -84,7 +71,7 @@ impl SledAgentArgs {
     /// Run a `omdb sled-agent` subcommand.
     pub(crate) async fn run_cmd(
         &self,
-        omdb: &Omdb,
+        _omdb: &Omdb,
         log: &slog::Logger,
     ) -> Result<(), anyhow::Error> {
         // This is a little goofy. The sled URL is required, but can come
@@ -105,29 +92,6 @@ impl SledAgentArgs {
             SledAgentCommands::Bootstore(BootstoreCommands::Status) => {
                 cmd_bootstore_status(&client).await
             }
-            SledAgentCommands::ChickenSwitch(
-                ChickenSwitchCommands::DestroyOrphans(DestroyOrphansArgs {
-                    command: DestroyOrphansCommands::Get,
-                }),
-            ) => cmd_chicken_switch_destroy_orphans_get(&client).await,
-            SledAgentCommands::ChickenSwitch(
-                ChickenSwitchCommands::DestroyOrphans(DestroyOrphansArgs {
-                    command: DestroyOrphansCommands::Enable,
-                }),
-            ) => {
-                let token = omdb.check_allow_destructive()?;
-                cmd_chicken_switch_destroy_orphans_set(&client, true, token)
-                    .await
-            }
-            SledAgentCommands::ChickenSwitch(
-                ChickenSwitchCommands::DestroyOrphans(DestroyOrphansArgs {
-                    command: DestroyOrphansCommands::Disable,
-                }),
-            ) => {
-                let token = omdb.check_allow_destructive()?;
-                cmd_chicken_switch_destroy_orphans_set(&client, false, token)
-                    .await
-            }
         }
     }
 }
@@ -193,33 +157,3 @@ async fn cmd_bootstore_status(
 
     Ok(())
 }
-
-/// Runs `omdb sled-agent chicken-switch destroy-orphans get`
-async fn cmd_chicken_switch_destroy_orphans_get(
-    client: &sled_agent_client::Client,
-) -> Result<(), anyhow::Error> {
-    let ChickenSwitchDestroyOrphanedDatasets { destroy_orphans } = client
-        .chicken_switch_destroy_orphaned_datasets_get()
-        .await
-        .context("get chicken switch value")?
-        .into_inner();
-    let status = if destroy_orphans { "enabled" } else { "disabled" };
-    println!("destroy orphaned datasets {status}");
-    Ok(())
-}
-
-/// Runs `omdb sled-agent chicken-switch destroy-orphans {enable,disable}`
-async fn cmd_chicken_switch_destroy_orphans_set(
-    client: &sled_agent_client::Client,
-    destroy_orphans: bool,
-    _token: DestructiveOperationToken,
-) -> Result<(), anyhow::Error> {
-    let options = ChickenSwitchDestroyOrphanedDatasets { destroy_orphans };
-    client
-        .chicken_switch_destroy_orphaned_datasets_put(&options)
-        .await
-        .context("put chicken switch value")?;
-    let status = if destroy_orphans { "enabled" } else { "disabled" };
-    println!("destroy orphaned datasets {status}");
-    Ok(())
-}

dev-tools/omdb/tests/usage_errors.out

@@ -1063,11 +1063,9 @@ Debug a specific Sled
 Usage: omdb sled-agent [OPTIONS] <COMMAND>
 
 Commands:
-  zones           print information about zones
-  bootstore       print information about the local bootstore node
-  chicken-switch  control "chicken switches" (potentially-destructive sled-agent behavior that can
-                  be toggled on or off via `omdb`)
-  help            Print this message or the help of the given subcommand(s)
+  zones      print information about zones
+  bootstore  print information about the local bootstore node
+  help       Print this message or the help of the given subcommand(s)
 
 Options:
       --log-level <LOG_LEVEL>  log level filter [env: LOG_LEVEL=] [default: warn]

nexus/mgs-updates/src/test_util/host_phase_2_test_state.rs

@@ -124,6 +124,12 @@ impl HostPhase2TestContext {
                 bind_address: "[::1]:0".parse().unwrap(),
                 ..Default::default()
             })
+            .version_policy(dropshot::VersionPolicy::Dynamic(Box::new(
+                dropshot::ClientSpecifiesVersionInHeader::new(
+                    omicron_common::api::VERSION_HEADER,
+                    sled_agent_api::VERSION_REMOVE_DESTROY_ORPHANED_DATASETS_CHICKEN_SWITCH,
+                ),
+            )))
             .start()
             .context("failed to create dropshot server")?
         };