Adds external IPs for instance source NAT (#1298)

* Adds external IPs for instance source NAT

- Adds the `instance_external_ip` table for tracking external IP
  addresses and port ranges for instance source NAT, allocated out of an
  IP pool / range.
- Adds child-generation counter and tracking to the `ip_pool_range`
  table, and checks for allocated addresses when deleting a range.
- Adds query to insert next available address from any range, updating
  the parent rcgen. Adds tests for query behavior, including testing
  exhaustion.
- Add test helper to create an IP pool and range, since all
  instance-creation tests now need to have a pool available.
- Forwards external IP configuration to sled agent and passes to OPTE.

* Review feedback

- Deallocation of external IPs is idempotent
- Cleanup and comments

Author: bnaecker

Cargo.lock

@@ -983,7 +983,9 @@ CREATE TABLE omicron.public.ip_pool_range (
     first_address INET NOT NULL,
     /* The range is inclusive of the last address. */
     last_address INET NOT NULL,
-    ip_pool_id UUID NOT NULL
+    ip_pool_id UUID NOT NULL,
+    /* Tracks child resources, IP addresses allocated out of this range. */
+    rcgen INT8 NOT NULL
 );
 
 /* 
@@ -1002,6 +1004,66 @@ CREATE UNIQUE INDEX ON omicron.public.ip_pool_range (
 STORING (first_address)
 WHERE time_deleted IS NULL;
 
+/*
+ * External IP addresses used for instance source NAT.
+ *
+ * NOTE: This currently stores only address and port information for the
+ * automatic source NAT supplied for all guest instances. It does not currently
+ * store information about ephemeral or floating IPs.
+ */
+CREATE TABLE omicron.public.instance_external_ip (
+    id UUID PRIMARY KEY,
+    time_created TIMESTAMPTZ NOT NULL,
+    time_modified TIMESTAMPTZ NOT NULL,
+    time_deleted TIMESTAMPTZ,
+
+    /* FK to the `ip_pool` table. */
+    ip_pool_id UUID NOT NULL,
+
+    /* FK to the `ip_pool_range` table. */
+    ip_pool_range_id UUID NOT NULL,
+
+    /* FK to the `instance` table. */
+    instance_id UUID NOT NULL,
+
+    /* The actual external IP address. */
+    ip INET NOT NULL,
+
+    /* The first port in the allowed range, inclusive. */
+    first_port INT4 NOT NULL,
+
+    /* The last port in the allowed range, also inclusive. */
+    last_port INT4 NOT NULL
+);
+
+/*
+ * Index used to support quickly looking up children of the IP Pool range table,
+ * when checking for allocated addresses during deletion.
+ */
+CREATE INDEX ON omicron.public.instance_external_ip (
+    ip_pool_id,
+    ip_pool_range_id
+)
+    WHERE time_deleted IS NULL;
+
+/*
+ * Index used to enforce uniqueness of external IPs
+ *
+ * NOTE: This relies on the uniqueness constraint of IP addresses across all
+ * pools, _and_ on the fact that the number of ports assigned to each instance
+ * is fixed at compile time.
+ */
+CREATE UNIQUE INDEX ON omicron.public.instance_external_ip (
+    ip,
+    first_port
+)
+    WHERE time_deleted IS NULL;
+
+CREATE INDEX ON omicron.public.instance_external_ip (
+    instance_id
+)
+    WHERE time_deleted IS NULL;
+
 /*******************************************************************/
 
 /*

common/src/sql/dbinit.sql

@@ -144,7 +144,24 @@ command line interface.  Note that the `jq` command is required. In addition, th
     oxide org create myorg
     oxide project create -o myorg myproj
 
-2. Define a global image that will be used as initial disk contents.
+2. Create an IP Pool, for providing external connectivity to the instance later.
+We need to create an IP Pool itself, and a range of IP addresses in that pool.
+
+    oxide api /ip-pools --method POST --input - <<EOF
+    {
+      "name": "mypool",
+      "description": "an IP pool"
+    }
+    EOF
+
+    oxide api /ip-pools/mypool/ranges/add --method POST --input - <<EOF
+    {
+      "first": "10.0.0.1",
+      "last": "10.0.0.255"
+    }
+    EOF
+
+3. Define a global image that will be used as initial disk contents.
 
  a. This can be the alpine.iso image that ships with propolis:
 
@@ -181,7 +198,7 @@ command line interface.  Note that the `jq` command is required. In addition, th
     }
     EOF
 
-3. Create a disk from that global image (note that disk size must be greater than or equal to image size and a 1GiB multiple!). The example below creates a disk using the image made from the alpine ISO that ships with propolis, and sets the size to the next 1GiB multiple of the original alpine source:
+4. Create a disk from that global image (note that disk size must be greater than or equal to image size and a 1GiB multiple!). The example below creates a disk using the image made from the alpine ISO that ships with propolis, and sets the size to the next 1GiB multiple of the original alpine source:
 
     oxide api /organizations/myorg/projects/myproj/disks/ --method POST --input - <<EOF
     {
@@ -196,7 +213,7 @@ command line interface.  Note that the `jq` command is required. In addition, th
     }
     EOF
 
-4. Create an instance, attaching the alpine disk created above:
+5. Create an instance, attaching the alpine disk created above:
 
     oxide api /organizations/myorg/projects/myproj/instances --method POST --input - <<EOF
     {
@@ -214,7 +231,7 @@ command line interface.  Note that the `jq` command is required. In addition, th
     }
     EOF
 
-5. Optionally, attach to the propolis server serial console, though the serial console is under active development and these commands are subject to change:
+6. Optionally, attach to the propolis server serial console, though the serial console is under active development and these commands are subject to change:
 
   a. find the zone launched for the instance: `zoneadm list -c | grep oxz_propolis-server`
   b. get the instance uuid from the zone name. if the zone's name is `oxz_propolis-server_3b03ad43-4e9b-4f3a-866c-238d9ec4ac45`, then the uuid is `3b03ad43-4e9b-4f3a-866c-238d9ec4ac45`

docs/how-to-run.adoc

@@ -118,6 +118,7 @@ features = [ "serde", "v4" ]
 [dev-dependencies]
 criterion = { version = "0.3", features = [ "async_tokio" ] }
 expectorate = "1.0.5"
+itertools = "0.10.3"
 nexus-test-utils-macros = { path = "test-utils-macros" }
 nexus-test-utils = { path = "test-utils" }
 omicron-test-utils = { path = "../test-utils" }

nexus/Cargo.toml

@@ -26,6 +26,7 @@ use omicron_common::api::external::ListResultVec;
 use omicron_common::api::external::LookupResult;
 use omicron_common::api::external::UpdateResult;
 use omicron_common::api::internal::nexus;
+use sled_agent_client::types::ExternalIp;
 use sled_agent_client::types::InstanceRuntimeStateMigrateParams;
 use sled_agent_client::types::InstanceRuntimeStateRequested;
 use sled_agent_client::types::InstanceStateRequested;
@@ -199,7 +200,15 @@ impl super::Nexus {
                 .fetch()
                 .await?;
 
-        self.db_datastore.project_delete_instance(opctx, &authz_instance).await
+        self.db_datastore
+            .project_delete_instance(opctx, &authz_instance)
+            .await?;
+        self.db_datastore
+            .deallocate_instance_external_ip_by_instance_id(
+                opctx,
+                authz_instance.id(),
+            )
+            .await
     }
 
     pub async fn project_instance_migrate(
@@ -466,6 +475,12 @@ impl super::Nexus {
             .derive_guest_network_interface_info(&opctx, &authz_instance)
             .await?;
 
+        let external_ip = self
+            .db_datastore
+            .instance_lookup_external_ip(&opctx, authz_instance.id())
+            .await
+            .map(ExternalIp::from)?;
+
         // Gather the SSH public keys of the actor make the request so
         // that they may be injected into the new image via cloud-init.
         // TODO-security: this should be replaced with a lookup based on
@@ -502,6 +517,7 @@ impl super::Nexus {
                 db_instance.runtime().clone(),
             ),
             nics,
+            external_ip,
             disks: disk_reqs,
             cloud_init_bytes: Some(base64::encode(
                 db_instance.generate_cidata(&public_keys)?,

nexus/src/app/instance.rs

@@ -141,6 +141,17 @@ fn saga_instance_create() -> SagaTemplate<SagaInstanceCreate> {
         new_action_noop_undo(sic_create_network_interfaces),
     );
 
+    // Grab an external IP address and port range for the guest's Internet
+    // Gateway, allowing external connectivity.
+    template_builder.append(
+        "external_ip",
+        "ExternalIp",
+        ActionFunc::new_action(
+            sic_allocate_external_ip,
+            sic_allocate_external_ip_undo,
+        ),
+    );
+
     // Saga actions must be atomic - they have to fully complete or fully abort.
     // This is because Steno assumes that the saga actions are atomic and
     // therefore undo actions are *not* run for the failing node.
@@ -470,6 +481,40 @@ async fn sic_create_network_interfaces_undo(
     Ok(())
 }
 
+/// Create an external IP address for the instance.
+async fn sic_allocate_external_ip(
+    sagactx: ActionContext<SagaInstanceCreate>,
+) -> Result<Uuid, ActionError> {
+    let osagactx = sagactx.user_data();
+    let datastore = osagactx.datastore();
+    let saga_params = sagactx.saga_params();
+    let opctx =
+        OpContext::for_saga_action(&sagactx, &saga_params.serialized_authn);
+    let instance_id = sagactx.lookup::<Uuid>("instance_id")?;
+    let external_ip = datastore
+        .allocate_instance_external_ip(&opctx, instance_id)
+        .await
+        .map_err(ActionError::action_failed)?;
+    Ok(external_ip.id)
+}
+
+/// Destroy / release an external IP address allocated for the instance.
+async fn sic_allocate_external_ip_undo(
+    sagactx: ActionContext<SagaInstanceCreate>,
+) -> Result<(), anyhow::Error> {
+    let osagactx = sagactx.user_data();
+    let datastore = osagactx.datastore();
+    let saga_params = sagactx.saga_params();
+    let opctx =
+        OpContext::for_saga_action(&sagactx, &saga_params.serialized_authn);
+    let ip_id = sagactx.lookup::<Uuid>("external_ip")?;
+    datastore
+        .deallocate_instance_external_ip(&opctx, ip_id)
+        .await
+        .map_err(ActionError::action_failed)?;
+    Ok(())
+}
+
 /// Create disks during instance creation, and return a list of disk names
 // TODO implement
 async fn sic_create_disks_for_instance(

nexus/src/app/sagas/instance_create.rs

@@ -14,6 +14,7 @@ use omicron_common::api::external::Error;
 use omicron_common::api::internal::nexus::InstanceRuntimeState;
 use serde::Deserialize;
 use serde::Serialize;
+use sled_agent_client::types::ExternalIp;
 use sled_agent_client::types::InstanceEnsureBody;
 use sled_agent_client::types::InstanceHardware;
 use sled_agent_client::types::InstanceMigrateParams;
@@ -162,10 +163,17 @@ async fn sim_instance_migrate(
         )),
         ..old_runtime
     };
+    let external_ip = osagactx
+        .datastore()
+        .instance_lookup_external_ip(&opctx, instance_id)
+        .await
+        .map_err(ActionError::action_failed)
+        .map(ExternalIp::from)?;
     let instance_hardware = InstanceHardware {
         runtime: runtime.into(),
         // TODO: populate NICs
         nics: vec![],
+        external_ip,
         // TODO: populate disks
         disks: vec![],
         // TODO: populate cloud init bytes