[DRAFT] Allocate static Ipv6 addresses for propolis zones

common/src/sql/dbinit.sql

@@ -959,6 +959,14 @@ CREATE TABLE omicron.public.role_assignment_builtin (
     PRIMARY KEY(user_builtin_id, resource_type, resource_id, role_name)
 );
 
+/*
+ * Store addresses allocated for internal services (like propolis zones)
+ */
+CREATE TABLE omicron.public.static_v6_address (
+    address INET PRIMARY KEY NOT NULL,
+    associated_id UUID NOT NULL
+);
+
 /*******************************************************************/
 
 /*

nexus/src/db/datastore.rs

@@ -55,6 +55,7 @@ use omicron_common::api::external::{
 use omicron_common::api::internal::nexus::UpdateArtifact;
 use omicron_common::bail_unless;
 use std::convert::{TryFrom, TryInto};
+use std::net::IpAddr;
 use std::sync::Arc;
 use uuid::Uuid;
 
@@ -67,9 +68,10 @@ use crate::db::{
         Name, NetworkInterface, Organization, OrganizationUpdate, OximeterInfo,
         ProducerEndpoint, Project, ProjectUpdate, Region,
         RoleAssignmentBuiltin, RoleBuiltin, RouterRoute, RouterRouteUpdate,
-        Silo, SiloUser, Sled, UpdateArtifactKind, UpdateAvailableArtifact,
-        UserBuiltin, Volume, Vpc, VpcFirewallRule, VpcRouter, VpcRouterUpdate,
-        VpcSubnet, VpcSubnetUpdate, VpcUpdate, Zpool,
+        Silo, SiloUser, Sled, StaticV6Address, UpdateArtifactKind,
+        UpdateAvailableArtifact, UserBuiltin, Volume, Vpc, VpcFirewallRule,
+        VpcRouter, VpcRouterUpdate, VpcSubnet, VpcSubnetUpdate, VpcUpdate,
+        Zpool,
     },
     pagination::paginated,
     pagination::paginated_multicolumn,
@@ -1312,6 +1314,16 @@ impl DataStore {
         let failed = DbInstanceState::new(ApiInstanceState::Failed);
 
         let instance_id = authz_instance.id();
+
+        // XXX delete allocated ip
+        // XXX what if the address isn't in the table?
+        // XXX do in transaction (saga?)
+        //
+        //use db::schema::static_v6_address::dsl as address_dsl
+        //diesel::delete(address_dsl::static_v6_address)
+        //    .filter(address_dsl::associated_id.eq(instance_id))
+        //    .execute(conn)?;
+
         let result = diesel::update(dsl::instance)
             .filter(dsl::time_deleted.is_null())
             .filter(dsl::id.eq(instance_id))
@@ -1326,6 +1338,7 @@ impl DataStore {
                     ErrorHandler::NotFoundByResource(authz_instance),
                 )
             })?;
+
         match result.status {
             UpdateStatus::Updated => Ok(()),
             UpdateStatus::NotUpdatedButExists => {
@@ -3714,6 +3727,93 @@ impl DataStore {
 
         Ok(())
     }
+
+    pub async fn allocate_static_v6_address(
+        &self,
+        associated_id: Uuid,
+    ) -> Result<IpAddr, Error> {
+        use db::schema::static_v6_address::dsl;
+
+        self.pool()
+            .transaction(move |conn| {
+                // Grab the next sequential address
+                // XXX should this use push_next_available_ip_subquery?
+                // XXX refactor push_next_available_ip_subquery?
+                // XXX this is a full table scan
+                let query = vec![
+                    "select".to_string(),
+                    format!("'{}'::UUID", associated_id),
+                    "as associated_id, static_v6_address.address + 1 as address".to_string(),
+                    "from static_v6_address where not exists".to_string(),
+                    "(select 1 from static_v6_address tmp where tmp.address = static_v6_address.address + 1)".to_string(),
+                    "limit 1".to_string(),
+                ];
+                let new_address: Vec<StaticV6Address> = diesel::sql_query(query.join(" ")).load(conn)?;
+
+                let new_address = if new_address.len() != 1 {
+                    // This means there are no addresses in the table!
+                    // XXX normally this base address should come from something
+                    // else, hard code here.
+                    StaticV6Address {
+                        associated_id,
+                        address: "fd00:1234::101".parse().unwrap(),
+                    }
+                } else {
+                    new_address[0].clone()
+                };
+
+                let new_address = diesel::insert_into(dsl::static_v6_address)
+                    .values(new_address)
+                    .returning(StaticV6Address::as_returning())
+                    .get_result(conn)?;
+
+                return Ok(new_address.address.ip());
+            })
+            .await
+            .map_err(|e: async_bb8_diesel::PoolError| {
+                Error::internal_error(&format!("Transaction error: {}", e))
+            })
+    }
+
+    pub async fn static_v6_address_fetch(
+        &self,
+        associated_id: Uuid,
+    ) -> LookupResult<IpAddr> {
+        use db::schema::static_v6_address::dsl;
+
+        dsl::static_v6_address
+            .filter(dsl::associated_id.eq(associated_id))
+            .select(StaticV6Address::as_select())
+            .first_async(self.pool())
+            .await
+            .map(|x| x.address.ip())
+            .map_err(|e| {
+                Error::internal_error(&format!(
+                    "error fetching static v6 address for {:?}: {:?}",
+                    associated_id, e
+                ))
+            })
+    }
+
+    pub async fn free_static_v6_address(
+        &self,
+        address: IpAddr,
+    ) -> UpdateResult<()> {
+        use db::schema::static_v6_address::dsl;
+
+        // XXX what if the address isn't in the table?
+        diesel::delete(dsl::static_v6_address)
+            .filter(dsl::address.eq(ipnetwork::IpNetwork::from(address)))
+            .execute_async(self.pool())
+            .await
+            .map(|_| ())
+            .map_err(|e| {
+                Error::internal_error(&format!(
+                    "error freeing static v6 address: {:?}",
+                    e
+                ))
+            })
+    }
 }
 
 /// Constructs a DataStore for use in test suites that has preloaded the
@@ -3766,7 +3866,7 @@ mod test {
     };
     use omicron_test_utils::dev;
     use std::collections::HashSet;
-    use std::net::{IpAddr, Ipv4Addr, SocketAddr};
+    use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
     use std::sync::Arc;
     use uuid::Uuid;
 
@@ -4213,4 +4313,83 @@ mod test {
 
         let _ = db.cleanup().await;
     }
+
+    // Test static v6 address allocation
+    #[tokio::test]
+    async fn test_static_v6_address_allocation() -> Result<(), Error> {
+        let logctx = dev::test_setup_log("test_static_v6_address_allocation");
+        let mut db = test_setup_database(&logctx.log).await;
+        let cfg = db::Config { url: db.pg_config().clone() };
+        let pool = db::Pool::new(&cfg);
+        let datastore = DataStore::new(Arc::new(pool));
+
+        // db contents: []
+        let address1 =
+            datastore.allocate_static_v6_address(Uuid::new_v4()).await?;
+        // db contents: [101]
+        assert_eq!(address1, "fd00:1234::101".parse::<Ipv6Addr>().unwrap());
+
+        // db contents: [101]
+        let address2 =
+            datastore.allocate_static_v6_address(Uuid::new_v4()).await?;
+        // db contents: [101, 102]
+        assert_eq!(address2, "fd00:1234::102".parse::<Ipv6Addr>().unwrap());
+
+        // db contents: [101, 102]
+        let address3 =
+            datastore.allocate_static_v6_address(Uuid::new_v4()).await?;
+        // db contents: [101, 102, 103]
+        assert_eq!(address3, "fd00:1234::103".parse::<Ipv6Addr>().unwrap());
+
+        // db contents: [101, 102, 103]
+        datastore.free_static_v6_address(address1).await?;
+        // db contents: [102, 103]
+
+        // db contents: [102, 103]
+        let address4 =
+            datastore.allocate_static_v6_address(Uuid::new_v4()).await?;
+        // db contents: [102, 103, 104]
+        assert_eq!(address4, "fd00:1234::104".parse::<Ipv6Addr>().unwrap());
+
+        // db contents: [102, 103, 104]
+        datastore.free_static_v6_address(address4).await?;
+        // db contents: [102, 103]
+
+        // db contents: [102, 103]
+        datastore.free_static_v6_address(address2).await?;
+        // db contents: [103]
+
+        // db contents: [103]
+        let address5 =
+            datastore.allocate_static_v6_address(Uuid::new_v4()).await?;
+        // db contents: [103, 104]
+        assert_eq!(address5, "fd00:1234::104".parse::<Ipv6Addr>().unwrap());
+
+        // db contents: [103, 104]
+        let address6 =
+            datastore.allocate_static_v6_address(Uuid::new_v4()).await?;
+        // db contents: [103, 104, 105]
+        assert_eq!(address6, "fd00:1234::105".parse::<Ipv6Addr>().unwrap());
+
+        // db contents: [103, 104, 105]
+        let address7 =
+            datastore.allocate_static_v6_address(Uuid::new_v4()).await?;
+        // db contents: [103, 104, 105, 106]
+        assert_eq!(address7, "fd00:1234::106".parse::<Ipv6Addr>().unwrap());
+
+        // get associated address
+        let id = Uuid::new_v4();
+        let address = datastore.allocate_static_v6_address(id.clone()).await?;
+        assert_eq!(address, datastore.static_v6_address_fetch(id).await?);
+
+        // assert that getting the address of an unknown ID returns an error
+        assert!(datastore
+            .static_v6_address_fetch(Uuid::new_v4())
+            .await
+            .is_err());
+
+        let _ = db.cleanup().await;
+
+        Ok(())
+    }
 }

nexus/src/db/model.rs

@@ -10,8 +10,8 @@ use crate::db::schema::{
     console_session, dataset, disk, instance, metric_producer,
     network_interface, organization, oximeter, project, rack, region,
     role_assignment_builtin, role_builtin, router_route, silo, silo_user, sled,
-    snapshot, update_available_artifact, user_builtin, volume, vpc,
-    vpc_firewall_rule, vpc_router, vpc_subnet, zpool,
+    snapshot, static_v6_address, update_available_artifact, user_builtin,
+    volume, vpc, vpc_firewall_rule, vpc_router, vpc_subnet, zpool,
 };
 use crate::defaults;
 use crate::external_api::params;
@@ -2460,6 +2460,21 @@ pub struct UpdateAvailableArtifact {
     pub target_length: i64,
 }
 
+#[derive(
+    Queryable,
+    QueryableByName,
+    Insertable,
+    Clone,
+    Debug,
+    Selectable,
+    AsChangeset,
+)]
+#[table_name = "static_v6_address"]
+pub struct StaticV6Address {
+    pub address: IpNetwork,
+    pub associated_id: Uuid,
+}
+
 #[cfg(test)]
 mod tests {
     use super::Uuid;

nexus/src/db/schema.rs

@@ -399,6 +399,13 @@ table! {
     }
 }
 
+table! {
+    static_v6_address (address) {
+        address -> Inet,
+        associated_id -> Uuid,
+    }
+}
+
 allow_tables_to_appear_in_same_query!(
     dataset,
     disk,

nexus/src/internal_api/params.rs

@@ -7,7 +7,7 @@ use omicron_common::api::external::ByteCount;
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 use std::fmt;
-use std::net::SocketAddr;
+use std::net::{IpAddr, SocketAddr};
 use std::str::FromStr;
 use uuid::Uuid;
 
@@ -105,3 +105,15 @@ pub struct OximeterInfo {
     /// The address on which this oximeter instance listens for requests
     pub address: SocketAddr,
 }
+
+/// Response when allocating a static v6 address
+#[derive(Debug, Clone, Copy, JsonSchema, Serialize, Deserialize)]
+pub struct AllocateStaticV6AddressResponse {
+    pub address: IpAddr,
+}
+
+/// Static Ipv6 address to free
+#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)]
+pub struct FreeStaticV6AddressParams {
+    pub address: IpAddr,
+}

nexus/src/nexus.rs

@@ -70,7 +70,7 @@ use sled_agent_client::types::InstanceStateRequested;
 use sled_agent_client::Client as SledAgentClient;
 use slog::Logger;
 use std::convert::{TryFrom, TryInto};
-use std::net::SocketAddr;
+use std::net::{IpAddr, SocketAddr};
 use std::num::NonZeroU32;
 use std::path::Path;
 use std::sync::Arc;
@@ -1370,6 +1370,9 @@ impl Nexus {
             disks: disk_reqs,
         };
 
+        let allocated_control_ip =
+            self.db_datastore.static_v6_address_fetch(db_instance.id()).await?;
+
         let sa = self.instance_sled(&db_instance).await?;
 
         let new_runtime = sa
@@ -1379,6 +1382,7 @@ impl Nexus {
                     initial: instance_hardware,
                     target: requested,
                     migrate: None,
+                    allocated_control_ip: allocated_control_ip.to_string(),
                 },
             )
             .await
@@ -3287,6 +3291,20 @@ impl Nexus {
     ) -> LookupResult<SiloUser> {
         self.db_datastore.silo_user_fetch(silo_user_id).await
     }
+
+    pub async fn allocate_static_v6_address(
+        &self,
+        associated_id: Uuid,
+    ) -> Result<IpAddr, Error> {
+        self.db_datastore.allocate_static_v6_address(associated_id).await
+    }
+
+    pub async fn free_static_v6_address(
+        &self,
+        address: IpAddr,
+    ) -> Result<(), Error> {
+        self.db_datastore.free_static_v6_address(address).await
+    }
 }
 
 fn generate_session_token() -> String {