MGMT-15088: Add multi VIP support

src/assisted_test_infra/test_infra/controllers/node_controllers/nutanix_controller.py

@@ -61,7 +61,10 @@ def get_ingress_and_api_vips(self):
                 raise ValueError("API VIP is not set")
             if not self._entity_config.ingress_vip:
                 raise ValueError("Ingress VIP is not set")
-            return {"api_vip": self._entity_config.api_vip, "ingress_vip": self._entity_config.ingress_vip}
+            return {
+                "api_vips": [{"ip": self._entity_config.api_vip}],
+                "ingress_vips": [{"ip": self._entity_config.ingress_vip}],
+            }
 
         nutanix_subnet = next(
             s for s in NutanixSubnet.list_entities(self._provider_client) if s.name == self._config.nutanix_subnet
@@ -88,7 +91,7 @@ def get_ingress_and_api_vips(self):
             raise ConnectionError("Failed to locate free API and Ingress VIPs")
 
         log.info(f"Found 2 optional VIPs: {free_ips}")
-        return {"api_vip": free_ips.pop(), "ingress_vip": free_ips.pop()}
+        return {"api_vips": [free_ips.pop()], "ingress_vips": [free_ips.pop()]}
 
     def set_boot_order(self, node_name, cd_first=False, cdrom_iso_path=None) -> None:
         vm = self._get_provider_vm(tf_vm_name=node_name)

src/assisted_test_infra/test_infra/controllers/node_controllers/terraform_controller.py

@@ -154,6 +154,20 @@ def _create_nodes(self, running=True):
         if self._params.running:
             self.wait_till_nodes_are_ready(network_name=self._params.libvirt_network_name)
 
+    def _get_vips(self):
+        vips = self.get_ingress_and_api_vips()
+        if self._config.api_vips:
+            api_vips = [i.get("ip") for i in self._config.api_vips]
+        else:
+            api_vips = [i.get("ip") for i in vips["api_vips"]]
+
+        if self._config.ingress_vips:
+            ingress_vips = [i.get("ip") for i in self._config.ingress_vips]
+        else:
+            ingress_vips = [i.get("ip") for i in vips["ingress_vips"]]
+
+        return api_vips, ingress_vips
+
     # Filling tfvars json files with terraform needed variables to spawn vms
     def _fill_tfvars(self, running=True):
         log.info("Filling tfvars")
@@ -183,9 +197,12 @@ def _fill_tfvars(self, running=True):
         tfvars["provisioning_cidr_addresses"] = self.get_all_provisioning_addresses()
         tfvars["bootstrap_in_place"] = self._config.bootstrap_in_place
 
-        vips = self.get_ingress_and_api_vips()
-        tfvars["api_vip"] = self._config.api_vip or vips["api_vip"]
-        tfvars["ingress_vip"] = self._config.ingress_vip or vips["ingress_vip"]
+        tfvars["api_vips"], tfvars["ingress_vips"] = self._get_vips()
+
+        # TODO: Remove singular VIPs once MGMT-14810 gets merged.
+        tfvars["api_vip"] = tfvars["api_vips"][0]
+        tfvars["ingress_vip"] = tfvars["ingress_vips"][0]
+
         if self._config.base_cluster_domain:
             tfvars["base_cluster_domain"] = self._config.base_cluster_domain
 
@@ -233,17 +250,17 @@ def format_node_disk(self, node_name: str, disk_index: int = 0):
         log.info("Formatting disk for %s", node_name)
         self.format_disk(f"{self._params.libvirt_storage_pool_path}/{self.entity_name}/{node_name}-disk-{disk_index}")
 
-    def get_ingress_and_api_vips(self) -> Dict[str, str]:
+    def get_ingress_and_api_vips(self) -> Dict[str, List[dict]]:
         """Pick two IPs for setting static access endpoint IPs.
 
-        Using <sub-net>.100 for the API endpoint and <sub-net>.101 for the ingress endpoint
+        Using <subnet>.100 for the API endpoint and <subnet>.101 for the ingress endpoint
         (the IPv6 values are appropriately <sub-net>:64 and <sub-net>:65).
 
         This method is not applicable for SNO clusters, where access IPs should be the node's IP.
         """
         network_subnet_starting_ip = ip_address(ip_network(self.get_primary_machine_cidr()).network_address)
         ips = utils.create_ip_address_list(2, starting_ip_addr=network_subnet_starting_ip + 100)
-        return {"api_vip": ips[0], "ingress_vip": ips[1]}
+        return {"api_vips": [{"ip": ips[0]}], "ingress_vips": [{"ip": ips[1]}]}
 
     @utils.on_exception(message="Failed to run terraform delete", silent=True)
     def _create_address_list(self, num, starting_ip_addr):

src/assisted_test_infra/test_infra/controllers/node_controllers/tf_controller.py

@@ -109,9 +109,12 @@ def set_dns(self, api_ip: str, ingress_ip: str) -> None:
             ingress_ip=ingress_ip,
         )
 
-    def get_ingress_and_api_vips(self) -> Optional[Dict[str, str]]:
+    def get_ingress_and_api_vips(self) -> dict[str, list[dict]] | None:
         if self._entity_config.api_vip and self._entity_config.ingress_vip:
-            return {"api_vip": self._entity_config.api_vip, "ingress_vip": self._entity_config.ingress_vip}
+            return {
+                "api_vips": [{"ip": self._entity_config.api_vip}],
+                "ingress_vips": [{"ip": self._entity_config.ingress_vip}],
+            }
 
         return None
 

src/assisted_test_infra/test_infra/helper_classes/cluster.py

@@ -357,18 +357,18 @@ def set_network_params(self, controller=None):
 
         if self._config.platform == consts.Platforms.NONE:
             log.info("On None platform, leaving network management to the user")
-            api_vip = ingress_vip = machine_networks = None
+            api_vips = ingress_vips = machine_networks = None
 
         elif self._config.vip_dhcp_allocation or self._high_availability_mode == consts.HighAvailabilityMode.NONE:
             log.info("Letting access VIPs be deducted from machine networks")
-            api_vip = ingress_vip = None
+            api_vips = ingress_vips = None
             machine_networks = [self.get_machine_networks()[0]]
 
         else:
             log.info("Assigning VIPs statically")
             access_vips = controller.get_ingress_and_api_vips()
-            api_vip = access_vips["api_vip"] if access_vips else None
-            ingress_vip = access_vips["ingress_vip"] if access_vips else None
+            api_vips = access_vips["api_vips"] if access_vips else None
+            ingress_vips = access_vips["ingress_vips"] if access_vips else None
             machine_networks = None
 
         if self._config.is_ipv4 and self._config.is_ipv6:
@@ -379,8 +379,8 @@ def set_network_params(self, controller=None):
             cluster_networks=self._config.cluster_networks,
             service_networks=self._config.service_networks,
             machine_networks=machine_networks,
-            api_vip=api_vip,
-            ingress_vip=ingress_vip,
+            api_vips=api_vips,
+            ingress_vips=ingress_vips,
         )
 
     def get_primary_machine_cidr(self):
@@ -441,8 +441,8 @@ def set_advanced_networking(
         cluster_networks: Optional[List[models.ClusterNetwork]] = None,
         service_networks: Optional[List[models.ServiceNetwork]] = None,
         machine_networks: Optional[List[models.MachineNetwork]] = None,
-        api_vip: Optional[str] = None,
-        ingress_vip: Optional[str] = None,
+        api_vips: Optional[List[models.ApiVip]] = None,
+        ingress_vips: Optional[List[models.IngressVip]] = None,
     ):
         if machine_networks is None:
             machine_networks = self._config.machine_networks
@@ -459,13 +459,31 @@ def set_advanced_networking(
             "cluster_networks": cluster_networks if cluster_networks is not None else self._config.cluster_networks,
             "service_networks": service_networks if service_networks is not None else self._config.service_networks,
             "machine_networks": machine_networks,
-            "api_vip": api_vip if api_vip is not None else self._config.api_vip,
-            "ingress_vip": ingress_vip if ingress_vip is not None else self._config.ingress_vip,
+            "api_vips": api_vips if api_vips is not None else self._config.api_vips,
+            "ingress_vips": ingress_vips if ingress_vips is not None else self._config.ingress_vips,
             **extra_vars,
         }
 
         log.info(f"Updating advanced networking with {advanced_networking} for cluster: {self.id}")
 
+        # TODO: Remove singular VIPs once MGMT-14810 gets merged.
+        advanced_networking["api_vip"] = ""
+
+        if (
+            advanced_networking["api_vips"] is not None
+            and len(advanced_networking["api_vips"]) > 0
+            and advanced_networking["api_vips"][0]["ip"] is not None
+        ):
+            advanced_networking["api_vip"] = advanced_networking["api_vips"][0]["ip"]
+
+        advanced_networking["ingress_vip"] = ""
+        if (
+            advanced_networking["ingress_vips"] is not None
+            and len(advanced_networking["ingress_vips"]) > 0
+            and advanced_networking["ingress_vips"][0]["ip"] is not None
+        ):
+            advanced_networking["ingress_vip"] = advanced_networking["ingress_vips"][0]["ip"]
+
         self.update_config(**advanced_networking)
         self.api_client.update_cluster(self.id, advanced_networking)
 
@@ -935,7 +953,9 @@ def prepare_networking(self):
         # in our case when nodes are ready, vips will be there for sure
         if self._ha_not_none():
             vips_info = self.api_client.get_vips_from_cluster(self.id)
-            self.nodes.controller.set_dns(api_ip=vips_info["api_vip"], ingress_ip=vips_info["ingress_vip"])
+            api_ip = vips_info["api_vips"][0].ip if len(vips_info["api_vips"]) > 0 else ""
+            ingress_ip = vips_info["ingress_vips"][0].ip if len(vips_info["ingress_vips"]) > 0 else ""
+            self.nodes.controller.set_dns(api_ip=api_ip, ingress_ip=ingress_ip)
 
     def download_kubeconfig_no_ingress(self, kubeconfig_path: str = None):
         self.api_client.download_kubeconfig_no_ingress(self.id, kubeconfig_path or self._config.kubeconfig_path)

src/assisted_test_infra/test_infra/helper_classes/config/base_cluster_config.py

@@ -25,8 +25,10 @@ class BaseClusterConfig(BaseEntityConfig, ABC):
     machine_networks: List[models.MachineNetwork] = None
     kubeconfig_path: str = None
     network_type: str = None
-    api_vip: str = None
-    ingress_vip: str = None
+    api_vip: str = None  # TODO: Remove singular VIPs once MGMT-14810 gets merged.
+    ingress_vip: str = None  # TODO: Remove singular VIPs once MGMT-14810 gets merged.
+    api_vips: List[models.ApiVip] = None
+    ingress_vips: List[models.IngressVip] = None
     metallb_api_ip: str = None
     metallb_ingress_ip: str = None
     disk_encryption_mode: str = None

src/assisted_test_infra/test_infra/helper_classes/config/base_nodes_config.py

@@ -4,6 +4,8 @@
 from pathlib import Path
 from typing import List, Optional
 
+from assisted_service_client import models
+
 import consts
 
 from .base_config import BaseConfig
@@ -35,8 +37,10 @@ class BaseNodesConfig(BaseConfig, ABC):
     worker_disk_count: int = None
     worker_boot_devices: List[str] = None
 
-    api_vip: Optional[str] = None
-    ingress_vip: Optional[str] = None
+    api_vip: str = None  # TODO: Remove singular VIPs once MGMT-14810 gets merged.
+    ingress_vip: str = None  # TODO: Remove singular VIPs once MGMT-14810 gets merged.
+    api_vips: List[models.ApiVip] = None
+    ingress_vips: List[models.IngressVip] = None
     base_cluster_domain: Optional[str] = None
 
     network_mtu: int = None

src/assisted_test_infra/test_infra/helper_classes/kube_helpers/agent_cluster_install.py

@@ -102,10 +102,10 @@ def set_machinenetwork(self, machine_cidr):
         )
 
     def set_api_vip(self, vip):
-        self.patch(apiVIP=vip)
+        self.patch(apiVIPs=[vip])
 
     def set_ingress_vip(self, vip):
-        self.patch(ingressVIP=vip)
+        self.patch(ingressVIPs=[vip])
 
     def _get_spec_dict(
         self,
@@ -134,11 +134,11 @@ def _get_spec_dict(
             },
         }
 
-        if "api_vip" in kwargs:
-            spec["apiVIP"] = kwargs.pop("api_vip")
+        if "api_vips" in kwargs:
+            spec["apiVIPs"] = kwargs.pop("api_vips")
 
-        if "ingress_vip" in kwargs:
-            spec["ingressVIP"] = kwargs.pop("ingress_vip")
+        if "ingress_vips" in kwargs:
+            spec["ingressVIPs"] = kwargs.pop("ingress_vips")
 
         if "ssh_pub_key" in kwargs:
             spec["sshPublicKey"] = kwargs.pop("ssh_pub_key")

src/service_client/assisted_service_api.py

@@ -473,7 +473,7 @@ def get_hosts_by_role(self, cluster_id: str, role, hosts=None):
 
     def get_api_vip(self, cluster_info: dict, cluster_id: str = None):
         cluster = cluster_info or self.cluster_get(cluster_id)
-        api_vip = cluster.get("api_vip")
+        api_vip = cluster.get("api_vips")[0]["ip"] if len(cluster.get("api_vips")) > 0 else ""
         user_managed_networking = cluster.get("user_managed_networking")
 
         if not api_vip and user_managed_networking:
@@ -560,9 +560,10 @@ def get_ips_for_role(self, cluster_id, network, role) -> List[str]:
                     ret = ret + [intf["ip"]]
         return ret
 
-    def get_vips_from_cluster(self, cluster_id: str) -> Dict[str, str]:
+    def get_vips_from_cluster(self, cluster_id: str) -> Dict[str, Any]:
         cluster_info = self.cluster_get(cluster_id)
-        return dict(api_vip=cluster_info.api_vip, ingress_vip=cluster_info.ingress_vip)
+
+        return dict(api_vips=cluster_info.api_vips, ingress_vips=cluster_info.ingress_vips)
 
     def get_cluster_supported_platforms(self, cluster_id: str) -> List[str]:
         return self.client.get_cluster_supported_platforms(cluster_id)

src/tests/base_test.py

@@ -1083,15 +1083,21 @@ def verify_no_logs_uploaded(cluster, cluster_tar_path):
     @staticmethod
     def update_oc_config(nodes: Nodes, cluster: Cluster):
         os.environ["KUBECONFIG"] = cluster.kubeconfig_path
+        api_vip = ""
 
         if nodes.nodes_count == 1:
-            api_vip = cluster.get_ip_for_single_node(cluster.api_client, cluster.id, cluster.get_primary_machine_cidr())
+            api_vips = [
+                cluster.get_ip_for_single_node(cluster.api_client, cluster.id, cluster.get_primary_machine_cidr())
+            ]
         else:
             vips = nodes.controller.get_ingress_and_api_vips()
-            api_vip = vips["api_vip"]
+            api_vips = vips["api_vips"]
+
+        if len(api_vips) > 0:
+            api_vip = api_vips[0].get("ip", "")
 
         utils.config_etc_hosts(
-            cluster_name=cluster.name, base_dns_domain=global_variables.base_dns_domain, api_vip=api_vip
+            cluster_name=cluster.name, base_dns_domain=global_variables.base_dns_domain, api_vips=api_vip
         )
 
     def wait_for_controller(self, cluster, nodes):

src/tests/test_kube_api.py

@@ -160,8 +160,8 @@ def kube_api_test(
         else:
             # patch the aci with the vips. The cidr will be derived from the range
             access_vips = nodes.controller.get_ingress_and_api_vips()
-            api_vip = access_vips["api_vip"]
-            ingress_vip = access_vips["ingress_vip"]
+            api_vip = access_vips["api_vips"][0].get("ip", "") if len(access_vips["api_vips"]) > 0 else ""
+            ingress_vip = access_vips["ingress_vips"][0].get("ip", "") if len(access_vips["ingress_vips"]) > 0 else ""
 
             agent_cluster_install.set_api_vip(api_vip)
             agent_cluster_install.set_ingress_vip(ingress_vip)

terraform_files/baremetal/main.tf

@@ -181,47 +181,47 @@ data "libvirt_network_dns_host_template" "api" {
   # API VIP is always present. A value is set by the installation flow that updates
   # either the single node IP or API VIP, depending on the scenario
   count    = 1
-  ip       = var.bootstrap_in_place ? var.single_node_ip : var.api_vip
+  ip       = var.bootstrap_in_place ? var.single_node_ip : var.api_vips[0]
   hostname = "api.${local.base_cluster_domain}"
 }
 
 data "libvirt_network_dns_host_template" "api-int" {
   count    = 1
-  ip       = var.bootstrap_in_place ? var.single_node_ip : var.api_vip
+  ip       = var.bootstrap_in_place ? var.single_node_ip : var.api_vips[0]
   hostname = "api-int.${local.base_cluster_domain}"
 }
 
 # TODO: leave only the wildcard address entry defined and remove the other specific DNS assignments
 # Read more at: https://bugzilla.redhat.com/show_bug.cgi?id=1532856
 data "libvirt_network_dnsmasq_options_template" "wildcard-apps-ingress" {
   # Enable "apps" wildcard in case of SNO and when we try to add day2 worker to SNO
-  count        = var.ingress_vip == var.api_vip ? 1 : 0
+  count        = var.ingress_vips == var.api_vips ? 1 : 0
   option_name  = "address"
-  option_value = "/apps.${local.base_cluster_domain}/${var.ingress_vip}"
+  option_value = "/apps.${local.base_cluster_domain}/${var.ingress_vips[0]}"
 }
 
 data "libvirt_network_dns_host_template" "oauth" {
   count    = var.master_count == 1 ? 1 : 0
-  ip       = var.bootstrap_in_place ? var.single_node_ip : var.ingress_vip
+  ip       = var.bootstrap_in_place ? var.single_node_ip : var.ingress_vips[0]
   hostname = "oauth-openshift.apps.${local.base_cluster_domain}"
 }
 
 data "libvirt_network_dns_host_template" "console" {
   count    = var.master_count == 1 ? 1 : 0
-  ip       = var.bootstrap_in_place ? var.single_node_ip : var.ingress_vip
+  ip       = var.bootstrap_in_place ? var.single_node_ip : var.ingress_vips[0]
   hostname = "console-openshift-console.apps.${local.base_cluster_domain}"
 }
 
 data "libvirt_network_dns_host_template" "canary" {
   count    = var.master_count == 1 ? 1 : 0
-  ip       = var.bootstrap_in_place ? var.single_node_ip : var.ingress_vip
+  ip       = var.bootstrap_in_place ? var.single_node_ip : var.ingress_vips[0]
   hostname = "canary-openshift-ingress-canary.apps.${local.base_cluster_domain}"
 }
 
 data "libvirt_network_dns_host_template" "assisted_service" {
   # Ingress VIP is always present. A value is set by the installation flow that updates
   # either the single node IP or API VIP, depending on the scenario
   count    = 1
-  ip       = var.bootstrap_in_place ? var.single_node_ip : var.ingress_vip
+  ip       = var.bootstrap_in_place ? var.single_node_ip : var.ingress_vips[0]
   hostname = "assisted-service-assisted-installer.apps.${local.base_cluster_domain}"
 }

terraform_files/baremetal/variables-libvirt.tf

@@ -118,14 +118,14 @@ variable "libvirt_secondary_worker_macs" {
   description = "the list of the desired macs for secondary worker interface"
 }
 
-variable "api_vip" {
-  type        = string
-  description = "the API virtual IP"
+variable "api_vips" {
+  type        = list(string)
+  description = "the API virtual IPs"
 }
 
-variable "ingress_vip" {
-  type        = string
-  description = "the Ingress virtual IP"
+variable "ingress_vips" {
+  type        = list(string)
+  description = "the Ingress virtual IPs"
 }
 
 # It's definitely recommended to bump this if you can.

terraform_files/none/variables-libvirt.tf

@@ -88,9 +88,9 @@ variable "libvirt_worker_macs" {
   description = "the list of the desired macs for worker interface"
 }
 
-variable "api_vip" {
-  type        = string
-  description = "the API virtual IP"
+variable "api_vips" {
+  type        = list(string)
+  description = "the API virtual IPs"
 }
 
 # It's definitely recommended to bump this if you can.