diff --git a/src/assisted_test_infra/test_infra/controllers/node_controllers/nutanix_controller.py b/src/assisted_test_infra/test_infra/controllers/node_controllers/nutanix_controller.py index d1b20725650..d04fcdd1c04 100644 --- a/src/assisted_test_infra/test_infra/controllers/node_controllers/nutanix_controller.py +++ b/src/assisted_test_infra/test_infra/controllers/node_controllers/nutanix_controller.py @@ -61,7 +61,10 @@ def get_ingress_and_api_vips(self): raise ValueError("API VIP is not set") if not self._entity_config.ingress_vip: raise ValueError("Ingress VIP is not set") - return {"api_vip": self._entity_config.api_vip, "ingress_vip": self._entity_config.ingress_vip} + return { + "api_vips": [{"ip": self._entity_config.api_vip}], + "ingress_vips": [{"ip": self._entity_config.ingress_vip}], + } nutanix_subnet = next( s for s in NutanixSubnet.list_entities(self._provider_client) if s.name == self._config.nutanix_subnet @@ -88,7 +91,7 @@ def get_ingress_and_api_vips(self): raise ConnectionError("Failed to locate free API and Ingress VIPs") log.info(f"Found 2 optional VIPs: {free_ips}") - return {"api_vip": free_ips.pop(), "ingress_vip": free_ips.pop()} + return {"api_vips": [free_ips.pop()], "ingress_vips": [free_ips.pop()]} def set_boot_order(self, node_name, cd_first=False, cdrom_iso_path=None) -> None: vm = self._get_provider_vm(tf_vm_name=node_name) diff --git a/src/assisted_test_infra/test_infra/controllers/node_controllers/terraform_controller.py b/src/assisted_test_infra/test_infra/controllers/node_controllers/terraform_controller.py index e88cfcde1d6..e698faace1a 100644 --- a/src/assisted_test_infra/test_infra/controllers/node_controllers/terraform_controller.py +++ b/src/assisted_test_infra/test_infra/controllers/node_controllers/terraform_controller.py @@ -154,6 +154,20 @@ def _create_nodes(self, running=True): if self._params.running: self.wait_till_nodes_are_ready(network_name=self._params.libvirt_network_name) + def _get_vips(self): + vips = self.get_ingress_and_api_vips() + if self._config.api_vips: + api_vips = [i.get("ip") for i in self._config.api_vips] + else: + api_vips = [i.get("ip") for i in vips["api_vips"]] + + if self._config.ingress_vips: + ingress_vips = [i.get("ip") for i in self._config.ingress_vips] + else: + ingress_vips = [i.get("ip") for i in vips["ingress_vips"]] + + return api_vips, ingress_vips + # Filling tfvars json files with terraform needed variables to spawn vms def _fill_tfvars(self, running=True): log.info("Filling tfvars") @@ -183,9 +197,12 @@ def _fill_tfvars(self, running=True): tfvars["provisioning_cidr_addresses"] = self.get_all_provisioning_addresses() tfvars["bootstrap_in_place"] = self._config.bootstrap_in_place - vips = self.get_ingress_and_api_vips() - tfvars["api_vip"] = self._config.api_vip or vips["api_vip"] - tfvars["ingress_vip"] = self._config.ingress_vip or vips["ingress_vip"] + tfvars["api_vips"], tfvars["ingress_vips"] = self._get_vips() + + # TODO: Remove singular VIPs once MGMT-14810 gets merged. + tfvars["api_vip"] = tfvars["api_vips"][0] + tfvars["ingress_vip"] = tfvars["ingress_vips"][0] + if self._config.base_cluster_domain: tfvars["base_cluster_domain"] = self._config.base_cluster_domain @@ -233,17 +250,17 @@ def format_node_disk(self, node_name: str, disk_index: int = 0): log.info("Formatting disk for %s", node_name) self.format_disk(f"{self._params.libvirt_storage_pool_path}/{self.entity_name}/{node_name}-disk-{disk_index}") - def get_ingress_and_api_vips(self) -> Dict[str, str]: + def get_ingress_and_api_vips(self) -> Dict[str, List[dict]]: """Pick two IPs for setting static access endpoint IPs. - Using .100 for the API endpoint and .101 for the ingress endpoint + Using .100 for the API endpoint and .101 for the ingress endpoint (the IPv6 values are appropriately :64 and :65). This method is not applicable for SNO clusters, where access IPs should be the node's IP. """ network_subnet_starting_ip = ip_address(ip_network(self.get_primary_machine_cidr()).network_address) ips = utils.create_ip_address_list(2, starting_ip_addr=network_subnet_starting_ip + 100) - return {"api_vip": ips[0], "ingress_vip": ips[1]} + return {"api_vips": [{"ip": ips[0]}], "ingress_vips": [{"ip": ips[1]}]} @utils.on_exception(message="Failed to run terraform delete", silent=True) def _create_address_list(self, num, starting_ip_addr): diff --git a/src/assisted_test_infra/test_infra/controllers/node_controllers/tf_controller.py b/src/assisted_test_infra/test_infra/controllers/node_controllers/tf_controller.py index 8eacf8b77ce..c0a816d9168 100644 --- a/src/assisted_test_infra/test_infra/controllers/node_controllers/tf_controller.py +++ b/src/assisted_test_infra/test_infra/controllers/node_controllers/tf_controller.py @@ -109,9 +109,12 @@ def set_dns(self, api_ip: str, ingress_ip: str) -> None: ingress_ip=ingress_ip, ) - def get_ingress_and_api_vips(self) -> Optional[Dict[str, str]]: + def get_ingress_and_api_vips(self) -> dict[str, list[dict]] | None: if self._entity_config.api_vip and self._entity_config.ingress_vip: - return {"api_vip": self._entity_config.api_vip, "ingress_vip": self._entity_config.ingress_vip} + return { + "api_vips": [{"ip": self._entity_config.api_vip}], + "ingress_vips": [{"ip": self._entity_config.ingress_vip}], + } return None diff --git a/src/assisted_test_infra/test_infra/helper_classes/cluster.py b/src/assisted_test_infra/test_infra/helper_classes/cluster.py index 1fa4e3884f6..d7d30e5c644 100644 --- a/src/assisted_test_infra/test_infra/helper_classes/cluster.py +++ b/src/assisted_test_infra/test_infra/helper_classes/cluster.py @@ -357,18 +357,18 @@ def set_network_params(self, controller=None): if self._config.platform == consts.Platforms.NONE: log.info("On None platform, leaving network management to the user") - api_vip = ingress_vip = machine_networks = None + api_vips = ingress_vips = machine_networks = None elif self._config.vip_dhcp_allocation or self._high_availability_mode == consts.HighAvailabilityMode.NONE: log.info("Letting access VIPs be deducted from machine networks") - api_vip = ingress_vip = None + api_vips = ingress_vips = None machine_networks = [self.get_machine_networks()[0]] else: log.info("Assigning VIPs statically") access_vips = controller.get_ingress_and_api_vips() - api_vip = access_vips["api_vip"] if access_vips else None - ingress_vip = access_vips["ingress_vip"] if access_vips else None + api_vips = access_vips["api_vips"] if access_vips else None + ingress_vips = access_vips["ingress_vips"] if access_vips else None machine_networks = None if self._config.is_ipv4 and self._config.is_ipv6: @@ -379,8 +379,8 @@ def set_network_params(self, controller=None): cluster_networks=self._config.cluster_networks, service_networks=self._config.service_networks, machine_networks=machine_networks, - api_vip=api_vip, - ingress_vip=ingress_vip, + api_vips=api_vips, + ingress_vips=ingress_vips, ) def get_primary_machine_cidr(self): @@ -441,8 +441,8 @@ def set_advanced_networking( cluster_networks: Optional[List[models.ClusterNetwork]] = None, service_networks: Optional[List[models.ServiceNetwork]] = None, machine_networks: Optional[List[models.MachineNetwork]] = None, - api_vip: Optional[str] = None, - ingress_vip: Optional[str] = None, + api_vips: Optional[List[models.ApiVip]] = None, + ingress_vips: Optional[List[models.IngressVip]] = None, ): if machine_networks is None: machine_networks = self._config.machine_networks @@ -459,13 +459,31 @@ def set_advanced_networking( "cluster_networks": cluster_networks if cluster_networks is not None else self._config.cluster_networks, "service_networks": service_networks if service_networks is not None else self._config.service_networks, "machine_networks": machine_networks, - "api_vip": api_vip if api_vip is not None else self._config.api_vip, - "ingress_vip": ingress_vip if ingress_vip is not None else self._config.ingress_vip, + "api_vips": api_vips if api_vips is not None else self._config.api_vips, + "ingress_vips": ingress_vips if ingress_vips is not None else self._config.ingress_vips, **extra_vars, } log.info(f"Updating advanced networking with {advanced_networking} for cluster: {self.id}") + # TODO: Remove singular VIPs once MGMT-14810 gets merged. + advanced_networking["api_vip"] = "" + + if ( + advanced_networking["api_vips"] is not None + and len(advanced_networking["api_vips"]) > 0 + and advanced_networking["api_vips"][0]["ip"] is not None + ): + advanced_networking["api_vip"] = advanced_networking["api_vips"][0]["ip"] + + advanced_networking["ingress_vip"] = "" + if ( + advanced_networking["ingress_vips"] is not None + and len(advanced_networking["ingress_vips"]) > 0 + and advanced_networking["ingress_vips"][0]["ip"] is not None + ): + advanced_networking["ingress_vip"] = advanced_networking["ingress_vips"][0]["ip"] + self.update_config(**advanced_networking) self.api_client.update_cluster(self.id, advanced_networking) @@ -935,7 +953,9 @@ def prepare_networking(self): # in our case when nodes are ready, vips will be there for sure if self._ha_not_none(): vips_info = self.api_client.get_vips_from_cluster(self.id) - self.nodes.controller.set_dns(api_ip=vips_info["api_vip"], ingress_ip=vips_info["ingress_vip"]) + api_ip = vips_info["api_vips"][0].ip if len(vips_info["api_vips"]) > 0 else "" + ingress_ip = vips_info["ingress_vips"][0].ip if len(vips_info["ingress_vips"]) > 0 else "" + self.nodes.controller.set_dns(api_ip=api_ip, ingress_ip=ingress_ip) def download_kubeconfig_no_ingress(self, kubeconfig_path: str = None): self.api_client.download_kubeconfig_no_ingress(self.id, kubeconfig_path or self._config.kubeconfig_path) diff --git a/src/assisted_test_infra/test_infra/helper_classes/config/base_cluster_config.py b/src/assisted_test_infra/test_infra/helper_classes/config/base_cluster_config.py index 40e4a29d07a..9583a954c62 100644 --- a/src/assisted_test_infra/test_infra/helper_classes/config/base_cluster_config.py +++ b/src/assisted_test_infra/test_infra/helper_classes/config/base_cluster_config.py @@ -25,8 +25,10 @@ class BaseClusterConfig(BaseEntityConfig, ABC): machine_networks: List[models.MachineNetwork] = None kubeconfig_path: str = None network_type: str = None - api_vip: str = None - ingress_vip: str = None + api_vip: str = None # TODO: Remove singular VIPs once MGMT-14810 gets merged. + ingress_vip: str = None # TODO: Remove singular VIPs once MGMT-14810 gets merged. + api_vips: List[models.ApiVip] = None + ingress_vips: List[models.IngressVip] = None metallb_api_ip: str = None metallb_ingress_ip: str = None disk_encryption_mode: str = None diff --git a/src/assisted_test_infra/test_infra/helper_classes/config/base_nodes_config.py b/src/assisted_test_infra/test_infra/helper_classes/config/base_nodes_config.py index 4bbc2f94da6..572d0d7fcb3 100644 --- a/src/assisted_test_infra/test_infra/helper_classes/config/base_nodes_config.py +++ b/src/assisted_test_infra/test_infra/helper_classes/config/base_nodes_config.py @@ -4,6 +4,8 @@ from pathlib import Path from typing import List, Optional +from assisted_service_client import models + import consts from .base_config import BaseConfig @@ -35,8 +37,10 @@ class BaseNodesConfig(BaseConfig, ABC): worker_disk_count: int = None worker_boot_devices: List[str] = None - api_vip: Optional[str] = None - ingress_vip: Optional[str] = None + api_vip: str = None # TODO: Remove singular VIPs once MGMT-14810 gets merged. + ingress_vip: str = None # TODO: Remove singular VIPs once MGMT-14810 gets merged. + api_vips: List[models.ApiVip] = None + ingress_vips: List[models.IngressVip] = None base_cluster_domain: Optional[str] = None network_mtu: int = None diff --git a/src/assisted_test_infra/test_infra/helper_classes/kube_helpers/agent_cluster_install.py b/src/assisted_test_infra/test_infra/helper_classes/kube_helpers/agent_cluster_install.py index db7e5e8b339..ec8f81e74d6 100644 --- a/src/assisted_test_infra/test_infra/helper_classes/kube_helpers/agent_cluster_install.py +++ b/src/assisted_test_infra/test_infra/helper_classes/kube_helpers/agent_cluster_install.py @@ -102,10 +102,10 @@ def set_machinenetwork(self, machine_cidr): ) def set_api_vip(self, vip): - self.patch(apiVIP=vip) + self.patch(apiVIPs=[vip]) def set_ingress_vip(self, vip): - self.patch(ingressVIP=vip) + self.patch(ingressVIPs=[vip]) def _get_spec_dict( self, @@ -134,11 +134,11 @@ def _get_spec_dict( }, } - if "api_vip" in kwargs: - spec["apiVIP"] = kwargs.pop("api_vip") + if "api_vips" in kwargs: + spec["apiVIPs"] = kwargs.pop("api_vips") - if "ingress_vip" in kwargs: - spec["ingressVIP"] = kwargs.pop("ingress_vip") + if "ingress_vips" in kwargs: + spec["ingressVIPs"] = kwargs.pop("ingress_vips") if "ssh_pub_key" in kwargs: spec["sshPublicKey"] = kwargs.pop("ssh_pub_key") diff --git a/src/service_client/assisted_service_api.py b/src/service_client/assisted_service_api.py index 14b110fdea1..14f7f2a8044 100644 --- a/src/service_client/assisted_service_api.py +++ b/src/service_client/assisted_service_api.py @@ -473,7 +473,7 @@ def get_hosts_by_role(self, cluster_id: str, role, hosts=None): def get_api_vip(self, cluster_info: dict, cluster_id: str = None): cluster = cluster_info or self.cluster_get(cluster_id) - api_vip = cluster.get("api_vip") + api_vip = cluster.get("api_vips")[0]["ip"] if len(cluster.get("api_vips")) > 0 else "" user_managed_networking = cluster.get("user_managed_networking") if not api_vip and user_managed_networking: @@ -560,9 +560,10 @@ def get_ips_for_role(self, cluster_id, network, role) -> List[str]: ret = ret + [intf["ip"]] return ret - def get_vips_from_cluster(self, cluster_id: str) -> Dict[str, str]: + def get_vips_from_cluster(self, cluster_id: str) -> Dict[str, Any]: cluster_info = self.cluster_get(cluster_id) - return dict(api_vip=cluster_info.api_vip, ingress_vip=cluster_info.ingress_vip) + + return dict(api_vips=cluster_info.api_vips, ingress_vips=cluster_info.ingress_vips) def get_cluster_supported_platforms(self, cluster_id: str) -> List[str]: return self.client.get_cluster_supported_platforms(cluster_id) diff --git a/src/tests/base_test.py b/src/tests/base_test.py index d8e456a964b..2a835c6e555 100644 --- a/src/tests/base_test.py +++ b/src/tests/base_test.py @@ -1083,15 +1083,21 @@ def verify_no_logs_uploaded(cluster, cluster_tar_path): @staticmethod def update_oc_config(nodes: Nodes, cluster: Cluster): os.environ["KUBECONFIG"] = cluster.kubeconfig_path + api_vip = "" if nodes.nodes_count == 1: - api_vip = cluster.get_ip_for_single_node(cluster.api_client, cluster.id, cluster.get_primary_machine_cidr()) + api_vips = [ + cluster.get_ip_for_single_node(cluster.api_client, cluster.id, cluster.get_primary_machine_cidr()) + ] else: vips = nodes.controller.get_ingress_and_api_vips() - api_vip = vips["api_vip"] + api_vips = vips["api_vips"] + + if len(api_vips) > 0: + api_vip = api_vips[0].get("ip", "") utils.config_etc_hosts( - cluster_name=cluster.name, base_dns_domain=global_variables.base_dns_domain, api_vip=api_vip + cluster_name=cluster.name, base_dns_domain=global_variables.base_dns_domain, api_vips=api_vip ) def wait_for_controller(self, cluster, nodes): diff --git a/src/tests/test_kube_api.py b/src/tests/test_kube_api.py index 03257fa586b..eb6206fdc5b 100644 --- a/src/tests/test_kube_api.py +++ b/src/tests/test_kube_api.py @@ -160,8 +160,8 @@ def kube_api_test( else: # patch the aci with the vips. The cidr will be derived from the range access_vips = nodes.controller.get_ingress_and_api_vips() - api_vip = access_vips["api_vip"] - ingress_vip = access_vips["ingress_vip"] + api_vip = access_vips["api_vips"][0].get("ip", "") if len(access_vips["api_vips"]) > 0 else "" + ingress_vip = access_vips["ingress_vips"][0].get("ip", "") if len(access_vips["ingress_vips"]) > 0 else "" agent_cluster_install.set_api_vip(api_vip) agent_cluster_install.set_ingress_vip(ingress_vip) diff --git a/terraform_files/baremetal/main.tf b/terraform_files/baremetal/main.tf index 33f93ef9088..543676198e2 100644 --- a/terraform_files/baremetal/main.tf +++ b/terraform_files/baremetal/main.tf @@ -181,13 +181,13 @@ data "libvirt_network_dns_host_template" "api" { # API VIP is always present. A value is set by the installation flow that updates # either the single node IP or API VIP, depending on the scenario count = 1 - ip = var.bootstrap_in_place ? var.single_node_ip : var.api_vip + ip = var.bootstrap_in_place ? var.single_node_ip : var.api_vips[0] hostname = "api.${local.base_cluster_domain}" } data "libvirt_network_dns_host_template" "api-int" { count = 1 - ip = var.bootstrap_in_place ? var.single_node_ip : var.api_vip + ip = var.bootstrap_in_place ? var.single_node_ip : var.api_vips[0] hostname = "api-int.${local.base_cluster_domain}" } @@ -195,26 +195,26 @@ data "libvirt_network_dns_host_template" "api-int" { # Read more at: https://bugzilla.redhat.com/show_bug.cgi?id=1532856 data "libvirt_network_dnsmasq_options_template" "wildcard-apps-ingress" { # Enable "apps" wildcard in case of SNO and when we try to add day2 worker to SNO - count = var.ingress_vip == var.api_vip ? 1 : 0 + count = var.ingress_vips == var.api_vips ? 1 : 0 option_name = "address" - option_value = "/apps.${local.base_cluster_domain}/${var.ingress_vip}" + option_value = "/apps.${local.base_cluster_domain}/${var.ingress_vips[0]}" } data "libvirt_network_dns_host_template" "oauth" { count = var.master_count == 1 ? 1 : 0 - ip = var.bootstrap_in_place ? var.single_node_ip : var.ingress_vip + ip = var.bootstrap_in_place ? var.single_node_ip : var.ingress_vips[0] hostname = "oauth-openshift.apps.${local.base_cluster_domain}" } data "libvirt_network_dns_host_template" "console" { count = var.master_count == 1 ? 1 : 0 - ip = var.bootstrap_in_place ? var.single_node_ip : var.ingress_vip + ip = var.bootstrap_in_place ? var.single_node_ip : var.ingress_vips[0] hostname = "console-openshift-console.apps.${local.base_cluster_domain}" } data "libvirt_network_dns_host_template" "canary" { count = var.master_count == 1 ? 1 : 0 - ip = var.bootstrap_in_place ? var.single_node_ip : var.ingress_vip + ip = var.bootstrap_in_place ? var.single_node_ip : var.ingress_vips[0] hostname = "canary-openshift-ingress-canary.apps.${local.base_cluster_domain}" } @@ -222,6 +222,6 @@ data "libvirt_network_dns_host_template" "assisted_service" { # Ingress VIP is always present. A value is set by the installation flow that updates # either the single node IP or API VIP, depending on the scenario count = 1 - ip = var.bootstrap_in_place ? var.single_node_ip : var.ingress_vip + ip = var.bootstrap_in_place ? var.single_node_ip : var.ingress_vips[0] hostname = "assisted-service-assisted-installer.apps.${local.base_cluster_domain}" } diff --git a/terraform_files/baremetal/variables-libvirt.tf b/terraform_files/baremetal/variables-libvirt.tf index b4542a2fd62..99ab4422234 100644 --- a/terraform_files/baremetal/variables-libvirt.tf +++ b/terraform_files/baremetal/variables-libvirt.tf @@ -118,14 +118,14 @@ variable "libvirt_secondary_worker_macs" { description = "the list of the desired macs for secondary worker interface" } -variable "api_vip" { - type = string - description = "the API virtual IP" +variable "api_vips" { + type = list(string) + description = "the API virtual IPs" } -variable "ingress_vip" { - type = string - description = "the Ingress virtual IP" +variable "ingress_vips" { + type = list(string) + description = "the Ingress virtual IPs" } # It's definitely recommended to bump this if you can. diff --git a/terraform_files/none/variables-libvirt.tf b/terraform_files/none/variables-libvirt.tf index 022fff53364..bfff9efb1fd 100644 --- a/terraform_files/none/variables-libvirt.tf +++ b/terraform_files/none/variables-libvirt.tf @@ -88,9 +88,9 @@ variable "libvirt_worker_macs" { description = "the list of the desired macs for worker interface" } -variable "api_vip" { - type = string - description = "the API virtual IP" +variable "api_vips" { + type = list(string) + description = "the API virtual IPs" } # It's definitely recommended to bump this if you can.