Support REST API dual-stack

A lot of changes in order to support `IPv4=true IPv6=true`.

- Remove env vars for SERVICE_CIDR, CLUSTER_CIDR, CLUSTER_HOST_PREFIX as
  no one uses them.
- Support sending multiple values for `cluster_networks`,
  `service_networks`, and `machine_networks`.
- Add constants for IPv4, IPv6, and IPv4v6 subnets.
- Change controllers signatures from `get_machine_cidr` to
  `get_machine_networks`.
- Support provisioning multiple networks via Terraform.
  Create a `libvirt_network` resource and a dynamic `network_interface`
  for each machine network available.
- Deprecate `provisioning_cidr` network as it is mostly confusing and we
  would like to be allowed to add/remove networks without indication
  whether they are primary/secondary in Terraform.
- Change assets to work with lists of two fields - `cidrs` and
  `libvirt_network_interfaces`.
  Interfaces are named `ai-net<x>-<number>`. The `x` represents
  the network number and the `number` represents the namespace index.

discovery-infra/delete_nodes.py

@@ -39,19 +39,16 @@ def try_to_delete_cluster(namespace, tfvars):
     client.delete_cluster(cluster_id=cluster_id)
 
 
-def _get_namespace_index(libvirt_network_if):
+def _get_namespace_index(libvirt_network_interface):
     # Hack to retrieve namespace index - does not exist in tests
-    matcher = re.match(r'^tt(\d+)$', libvirt_network_if)
+    matcher = re.match(r'^.+-(\d+)$', libvirt_network_interface)
     return int(matcher.groups()[0]) if matcher is not None else 0
 
 
 @utils.on_exception(message='Failed to remove nat', silent=True)
 def _try_remove_nat(tfvars):
-    primary_interface = tfvars.get('libvirt_network_if')
-    if primary_interface is None:
-        raise Exception("Could not get primary interface")
-    secondary_interface = tfvars.get('libvirt_secondary_network_if', f's{primary_interface}')
-    nat_controller = NatController([primary_interface, secondary_interface], _get_namespace_index(primary_interface))
+    interfaces = tfvars['libvirt_network_interfaces']
+    nat_controller = NatController(interfaces, _get_namespace_index(interfaces[0]))
     nat_controller.remove_nat_rules()
 
 
@@ -64,11 +61,9 @@ def delete_nodes(cluster_name, namespace, tf_folder, tfvars):
         _try_to_delete_nodes(tf_folder)
 
     default_network_name = consts.TEST_NETWORK + namespace
-    default_sec_network_name = consts.TEST_SECONDARY_NETWORK + namespace
     _delete_virsh_resources(
         tfvars.get('cluster_name', cluster_name),
         tfvars.get('libvirt_network_name', default_network_name),
-        tfvars.get('libvirt_secondary_network_name', default_sec_network_name),
     )
     if os.path.exists(tf_folder):
         log.info('Deleting %s', tf_folder)
@@ -228,6 +223,4 @@ def main():
 
     oc_utils.extend_parser_with_oc_arguments(parser)
     args = parser.parse_args()
-    if not args.kube_api:
-        args.namespace = ""
     main()

discovery-infra/deprecated_utils.py

@@ -125,19 +125,4 @@ def get_all_namespaced_clusters():
         return
 
     for dirname in os.listdir(consts.TF_FOLDER):
-        res = get_name_and_namespace_from_dirname(dirname)
-        if not res:
-            continue
-        name, namespace = res
-        yield name, namespace
-
-
-def get_name_and_namespace_from_dirname(dirname):
-    if "__" in dirname:
-        return dirname.rsplit("__", 1)
-
-    log.warning(
-        "Unable to extract cluster name and namespace from directory name %s. "
-        "Directory name convention must be <cluster_name>:<namespace>",
-        dirname,
-    )
+        yield dirname, dirname.split('-')[-1]

discovery-infra/start_discovery.py

@@ -132,8 +132,8 @@ def fill_tfvars(
     tfvars['provisioning_cidr_addresses'] = machine_net.provisioning_cidr_addresses
     tfvars['api_vip'] = _get_vips_ips(machine_net)[0]
     tfvars['libvirt_storage_pool_path'] = storage_path
-    tfvars['libvirt_master_macs'] = static_network.generate_macs(master_count)
-    tfvars['libvirt_worker_macs'] = static_network.generate_macs(worker_count)
+    tfvars['libvirt_master_macs'] = [static_network.generate_macs(master_count)]
+    tfvars['libvirt_worker_macs'] = [static_network.generate_macs(worker_count)]
     tfvars.update(nodes_details)
 
     tfvars.update(_secondary_tfvars(master_count, nodes_details, machine_net))
@@ -373,8 +373,6 @@ def _create_node_details(cluster_name):
         "libvirt_network_if": args.network_bridge,
         "libvirt_worker_disk": args.worker_disk,
         "libvirt_master_disk": args.master_disk,
-        "libvirt_secondary_network_name": consts.TEST_SECONDARY_NETWORK + args.namespace,
-        "libvirt_secondary_network_if": f's{args.network_bridge}',
         "bootstrap_in_place": args.master_count == 1,
         "master_disk_count": args.master_disk_count,
         "worker_disk_count": args.worker_disk_count,
@@ -635,23 +633,19 @@ def set_single_node_ip(
 
 
 def set_hosts_roles(client, cluster, nodes_details, machine_net, tf, master_count, static_network_mode):
-    networks_names = (
-        nodes_details["libvirt_network_name"],
-        nodes_details["libvirt_secondary_network_name"]
-    )
+    networks_name = nodes_details["libvirt_network_name"]
 
     # don't set roles in bip role
     if not machine_net.has_ip_v6:
-        libvirt_nodes = get_libvirt_nodes_mac_role_ip_and_name(networks_names[0])
-        libvirt_nodes.update(get_libvirt_nodes_mac_role_ip_and_name(networks_names[1]))
+        libvirt_nodes = get_libvirt_nodes_mac_role_ip_and_name(networks_name)
         if static_network_mode:
             log.info("Setting hostnames when running in static network config mode")
             update_hostnames = True
         else:
             update_hostnames = False
     else:
         log.warning("Work around libvirt for Terrafrom not setting hostnames of IPv6 hosts")
-        libvirt_nodes = utils.get_libvirt_nodes_from_tf_state(networks_names, tf.get_state())
+        libvirt_nodes = utils.get_libvirt_nodes_from_tf_state(networks_name, tf.get_state())
         update_hostnames = True
 
     utils.update_hosts(client, cluster.id, libvirt_nodes, update_hostnames=update_hostnames,

discovery-infra/test_infra/consts/consts.py

@@ -1,7 +1,10 @@
 from enum import Enum
+from typing import List
 
 from .durations import MINUTE, HOUR
 
+from assisted_service_client import models
+
 
 class OpenshiftVersion(Enum):
     VERSION_4_6 = "4.6"
@@ -15,6 +18,7 @@ class NetworkType:
     OVNKubernetes = "OVNKubernetes"
 
 
+# Files & Directories
 WORKING_DIR = "build"
 TF_FOLDER = f"{WORKING_DIR}/terraform"
 TFVARS_JSON_NAME = "terraform.tfvars.json"
@@ -23,9 +27,16 @@ class NetworkType:
 BASE_IMAGE_FOLDER = "/tmp/images"
 IMAGE_NAME = "installer-image.iso"
 STORAGE_PATH = "/var/lib/libvirt/openshift-images"
-HOST_PASSTHROUGH_CPU_MODE = "host-passthrough"
-MASTER_TF_CPU_MODE = HOST_PASSTHROUGH_CPU_MODE
-WORKER_TF_CPU_MODE = HOST_PASSTHROUGH_CPU_MODE
+DEFAULT_CLUSTER_KUBECONFIG_DIR_PATH = "build/kubeconfig"
+OCP_VERSIONS_JSON_PATH = "assisted-service/data/default_ocp_versions.json"
+
+TF_TEMPLATES_ROOT = "terraform_files"
+TF_TEMPLATE_BARE_METAL_FLOW = f"{TF_TEMPLATES_ROOT}/baremetal"
+TF_TEMPLATE_NONE_PLATFORM_FLOW = f"{TF_TEMPLATES_ROOT}/none"
+TF_TEMPLATE_BARE_METAL_INFRA_ENV_FLOW = f"{TF_TEMPLATES_ROOT}/baremetal_infra_env"
+TF_NETWORK_POOL_PATH = "/tmp/tf_network_pool.json"
+
+# Timeouts
 NODES_REGISTERED_TIMEOUT = 20 * MINUTE
 DEFAULT_CHECK_STATUSES_INTERVAL = 5
 CLUSTER_READY_FOR_INSTALL_TIMEOUT = 10 * MINUTE
@@ -41,19 +52,35 @@ class NetworkType:
 DISCONNECTED_TIMEOUT = 10 * MINUTE
 PENDING_USER_ACTION_TIMEOUT = 30 * MINUTE
 ERROR_TIMEOUT = 10 * MINUTE
-TF_TEMPLATES_ROOT = "terraform_files"
-TF_TEMPLATE_BARE_METAL_FLOW = f"{TF_TEMPLATES_ROOT}/baremetal"
-TF_TEMPLATE_NONE_PLATFORM_FLOW = f"{TF_TEMPLATES_ROOT}/none"
-TF_TEMPLATE_BARE_METAL_INFRA_ENV_FLOW = f"{TF_TEMPLATES_ROOT}/baremetal_infra_env"
-TF_NETWORK_POOL_PATH = "/tmp/tf_network_pool.json"
-NUMBER_OF_MASTERS = 3
+WAIT_FOR_BM_API = 15 * MINUTE
+
+# Networking
+DEFAULT_CLUSTER_NETWORKS_IPV4: List[models.ClusterNetwork] = [models.ClusterNetwork(cidr="172.30.0.0/16", host_prefix=23)]
+DEFAULT_SERVICE_NETWORKS_IPV4: List[models.ServiceNetwork] = [models.ServiceNetwork(cidr="10.128.0.0/14")]
+DEFAULT_MACHINE_NETWORKS_IPV4: List[models.MachineNetwork] = \
+    [models.MachineNetwork(cidr="192.168.127.0/24"), models.MachineNetwork(cidr="192.168.145.0/24")]
+
+DEFAULT_CLUSTER_NETWORKS_IPV6: List[models.ClusterNetwork] = [models.ClusterNetwork(cidr="2002:db8::/53", host_prefix=64)]
+DEFAULT_SERVICE_NETWORKS_IPV6: List[models.ServiceNetwork] = [models.ServiceNetwork(cidr="2003:db8::/112")]
+DEFAULT_MACHINE_NETWORKS_IPV6: List[models.MachineNetwork] = \
+    [models.MachineNetwork(cidr="1001:db9::/120"), models.MachineNetwork(cidr="3001:db9::/120")]
+
+DEFAULT_CLUSTER_NETWORKS_IPV4V6 = DEFAULT_CLUSTER_NETWORKS_IPV4 + DEFAULT_CLUSTER_NETWORKS_IPV6
+DEFAULT_SERVICE_NETWORKS_IPV4V6 = DEFAULT_SERVICE_NETWORKS_IPV4 + DEFAULT_SERVICE_NETWORKS_IPV6
+DEFAULT_MACHINE_NETWORKS_IPV4V6 = [DEFAULT_MACHINE_NETWORKS_IPV4[0], DEFAULT_MACHINE_NETWORKS_IPV6[0]]
+
+DEFAULT_PROXY_SERVER_PORT = 3129
+DEFAULT_LOAD_BALANCER_PORT = 6443
+
 TEST_INFRA = "test-infra"
 CLUSTER = CLUSTER_PREFIX = "%s-cluster" % TEST_INFRA
 INFRA_ENV_PREFIX = "%s-infra-env" % TEST_INFRA
 TEST_NETWORK = "test-infra-net-"
-TEST_SECONDARY_NETWORK = "test-infra-secondary-network-"
-DEFAULT_CLUSTER_KUBECONFIG_DIR_PATH = "build/kubeconfig"
-WAIT_FOR_BM_API = 15 * MINUTE
+
+HOST_PASSTHROUGH_CPU_MODE = "host-passthrough"
+MASTER_TF_CPU_MODE = HOST_PASSTHROUGH_CPU_MODE
+WORKER_TF_CPU_MODE = HOST_PASSTHROUGH_CPU_MODE
+NUMBER_OF_MASTERS = 3
 NAMESPACE_POOL_SIZE = 15
 PODMAN_FLAGS = "--cgroup-manager=cgroupfs --storage-driver=vfs --events-backend=file"
 DEFAULT_ADDITIONAL_NTP_SOURCE = "clock.redhat.com"
@@ -63,27 +90,6 @@ class NetworkType:
 DEFAULT_TEST_INFRA_DOMAIN = f".{CLUSTER_PREFIX}-{DEFAULT_NAMESPACE}.{DEFAULT_BASE_DNS_DOMAIN}"
 TEST_TARGET_INTERFACE = "vnet3"
 SUFFIX_LENGTH = 8
-OCP_VERSIONS_JSON_PATH = "assisted-service/data/default_ocp_versions.json"
-
-DEFAULT_IPV6_SERVICE_CIDR = "2003:db8::/112"
-DEFAULT_IPV6_CLUSTER_CIDR = "2002:db8::/53"
-DEFAULT_IPV6_HOST_PREFIX = 64
-DEFAULT_PROXY_SERVER_PORT = 3129
-DEFAULT_LOAD_BALANCER_PORT = 6443
-
-IP_NETWORK_ASSET_FIELDS = (
-    "machine_cidr",
-    "machine_cidr6",
-    "provisioning_cidr",
-    "provisioning_cidr6",
-)
-REQUIRED_ASSET_FIELDS = (
-    "libvirt_network_if",
-    "libvirt_secondary_network_if",
-    *IP_NETWORK_ASSET_FIELDS,
-)
-
-
 class ImageType:
     FULL_ISO = "full-iso"
     MINIMAL_ISO = "minimal-iso"
@@ -212,9 +218,15 @@ class HighAvailabilityMode:
 
 
 class BaseAsset:
-    MACHINE_CIDR = "192.168.127.0/24"
-    MACHINE_CIDR6 = "1001:db9::/120"
-    PROVISIONING_CIDR = "192.168.145.0/24"
-    PROVISIONING_CIDR6 = "3001:db9::/120"
-    NETWORK_IF = "tt1"
-    SECONDARY_NETWORK_IF = "stt1"
+    ASSET_FIELD_CIDRS = "cidrs"
+    ASSET_FIELD_INTERFACES = "libvirt_network_interfaces"
+    REQUIRED_ASSET_FIELDS = (
+        ASSET_FIELD_CIDRS,
+        ASSET_FIELD_INTERFACES,
+    )
+
+    CIDRS = [network.cidr for network in DEFAULT_MACHINE_NETWORKS_IPV4 + DEFAULT_MACHINE_NETWORKS_IPV6]
+
+    # TODO: interfaces should be generated according to the CIDRS
+    # Bridge name needs to be less or equal to 15 characters
+    NETWORK_INTERFACES = ["ai-net1-0", "ai-net2-0", "ai-net3-0", "ai-net4-0"]

discovery-infra/test_infra/consts/env_defaults.py

@@ -10,9 +10,6 @@
 DEFAULT_SSH_PUBLIC_KEY_PATH: Path = Path.home() / ".ssh" / "id_rsa.pub"
 DEFAULT_INSTALLER_KUBECONFIG = None
 DEFAULT_LOG_FOLDER: Path = Path("/tmp/assisted_test_infra_logs")
-DEFAULT_SERVICE_CIDR: str = "172.30.0.0/16"
-DEFAULT_CLUSTER_CIDR: str = "10.128.0.0/14"
-DEFAULT_HOST_PREFIX: int = 23
 DEFAULT_IMAGE_TYPE: str = consts.ImageType.FULL_ISO
 DEFAULT_TEST_TEARDOWN: bool = True
 DEFAULT_PLATFORM: str = consts.Platforms.BARE_METAL

discovery-infra/test_infra/controllers/nat_controller.py

@@ -36,9 +36,9 @@ def remove_nat_rules(self) -> None:
             self._remove_rule(self._build_nat_string(output_interface))
 
     @classmethod
-    def get_namespace_index(cls, libvirt_network_if):
+    def get_namespace_index(cls, libvirt_network_interface):
         """ Hack to retrieve namespace index - does not exist in tests """
-        matcher = re.match(r'^tt(\d+)$', libvirt_network_if)
+        matcher = re.match(r'^tt(\d+)$', libvirt_network_interface)
         return int(matcher.groups()[0]) if matcher is not None else 0
 
     def _build_mark(self) -> int:

discovery-infra/test_infra/controllers/node_controllers/node_controller.py

@@ -168,11 +168,12 @@ def get_ram_kib(self, node_name: str) -> int:
     def set_ram_kib(self, node_name: str, ram_kib: int) -> None:
         pass
 
-    def get_machine_cidr(self) -> Optional[str]:
-        # Default to auto resolve by the cluster. see cluster.get_machine_cidr
-        return None
+    def get_primary_machine_cidr(self) -> Optional[str]:
+        # Default to auto resolve by the cluster. see cluster.get_primary_machine_cidr
+        return None        
 
-    def get_provisioning_cidr(self) -> Optional[str]:
+    def get_machine_networks(self) -> Optional[List[str]]:
+        # Default to auto resolve by the cluster. see cluster.get_machine_networks
         return None
 
     @abstractmethod