From 0e2f7437e884b28173420b3255684a37c94cac13 Mon Sep 17 00:00:00 2001 From: Avishay Traeger Date: Wed, 11 Nov 2020 09:41:33 +0200 Subject: [PATCH] changes --- discovery-infra/test_infra/consts.py | 1 + .../node_controllers/libvirt_controller.py | 19 ++-- .../controllers/node_controllers/node.py | 11 +++ .../node_controllers/node_controller.py | 5 +- .../test_infra/helper_classes/cluster.py | 97 ++++++++++++++++++- discovery-infra/test_infra/utils.py | 61 ------------ discovery-infra/tests/base_test.py | 12 +-- .../tests/test_cancel_and_reset.py | 2 +- discovery-infra/tests/test_validations.py | 48 ++++----- 9 files changed, 150 insertions(+), 106 deletions(-) diff --git a/discovery-infra/test_infra/consts.py b/discovery-infra/test_infra/consts.py index f677de54331..44a8ce9af00 100644 --- a/discovery-infra/test_infra/consts.py +++ b/discovery-infra/test_infra/consts.py @@ -10,6 +10,7 @@ NODES_REGISTERED_TIMEOUT = 600 CLUSTER_INSTALLATION_TIMEOUT = 60 * 60 # 60 minutes START_CLUSTER_INSTALLATION_TIMEOUT = 360 +VALIDATION_TIMEOUT = 3 * 60 TF_TEMPLATE = "terraform_files" NUMBER_OF_MASTERS = 3 TEST_INFRA = "test-infra" diff --git a/discovery-infra/test_infra/controllers/node_controllers/libvirt_controller.py b/discovery-infra/test_infra/controllers/node_controllers/libvirt_controller.py index 9da295ca4fa..88bcb57fa37 100644 --- a/discovery-infra/test_infra/controllers/node_controllers/libvirt_controller.py +++ b/discovery-infra/test_infra/controllers/node_controllers/libvirt_controller.py @@ -173,9 +173,7 @@ def get_host_id(self, node_name): return dom.UUIDString() def get_cpu_cores(self, node_name): - dom = self.libvirt_connection.lookupByName(node_name) - current_xml = dom.XMLDesc(0) - xml = minidom.parseString(current_xml.encode('utf-8')) + xml = self._get_xml(node_name) vcpu_element = xml.getElementsByTagName('vcpu')[0] return int(vcpu_element.firstChild.nodeValue) @@ -185,15 +183,22 @@ def set_cpu_cores(self, node_name, core_count): dom.setVcpusFlags(core_count) logging.info(f"Successfully set vcpus to {core_count} for node: {node_name}") + def get_ram_kib(self, node_name): + xml = self._get_xml(node_name) + memory_element = xml.getElementsByTagName('memory')[0] + return int(memory_element.firstChild.nodeValue) + def set_ram_kib(self, node_name, ram_kib): logging.info(f"Going to set memory to {ram_kib} for node: {node_name}") - dom = self.libvirt_connection.lookupByName(node_name) - # dom.setMemoryFlags(ram_kib) - current_xml = dom.XMLDesc(0) - xml = minidom.parseString(current_xml.encode('utf-8')) + xml = self._get_xml(node_name) memory_element = xml.getElementsByTagName('memory')[0] memory_element.firstChild.replaceWholeText(ram_kib) dom = self.libvirt_connection.defineXML(xml.toprettyxml()) if dom is None: raise Exception(f"Failed to set memory for node: {node_name}") logging.info(f"Successfully set memory to {ram_kib} for node: {node_name}") + + def _get_xml(self, node_name): + dom = self.libvirt_connection.lookupByName(node_name) + current_xml = dom.XMLDesc(0) + return minidom.parseString(current_xml.encode('utf-8')) \ No newline at end of file diff --git a/discovery-infra/test_infra/controllers/node_controllers/node.py b/discovery-infra/test_infra/controllers/node_controllers/node.py index 34e9495c20a..eb136e3d22d 100644 --- a/discovery-infra/test_infra/controllers/node_controllers/node.py +++ b/discovery-infra/test_infra/controllers/node_controllers/node.py @@ -10,6 +10,8 @@ def __init__(self, name, node_controller, private_ssh_key_path=None, username="c self.private_ssh_key_path = private_ssh_key_path self.username = username self.node_controller = node_controller + self.original_vcpu_count = self.get_cpu_cores() + self.original_ram_kib = self.get_ram_kib() self._ips = [] self._macs = [] @@ -108,5 +110,14 @@ def get_cpu_cores(self): def set_cpu_cores(self, core_count): self.node_controller.set_cpu_cores(self.name, core_count) + def reset_cpu_cores(self): + self.set_cpu_cores(self.original_vcpu_count) + + def get_ram_kib(self): + return self.node_controller.get_ram_kib(self.name) + def set_ram_kib(self, ram_kib): self.node_controller.set_ram_kib(self.name, ram_kib) + + def reset_ram_kib(self): + self.set_ram_kib(self.original_ram_kib) \ No newline at end of file diff --git a/discovery-infra/test_infra/controllers/node_controllers/node_controller.py b/discovery-infra/test_infra/controllers/node_controllers/node_controller.py index 33864f19727..870e0712979 100644 --- a/discovery-infra/test_infra/controllers/node_controllers/node_controller.py +++ b/discovery-infra/test_infra/controllers/node_controllers/node_controller.py @@ -54,5 +54,8 @@ def get_cpu_cores(self, node_name: str) -> int: def set_cpu_cores(self, node_name: str, core_count: int) -> None: raise NotImplementedError - def set_ram_kib(self, node_name: str, ram_kib: int) -> None: + def get_ram_kib(self, node_name: str) -> int: raise NotImplementedError + + def set_ram_kib(self, node_name: str, ram_kib: int) -> None: + raise NotImplementedError \ No newline at end of file diff --git a/discovery-infra/test_infra/helper_classes/cluster.py b/discovery-infra/test_infra/helper_classes/cluster.py index dadc0c8d583..062ca5fe382 100644 --- a/discovery-infra/test_infra/helper_classes/cluster.py +++ b/discovery-infra/test_infra/helper_classes/cluster.py @@ -1,4 +1,5 @@ import logging +import waiting from tests.conftest import env_variables from test_infra import consts, utils @@ -47,12 +48,17 @@ def generate_and_download_image( image_path=iso_download_path ) - def wait_until_hosts_are_discovered(self,nodes_count=env_variables['num_nodes']): + def wait_until_hosts_are_discovered(self, nodes_count=env_variables['num_nodes'], + additional_statuses=None + ): + statuses=[consts.NodesStatus.PENDING_FOR_INPUT, consts.NodesStatus.KNOWN] + if additional_statuses: + statuses.extend(additional_statuses) utils.wait_till_all_hosts_are_in_status( client=self.api_client, cluster_id=self.id, nodes_count=nodes_count, - statuses=[consts.NodesStatus.PENDING_FOR_INPUT, consts.NodesStatus.KNOWN] + statuses=statuses ) def set_host_roles(self): @@ -83,7 +89,15 @@ def set_ingress_and_api_vips(self, vips): def set_ssh_key(self, ssh_key): logging.info(f"Setting SSH key:{ssh_key} for cluster: {self.id}") - self.api_client.update_cluster(self.id, ssh_key) + self.api_client.update_cluster(self.id, {"ssh_public_key": ssh_key}) + + def set_base_dns_domain(self, base_dns_domain): + logging.info(f"Setting base DNS domain:{base_dns_domain} for cluster: {self.id}") + self.api_client.update_cluster(self.id, {"base_dns_domain": base_dns_domain}) + + def set_pull_secret(self, pull_secret): + logging.info(f"Setting pull secret:{pull_secret} for cluster: {self.id}") + self.api_client.update_cluster(self.id, {"pull_secret": pull_secret}) def start_install(self): self.api_client.install_cluster(cluster_id=self.id) @@ -119,6 +133,13 @@ def wait_for_cluster_in_error_status(self): statuses=[consts.ClusterStatus.ERROR] ) + def wait_for_cluster_in_pending_for_input_status(self): + utils.wait_till_cluster_is_in_status( + client=self.api_client, + cluster_id=self.id, + statuses=[consts.ClusterStatus.PENDING_FOR_INPUT] + ) + def wait_for_at_least_one_host_to_boot_during_install(self, nodes_count=1): utils.wait_till_at_least_one_host_is_in_stage( client=self.api_client, @@ -288,3 +309,73 @@ def setup_nodes(self, nodes): nodes.start_all() self.wait_until_hosts_are_discovered(nodes_count=len(nodes)) return nodes.create_nodes_cluster_hosts_mapping(cluster=self) + + def wait_for_cluster_validation( + self, validation_section, validation_id, statuses, + timeout=consts.VALIDATION_TIMEOUT, interval=2 + ): + logging.info(f"Wait until cluster %s validation %s is in status %s", + self.id, validation_id, statuses) + try: + waiting.wait( + lambda: self.is_cluster_validation_in_status( + validation_section=validation_section, + validation_id=validation_id, + statuses=statuses + ), + timeout_seconds=timeout, + sleep_seconds=interval, + waiting_for="Cluster validation to be in status %s" % statuses, + ) + except: + logging.error("Cluster validation status is: %s", + utils.get_cluster_validation_value( + self.api_client.cluster_get(self.id), validation_section, + validation_id)) + raise + + def is_cluster_validation_in_status( + self, validation_section, validation_id, statuses + ): + logging.info("Is cluster %s validation %s in status %s", + self.id, validation_id, statuses) + try: + return utils.get_cluster_validation_value( + self.api_client.cluster_get(self.id), + validation_section, validation_id) in statuses + except: + log.exception("Failed to get cluster %s validation info", self.id) + + def wait_for_host_validation( + self, host_id, validation_section, validation_id, statuses, + timeout=consts.VALIDATION_TIMEOUT, interval=2 + ): + logging.info("Wait until host %s validation %s is in status %s", host_id, + validation_id, statuses) + try: + waiting.wait( + lambda: self.is_host_validation_in_status( + host_id=host_id, + validation_section=validation_section, + validation_id=validation_id, + statuses=statuses + ), + timeout_seconds=timeout, + sleep_seconds=interval, + waiting_for="Host validation to be in status %s" % statuses, + ) + except: + logging.error("Host validation status is: %s", + utils.get_host_validation_value(self.api_client.cluster_get(self.id), + host_id, validation_section, validation_id)) + raise + + def is_host_validation_in_status( + self, host_id, validation_section, validation_id, statuses + ): + logging.info("Is host %s validation %s in status %s", host_id, validation_id, statuses) + try: + return utils.get_host_validation_value(self.api_client.cluster_get(self.id), + host_id, validation_section, validation_id) in statuses + except: + logging.exception("Failed to get cluster %s validation info", self.id) diff --git a/discovery-infra/test_infra/utils.py b/discovery-infra/test_infra/utils.py index ab892a19f9c..b77ed7e878f 100644 --- a/discovery-infra/test_infra/utils.py +++ b/discovery-infra/test_infra/utils.py @@ -346,36 +346,6 @@ def is_cluster_in_status(client, cluster_id, statuses): log.exception("Failed to get cluster %s info", cluster_id) -def wait_till_cluster_validation_is_in_status( - client, cluster_id, validation_section, validation_id, statuses, timeout=consts.VALIDATION_TIMEOUT, interval=2 -): - log.info("Wait till cluster %s validation %s is in status %s", cluster_id, validation_id, statuses) - try: - waiting.wait( - lambda: is_cluster_validation_in_status( - client=client, - cluster_id=cluster_id, - validation_section=validation_section, - validation_id=validation_id, - statuses=statuses - ), - timeout_seconds=timeout, - sleep_seconds=interval, - waiting_for="Cluster validation to be in status %s" % statuses, - ) - except: - log.error("Cluster validation status is: %s", get_cluster_validation_value(client.cluster_get(cluster_id), validation_section, validation_id)) - raise - - -def is_cluster_validation_in_status(client, cluster_id, validation_section, validation_id, statuses): - log.info("Is cluster %s validation %s in status %s", cluster_id, validation_id, statuses) - try: - return get_cluster_validation_value(client.cluster_get(cluster_id), validation_section, validation_id) in statuses - except: - log.exception("Failed to get cluster %s validation info", cluster_id) - - def get_cluster_validation_value(cluster_info, validation_section, validation_id): found_status = "validation not found" validations = json.loads(cluster_info.validations_info) @@ -386,37 +356,6 @@ def get_cluster_validation_value(cluster_info, validation_section, validation_id return found_status -def wait_till_host_validation_is_in_status( - client, cluster_id, host_id, validation_section, validation_id, statuses, timeout=consts.VALIDATION_TIMEOUT, interval=2 -): - log.info("Wait till host %s validation %s is in status %s", host_id, validation_id, statuses) - try: - waiting.wait( - lambda: is_host_validation_in_status( - client=client, - cluster_id=cluster_id, - host_id=host_id, - validation_section=validation_section, - validation_id=validation_id, - statuses=statuses - ), - timeout_seconds=timeout, - sleep_seconds=interval, - waiting_for="Host validation to be in status %s" % statuses, - ) - except: - log.error("Host validation status is: %s", get_host_validation_value(client.cluster_get(cluster_id), host_id, validation_section, validation_id)) - raise - - -def is_host_validation_in_status(client, cluster_id, host_id, validation_section, validation_id, statuses): - log.info("Is host %s validation %s in status %s", host_id, validation_id, statuses) - try: - return get_host_validation_value(client.cluster_get(cluster_id), host_id, validation_section, validation_id) in statuses - except: - log.exception("Failed to get cluster %s validation info", cluster_id) - - def get_host_validation_value(cluster_info, host_id, validation_section, validation_id): for host in cluster_info.hosts: if host.id != host_id: diff --git a/discovery-infra/tests/base_test.py b/discovery-infra/tests/base_test.py index 1ddc2f05534..e85245a09b9 100644 --- a/discovery-infra/tests/base_test.py +++ b/discovery-infra/tests/base_test.py @@ -26,6 +26,8 @@ def nodes(self, setup_node_controller): controller = setup_node_controller nodes = Nodes(controller, env_variables["private_ssh_key_path"]) nodes.set_correct_boot_order(start_nodes=False) + nodes.run_for_all_nodes("reset_cpu_cores") + nodes.run_for_all_nodes("reset_ram_kib") yield nodes nodes.shutdown_all() nodes.format_all_disks() @@ -341,12 +343,4 @@ def assert_http_error_code(api_call, status, reason, **kwargs): @staticmethod def assert_cluster_validation(cluster_info, validation_section, validation_id, expected_status): found_status = utils.get_cluster_validation_value(cluster_info, validation_section, validation_id) - assert found_status == expected_status, "Found validation status " + found_status + " rather than " + expected_status + " for validation " + validation_id - - @staticmethod - def wait_for_cluster_validation(client, cluster_id, validation_section, validation_id, statuses): - utils.wait_till_cluster_validation_is_in_status(client, cluster_id, validation_section, validation_id, statuses) - - @staticmethod - def wait_for_host_validation(client, cluster_id, host_id, validation_section, validation_id, statuses): - utils.wait_till_host_validation_is_in_status(client, cluster_id, host_id, validation_section, validation_id, statuses) + assert found_status == expected_status, "Found validation status " + found_status + " rather than " + expected_status + " for validation " + validation_id \ No newline at end of file diff --git a/discovery-infra/tests/test_cancel_and_reset.py b/discovery-infra/tests/test_cancel_and_reset.py index 33efc06edad..1e90541cd44 100644 --- a/discovery-infra/tests/test_cancel_and_reset.py +++ b/discovery-infra/tests/test_cancel_and_reset.py @@ -228,7 +228,7 @@ def test_reset_cluster_after_changing_cluster_configuration( new_cluster.set_ingress_and_api_vips({ 'api_vip': api_vip, 'ingress_vip': ingress_vip}) - new_cluster.set_ssh_key({'ssh_public_key': DUMMY_SSH_KEY}) + new_cluster.set_ssh_key(DUMMY_SSH_KEY) new_cluster.reboot_required_nodes_into_iso_after_reset( nodes) new_cluster.wait_until_hosts_are_discovered() diff --git a/discovery-infra/tests/test_validations.py b/discovery-infra/tests/test_validations.py index 88f7419ef5d..3e96daf6dce 100644 --- a/discovery-infra/tests/test_validations.py +++ b/discovery-infra/tests/test_validations.py @@ -8,12 +8,12 @@ class TestValidations(BaseTest): @pytest.mark.regression - def test_basic_cluster_validations(self, api_client, cluster): - cluster_id = cluster().id + def test_basic_cluster_validations(self, cluster): + new_cluster = cluster() # Check initial validations - self.wait_until_cluster_is_pending_for_input(cluster_id, api_client) - orig_cluster = api_client.cluster_get(cluster_id) + new_cluster.wait_for_cluster_in_pending_for_input_status() + orig_cluster = new_cluster.get_details() self.assert_cluster_validation(orig_cluster, "configuration", "pull-secret-set", "success") self.assert_cluster_validation(orig_cluster, "hosts-data", "all-hosts-are-ready-to-install", "success") self.assert_cluster_validation(orig_cluster, "hosts-data", "sufficient-masters-count", "failure") @@ -31,24 +31,24 @@ def test_basic_cluster_validations(self, api_client, cluster): self.assert_cluster_validation(orig_cluster, "network", "ntp-server-configured", "success") # Check base DNS domain - api_client.update_cluster(cluster_id, {"base_dns_domain": ""}) - self.wait_for_cluster_validation(api_client, cluster_id, "network", "dns-domain-defined", ["failure"]) - api_client.update_cluster(cluster_id, {"base_dns_domain": orig_cluster.base_dns_domain}) - self.wait_for_cluster_validation(api_client, cluster_id, "network", "dns-domain-defined", ["success"]) + new_cluster.set_base_dns_domain("") + new_cluster.wait_for_cluster_validation("network", "dns-domain-defined", ["failure"]) + new_cluster.set_base_dns_domain(orig_cluster.base_dns_domain) + new_cluster.wait_for_cluster_validation("network", "dns-domain-defined", ["success"]) # Check pull secret - api_client.update_cluster(cluster_id, {"pull_secret": ""}) - self.wait_for_cluster_validation(api_client, cluster_id, "configuration", "pull-secret-set", ["failure"]) - api_client.update_cluster(cluster_id, {"pull_secret": env_variables['pull_secret']}) - self.wait_for_cluster_validation(api_client, cluster_id, "configuration", "pull-secret-set", ["success"]) + new_cluster.set_pull_secret("") + new_cluster.wait_for_cluster_validation("configuration", "pull-secret-set", ["failure"]) + new_cluster.set_pull_secret(env_variables['pull_secret']) + new_cluster.wait_for_cluster_validation("configuration", "pull-secret-set", ["success"]) - def test_host_insufficient(self, api_client, cluster, nodes): - cluster_id = cluster().id - self.generate_and_download_image(cluster_id=cluster_id, api_client=api_client) + @pytest.mark.regression + def test_host_insufficient(self, cluster, nodes): + new_cluster = cluster() + new_cluster.generate_and_download_image() # Modify vCPU count on the first node to be insufficient vcpu_node = nodes.nodes[0] - orig_vcpu_value = vcpu_node.get_cpu_cores() vcpu_node.set_cpu_cores(1) vcpu_host_id = vcpu_node.get_host_id() @@ -58,19 +58,19 @@ def test_host_insufficient(self, api_client, cluster, nodes): ram_host_id = ram_node.get_host_id() nodes.start_all() - self.wait_until_hosts_are_discovered(cluster_id=cluster_id, api_client=api_client, additional_statuses=[consts.NodesStatus.INSUFFICIENT]) - self.wait_for_host_validation(api_client, cluster_id, vcpu_host_id, "hardware", "has-min-cpu-cores", ["failure"]) - self.wait_for_host_validation(api_client, cluster_id, ram_host_id, "hardware", "has-min-memory", ["failure"]) + new_cluster.wait_until_hosts_are_discovered(additional_statuses=[consts.NodesStatus.INSUFFICIENT]) + new_cluster.wait_for_host_validation(vcpu_host_id, "hardware", "has-min-cpu-cores", ["failure"]) + new_cluster.wait_for_host_validation(ram_host_id, "hardware", "has-min-memory", ["failure"]) # Return to original settings and verify that validations pass vcpu_node.shutdown() - vcpu_node.set_cpu_cores(orig_vcpu_value) + vcpu_node.reset_cpu_cores() vcpu_node.start() ram_node.shutdown() - ram_node.set_ram_kib(int(env_variables['master_memory']) * 1024) + ram_node.reset_ram_kib() ram_node.start() - self.wait_until_hosts_are_discovered(cluster_id=cluster_id, api_client=api_client, additional_statuses=[consts.NodesStatus.INSUFFICIENT]) - self.wait_for_host_validation(api_client, cluster_id, vcpu_host_id, "hardware", "has-min-cpu-cores", ["success"]) - self.wait_for_host_validation(api_client, cluster_id, ram_host_id, "hardware", "has-min-memory", ["success"]) + new_cluster.wait_until_hosts_are_discovered(additional_statuses=[consts.NodesStatus.INSUFFICIENT]) + new_cluster.wait_for_host_validation(vcpu_host_id, "hardware", "has-min-cpu-cores", ["success"]) + new_cluster.wait_for_host_validation(ram_host_id, "hardware", "has-min-memory", ["success"])