MGMT-3474 e2e test to oc must-gather (openshift#399)

discovery-infra/test_infra/helper_classes/cluster.py

@@ -240,6 +240,14 @@ def wait_for_non_bootstrap_masters_to_reach_configuring_state_during_install(sel
             nodes_count=env_variables['num_masters']-1
         )
 
+    def wait_for_non_bootstrap_masters_to_reach_joined_state_during_install(self):
+        utils.wait_till_at_least_one_host_is_in_stage(
+            client=self.api_client,
+            cluster_id=self.id,
+            stages=[consts.HostsProgressStages.JOINED],
+            nodes_count=env_variables['num_masters']-1
+        )
+
     def wait_for_hosts_stage(self, stage: str, nodes_count: int = env_variables['num_nodes'], inclusive: bool = True):
         index = consts.all_host_stages.index(stage)
         utils.wait_till_at_least_one_host_is_in_stage(

discovery-infra/test_infra/logs_utils.py

@@ -7,6 +7,8 @@
 
 from test_infra.consts import NUMBER_OF_MASTERS
 
+OC_DOWNLOAD_LOGS_INTERVAL = 10 * 60
+NUM_OF_RETRIES = 6
 
 def verify_logs_uploaded(cluster_tar_path, expected_min_log_num, installation_success):
     assert os.path.exists(cluster_tar_path), f"{cluster_tar_path} doesn't exist"
@@ -23,6 +25,34 @@ def verify_logs_uploaded(cluster_tar_path, expected_min_log_num, installation_su
                 elif "master" in gz or "worker" in gz:
                     _verify_node_logs_uploaded(tempdir, gz)
 
+def verify_oc_logs_uploaded(cluster):
+    cluster_tar_path = "/tmp/test_on-cluster-error-after-join.tar"
+    for retry in range(NUM_OF_RETRIES):
+        try:
+            time.sleep(OC_DOWNLOAD_LOGS_INTERVAL)
+            cluster.download_installation_logs(cluster_tar_path)
+            assert os.path.exists(cluster_tar_path), f"{cluster_tar_path} doesn't exist"
+
+            _check_entry_from_extracted_tar("controller", cluster_tar_path,
+                lambda path : _check_entry_from_extracted_tar("must-gather", path, lambda inner : None))
+            return
+        except AssertionError as err:
+            logging.info(f'attempt {retry + 1} to download failed with error {str(err)}')
+
+    assert False, "oc logs were not uploaded"
+
+
+def _check_entry_from_extracted_tar(component, tarpath, verify):
+    with TemporaryDirectory() as tempdir:
+        logging.info(f'open tar file {tarpath}')
+        with tarfile.open(tarpath) as tar:
+            logging.info(f'verifying downloaded logs: {tar.getnames()}')
+            tar.extractall(tempdir)
+            extractedfiles = os.listdir(tempdir)
+            assert any(component in logfile for logfile in extractedfiles), f'can not find {component} in logs'
+            component_tar = [logfile for logfile in extractedfiles if component in logfile][0]
+            verify(os.path.join(tempdir, component_tar))
+
 
 def _verify_node_logs_uploaded(dir_path, file_path):
     gz = tarfile.open(os.path.join(dir_path, file_path))

discovery-infra/test_infra/utils.py

@@ -800,3 +800,14 @@ def update_hosts(client, cluster_id, libvirt_nodes, update_hostnames=False, upda
         roles = None
 
     client.update_hosts(cluster_id=cluster_id, hosts_with_roles=roles, hosts_names=hostnames)
+
+def get_assisted_controller_status(kubeconfig):
+    log.info("Getting controller status")
+    command = f"oc --insecure-skip-tls-verify --kubeconfig={kubeconfig} --no-headers=true -n assisted-installer get pods -l job-name=assisted-installer-controller"
+    response = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+    if response.returncode != 0:
+        log.error(f'failed to get controller status: {response.stderr}')
+        return b''
+
+    log.info(f'{response.stdout}')
+    return response.stdout

discovery-infra/tests/base_test.py

@@ -218,3 +218,11 @@ def verify_no_logs_uploaded(cluster, cluster_tar_path):
         with pytest.raises(ApiException) as ex:
             cluster.download_installation_logs(cluster_tar_path)
         assert "No log files" in str(ex.value)
+
+    def update_oc_config(self, nodes, cluster):
+        os.environ["KUBECONFIG"] = env_variables['kubeconfig_path']
+        vips = nodes.controller.get_ingress_and_api_vips()
+        api_vip = vips['api_vip']
+        infra_utils.config_etc_hosts(cluster_name=cluster.name,
+                               base_dns_domain=env_variables["base_domain"],
+                               api_vip=api_vip)