Add retry on artifactory calls and update wrapper (#2844)

##### Short description:
Partial backport of
#2819

##### More details:

##### What this PR does / why we need it:

##### Which issue(s) this PR fixes:

##### Special notes for reviewer:

##### jira-ticket:
<!-- full-ticket-url needs to be provided. This would add a link to the
pull request to the jira and close it when the pull request is merged
If the task is not tracked by a Jira ticket, just write "NONE".
-->

Author: rnetser

.pre-commit-config.yaml

@@ -80,4 +80,5 @@ repos:
             "types-setuptools",
             "types-pexpect",
             "types-netaddr",
+            "types-cachetools"
           ]

pyproject.toml

@@ -79,6 +79,7 @@ dependencies = [
   "python-simple-logger>=2.0.13",
   "pytest-html>=4.1.1",
   "openshift-python-wrapper~=4.20.0",
+  "cachetools>=6.2.1",
 ]
 
 [project.optional-dependencies]

tests/utils.py

@@ -19,12 +19,13 @@
 from ocp_resources.virtual_machine import VirtualMachine
 from ocp_resources.virtual_machine_instance_migration import VirtualMachineInstanceMigration
 from pyhelper_utils.shell import run_ssh_commands
-from timeout_sampler import TimeoutExpiredError, TimeoutSampler
+from timeout_sampler import TimeoutExpiredError, TimeoutSampler, retry
 
 from utilities.constants import (
     DISK_SERIAL,
     HCO_DEFAULT_CPU_MODEL_KEY,
     RHSM_SECRET_NAME,
+    TIMEOUT_1MIN,
     TIMEOUT_1SEC,
     TIMEOUT_5SEC,
     TIMEOUT_10MIN,
@@ -467,13 +468,16 @@ def get_parameters_from_template(template, parameter_subset):
     }
 
 
+@retry(wait_timeout=TIMEOUT_1MIN, sleep=TIMEOUT_1SEC)
 def download_and_extract_tar(tarfile_url, dest_path):
     """Download and Extract the tar file."""
     artifactory_header = get_artifactory_header()
-    request = requests.get(tarfile_url, verify=False, headers=artifactory_header)
+    request = requests.get(tarfile_url, verify=False, headers=artifactory_header, timeout=10)
     thetarfile = tarfile.open(fileobj=BytesIO(request.content), mode="r|xz")
     thetarfile.extractall(path=dest_path)
 
+    return True
+
 
 @contextmanager
 def update_hco_with_persistent_storage_config(hco_cr, storage_class):

utilities/storage.py

@@ -4,6 +4,7 @@
 import shlex
 from contextlib import contextmanager
 
+import cachetools.func
 import kubernetes
 import requests
 from kubernetes.dynamic import DynamicClient
@@ -23,7 +24,7 @@
 from ocp_resources.volume_snapshot_class import VolumeSnapshotClass
 from pyhelper_utils.shell import run_ssh_commands
 from pytest_testconfig import config as py_config
-from timeout_sampler import TimeoutExpiredError, TimeoutSampler
+from timeout_sampler import TimeoutExpiredError, TimeoutSampler, retry
 
 import utilities.infra
 import utilities.virt as virt_util
@@ -296,6 +297,7 @@ def data_volume(
         yield dv
 
 
+@retry(wait_timeout=TIMEOUT_1MIN, sleep=TIMEOUT_1SEC)
 def get_downloaded_artifact(remote_name, local_name):
     """
     Download image or artifact to local tmpdir path
@@ -317,6 +319,8 @@ def get_downloaded_artifact(remote_name, local_name):
                 file_downloaded.write(chunk)
     try:
         assert os.path.isfile(local_name)
+        return True
+
     except FileNotFoundError as err:
         LOGGER.error(err)
         raise
@@ -1167,7 +1171,11 @@ def vm_snapshot(vm, name):
         yield snapshot
 
 
+@cachetools.func.ttl_cache(ttl=TIMEOUT_60MIN)
+@retry(wait_timeout=TIMEOUT_1MIN, sleep=TIMEOUT_10SEC)
 def validate_file_exists_in_url(url):
-    response = requests.head(url, headers=utilities.infra.get_artifactory_header(), verify=False)
+    response = requests.head(url, headers=utilities.infra.get_artifactory_header(), verify=False, timeout=10)
     if response.status_code != 200:
         raise UrlNotFoundError(url_request=response)
+
+    return True