fix: download error (#182)

* fix: retry 5 times + wait 45 sec for DAAC

* feat: make re-try configurable

* fix: retry on Temporary failure in name resolution

* chore: update version

* fix: add new test case for error

Author: wphyojpl

CHANGELOG.md

@@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [5.2.1] - 2023-07-10
+### Added
+- [#182](https://github.com/unity-sds/unity-data-services/pull/182) fix: Retry if Download Error in DAAC
+
 ## [5.2.0] - 2023-07-05
 ### Added
 - [#169](https://github.com/unity-sds/unity-data-services/pull/169) feat: parallelize download

cumulus_lambda_functions/lib/earthdata_login/urs_token.py

@@ -1,17 +1,42 @@
 import json
 import logging
+import time
 from http.cookiejar import CookieJar
 from typing import Dict
 from urllib import request
 
 import requests
 from requests.auth import HTTPBasicAuth
 from tenacity import retry, retry_if_result, stop_after_attempt, wait_random_exponential
-LOGGER = logging.getLogger(__name__)
 
+LOGGER = logging.getLogger(__name__)
 
 class URSToken(object):
-    def __init__(self, username: str, dwssap: str, edl_base_url: str = None) -> None:
+    DNS_ERROR_TXT = 'Name-Resolution-Error'
+    """
+    Traceback (most recent call last):
+ File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 174, in _new_conn
+  conn = connection.create_connection(
+ File "/usr/local/lib/python3.9/site-packages/urllib3/util/connection.py", line 72, in create_connection
+  for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):
+ File "/usr/local/lib/python3.9/socket.py", line 954, in getaddrinfo
+  for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
+socket.gaierror: [Errno -3] Temporary failure in name resolution
+During handling of the above exception, another exception occurred:
+Traceback (most recent call last):
+ File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 714, in urlopen
+  httplib_response = self._make_request(
+ File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 403, in _make_request
+  self._validate_conn(conn)
+ File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 1053, in _validate_conn
+  conn.connect()
+ File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 363, in connect
+  self.sock = conn = self._new_conn()
+ File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 186, in _new_conn
+  raise NewConnectionError(
+urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPSConnection object at 0x7f111cbc54f0>: Failed to establish a new connection: [Errno -3] Temporary failure in name resolution
+    """
+    def __init__(self, username: str, dwssap: str, edl_base_url: str = None, wait_time = 30, retry_times = 5) -> None:
         super().__init__()
         self.__default_edl_base_url = 'https://urs.earthdata.nasa.gov/'
         self.__username = username
@@ -22,6 +47,9 @@ def __init__(self, username: str, dwssap: str, edl_base_url: str = None) -> None
         if not self.__edl_base_url.startswith('http'):
             self.__edl_base_url = f'https://{self.__edl_base_url}'
         self.__token = None
+        self.__wait_time = wait_time
+        self.__retry_times = retry_times
+
 
     @retry(wait=wait_random_exponential(multiplier=1, max=60),
            stop=stop_after_attempt(3),
@@ -47,10 +75,11 @@ def create_token(self, url: str) -> str:
 
         # Add better error handling there
         # Max tokens
-        # Wrong Username/Passsword
+        # Wrong Username/Password
         # Other
-        except:  # noqa E722
+        except Exception as e:  # noqa E722
             LOGGER.warning("Error getting the token - check user name and password", exc_info=True)
+            raise RuntimeError(str(e))
         return token
 
     def list_tokens(self, url: str):
@@ -66,7 +95,7 @@ def list_tokens(self, url: str):
             for x in response_content:
                 tokens.append(x['access_token'])
 
-        except:  # noqa E722
+        except Exception as e:  # noqa E722
             LOGGER.warning("Error getting the token - check user name and password", exc_info=True)
         return tokens
 
@@ -107,10 +136,25 @@ def delete_token(self, token: str) -> bool:
             LOGGER.warning("Error deleting the token", exc_info=True)
         return False
 
+    def __get_token_once(self):
+        try:
+            token_url = f'{self.__edl_base_url}api/users'
+            tokens = self.list_tokens(token_url)
+            if len(tokens) == 0:
+                return self.create_token(token_url)
+        except Exception as conn_err:
+            if 'Temporary failure in name resolution' in str(conn_err):
+                return self.DNS_ERROR_TXT
+            raise conn_err
+        return tokens[0]
+
     def get_token(self) -> str:
-        token_url = f'{self.__edl_base_url}api/users'
-        tokens = self.list_tokens(token_url)
-        if len(tokens) == 0:
-            return self.create_token(token_url)
-        else:
-            return tokens[0]
+        token = self.__get_token_once()
+        retry_count = 0
+        while token == self.DNS_ERROR_TXT and retry_count < self.__retry_times:
+            LOGGER.error(f'{self.DNS_ERROR_TXT} for URS Token. attempt: {retry_count}')
+            time.sleep(self.__wait_time)
+            token = self.__get_token_once()
+            retry_count += 1
+        return token
+

cumulus_lambda_functions/stage_in_out/download_granules_abstract.py

@@ -131,5 +131,5 @@ def download(self, **kwargs) -> str:
         LOGGER.debug(f'writing errors if any')
         if len(error_list_list) > 0:
             with open(f'{self._download_dir}/error.log', 'w') as error_file:
-                error_file.write(json.dumps(error_list, indent=4))
+                error_file.write(json.dumps(error_list_list, indent=4))
         return json.dumps(granules_json_dict)

cumulus_lambda_functions/stage_in_out/download_granules_daac.py

@@ -1,3 +1,5 @@
+import time
+
 import requests
 
 from cumulus_lambda_functions.lib.earthdata_login.urs_token_retriever import URSTokenRetriever
@@ -15,6 +17,8 @@ class DownloadGranulesDAAC(DownloadGranulesAbstract):
     def __init__(self) -> None:
         super().__init__()
         self.__edl_token = None
+        self.__retry_wait_time_sec = int(os.environ.get('DOWNLOAD_RETRY_WAIT_TIME', '30'))
+        self.__retry_times = int(os.environ.get('DOWNLOAD_RETRY_TIMES', '5'))
 
     def _set_props_from_env(self):
         missing_keys = [k for k in [self.STAC_JSON, self.DOWNLOAD_DIR_KEY] if k not in os.environ]
@@ -30,6 +34,12 @@ def _download_one_item(self, downloading_url):
             'Authorization': f'Bearer {self.__edl_token}'
         }
         r = requests.get(downloading_url, headers=headers)
+        download_count = 1
+        while r.status_code in [502, 504] and download_count < self.__retry_times:
+            LOGGER.error(f'502 or 504 while downloading {downloading_url}. attempt: {download_count}')
+            time.sleep(self.__retry_wait_time_sec)
+            r = requests.get(downloading_url, headers=headers)
+            download_count += 1
         r.raise_for_status()
         local_file_path = os.path.join(self._download_dir, os.path.basename(downloading_url))
         with open(local_file_path, 'wb') as fd:

docker/stage-in-stage-out/dc-002-download.yaml

@@ -20,6 +20,9 @@ services:
 
       PARALLEL_COUNT: 'int. Ex: 10. can be defaulted to -1 which will create the same number of parallelism as CPU count'
 
+      DOWNLOAD_RETRY_WAIT_TIME: '30'
+      DOWNLOAD_RETRY_TIMES: '5'
+
       EDL_USERNAME: '/unity/uds/user/wphyo/edl_username'
       EDL_PASSWORD: '/unity/uds/user/wphyo/edl_dwssap'
       EDL_PASSWORD_TYPE: 'PARAM_STORE'

setup.py

@@ -18,7 +18,7 @@
 
 setup(
     name="cumulus_lambda_functions",
-    version="5.2.0",
+    version="5.2.1",
     packages=find_packages(),
     install_requires=install_requires,
     tests_require=['mock', 'nose', 'sphinx', 'sphinx_rtd_theme', 'coverage', 'pystac', 'python-dotenv', 'jsonschema'],

tests/cumulus_lambda_functions/lib/earthdata_login/test_urs_token_retriever.py

@@ -16,3 +16,14 @@ def test_01(self):
         result = URSTokenRetriever().start()
         self.assertTrue(len(result) > 0, 'empty token')
         return
+
+    def test_02(self):
+        os.environ[Constants.EDL_USERNAME] = '/unity/uds/user/wphyo/edl_username'
+        os.environ[Constants.EDL_PASSWORD] = '/unity/uds/user/wphyo/edl_dwssap'
+        # os.environ[Constants.USERNAME] = 'usps_username'
+        # os.environ[Constants.PASSWORD] = 'usps_password'
+        os.environ[Constants.EDL_PASSWORD_TYPE] = Constants.PARAM_STORE
+        os.environ[Constants.EDL_BASE_URL] = 'urs1.earthdata.nasa.gov'
+        with self.assertRaises(RuntimeError) as cm:
+            result = URSTokenRetriever().start()
+        return