Merge pull request #142 from unity-sds/daac-download-stac-file

feat: Support DAAC download files stac file, not just direct json text

Author: ngachung

CHANGELOG.md

@@ -5,12 +5,16 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [3.6.0] - 2023-04-24
+### Added
+- [#142](https://github.com/unity-sds/unity-data-services/pull/142) feat: Support DAAC download files stac file, not just direct json text
+
 ## [3.5.0] - 2023-04-18
-### Changed
+### Added
 - [#138](https://github.com/unity-sds/unity-data-services/pull/138) feat: Checkout stage with STAC catalog json
 
 ## [3.4.0] - 2023-04-17
-### Changed
+### Added
 - [#132](https://github.com/unity-sds/unity-data-services/pull/132) feat: add DAAC download logic
 
 ## [3.3.1] - 2023-04-13

cumulus_lambda_functions/stage_in_out/download_granules_abstract.py

@@ -1,7 +1,43 @@
+import json
+import logging
+import os
 from abc import ABC, abstractmethod
 
+from cumulus_lambda_functions.lib.utils.file_utils import FileUtils
+
+LOGGER = logging.getLogger(__name__)
+
 
 class DownloadGranulesAbstract(ABC):
+    STAC_JSON = 'STAC_JSON'
+    DOWNLOAD_DIR_KEY = 'DOWNLOAD_DIR'
+
+    def __init__(self) -> None:
+        super().__init__()
+        self._granules_json = []
+        self._download_dir = '/tmp'
+
+    def _setup_download_dir(self):
+        self._download_dir = os.environ.get(self.DOWNLOAD_DIR_KEY)
+        self._download_dir = self._download_dir[:-1] if self._download_dir.endswith('/') else self._download_dir
+        LOGGER.debug(f'creating download dir: {self._download_dir}')
+        FileUtils.mk_dir_p(self._download_dir)
+        return self
+    
+    def _retrieve_stac_json(self):
+        raw_stac_json = os.environ.get(self.STAC_JSON)
+        try:
+            self._granules_json = json.loads(raw_stac_json)
+            return self
+        except:
+            LOGGER.debug(f'raw_stac_json is not STAC_JSON: {raw_stac_json}. trying to see if file exists')
+        if not FileUtils.file_exist(raw_stac_json):
+            raise ValueError(f'missing file or not JSON: {raw_stac_json}')
+        self._granules_json = FileUtils.read_json(raw_stac_json)
+        if self._granules_json is None:
+            raise ValueError(f'{raw_stac_json} is not JSON')
+        return self
+    
     @abstractmethod
     def download(self, **kwargs) -> list:
         raise NotImplementedError()

cumulus_lambda_functions/stage_in_out/download_granules_daac.py

@@ -6,30 +6,21 @@
 import logging
 import os
 
-from cumulus_lambda_functions.lib.aws.aws_s3 import AwsS3
-from cumulus_lambda_functions.lib.utils.file_utils import FileUtils
-
 LOGGER = logging.getLogger(__name__)
 
 
 class DownloadGranulesDAAC(DownloadGranulesAbstract):
-    DOWNLOAD_DIR_KEY = 'DOWNLOAD_DIR'
-    STAC_JSON = 'STAC_JSON'
 
     def __init__(self) -> None:
         super().__init__()
-        self.__download_dir = '/tmp'
-        self.__s3 = AwsS3()
-        self.__granules_json = []
         self.__edl_token = None
 
     def __set_props_from_env(self):
         missing_keys = [k for k in [self.STAC_JSON, self.DOWNLOAD_DIR_KEY] if k not in os.environ]
         if len(missing_keys) > 0:
             raise ValueError(f'missing environment keys: {missing_keys}')
-        self.__granules_json = json.loads(os.environ.get(self.STAC_JSON))
-        self.__download_dir = os.environ.get(self.DOWNLOAD_DIR_KEY)
-        self.__download_dir = self.__download_dir[:-1] if self.__download_dir.endswith('/') else self.__download_dir
+        self._retrieve_stac_json()
+        self._setup_download_dir()
         self.__edl_token = URSTokenRetriever().start()
         return self
 
@@ -79,7 +70,7 @@ def __download_one_granule(self, assets: dict):
                 if r.status_code >= 400:
                     raise RuntimeError(f'wrong response status: {r.status_code}. details: {r.content}')
                 # TODO. how to correctly check redirecting to login page
-                with open(os.path.join(self.__download_dir, os.path.basename(v["href"])), 'wb') as fd:
+                with open(os.path.join(self._download_dir, os.path.basename(v["href"])), 'wb') as fd:
                     fd.write(r.content)
             except Exception as e:
                 LOGGER.exception(f'failed to download {v}')
@@ -89,15 +80,14 @@ def __download_one_granule(self, assets: dict):
 
     def download(self, **kwargs) -> list:
         self.__set_props_from_env()
-        LOGGER.debug(f'creating download dir: {self.__download_dir}')
-        FileUtils.mk_dir_p(self.__download_dir)
-        downloading_urls = self.__get_downloading_urls(self.__granules_json)
+        LOGGER.debug(f'creating download dir: {self._download_dir}')
+        downloading_urls = self.__get_downloading_urls(self._granules_json)
         error_list = []
         for each in downloading_urls:
             LOGGER.debug(f'working on {each}')
             current_error_list = self.__download_one_granule(each)
             error_list.extend(current_error_list)
         if len(error_list) > 0:
-            with open(f'{self.__download_dir}/error.log', 'w') as error_file:
+            with open(f'{self._download_dir}/error.log', 'w') as error_file:
                 error_file.write(json.dumps(error_list, indent=4))
         return downloading_urls

cumulus_lambda_functions/stage_in_out/download_granules_s3.py

@@ -4,42 +4,22 @@
 import os
 
 from cumulus_lambda_functions.lib.aws.aws_s3 import AwsS3
-from cumulus_lambda_functions.lib.utils.file_utils import FileUtils
 
 LOGGER = logging.getLogger(__name__)
 
 
 class DownloadGranulesS3(DownloadGranulesAbstract):
-    DOWNLOAD_DIR_KEY = 'DOWNLOAD_DIR'
-    STAC_JSON = 'STAC_JSON'
 
     def __init__(self) -> None:
         super().__init__()
-        self.__download_dir = '/tmp'
         self.__s3 = AwsS3()
-        self.__granules_json = []
-
-    def __retrieve_stac_json(self):
-        raw_stac_json = os.environ.get(self.STAC_JSON)
-        try:
-            self.__granules_json = json.loads(raw_stac_json)
-            return self
-        except:
-            LOGGER.debug(f'raw_stac_json is not STAC_JSON: {raw_stac_json}. trying to see if file exists')
-        if not FileUtils.file_exist(raw_stac_json):
-            raise ValueError(f'missing file or not JSON: {raw_stac_json}')
-        self.__granules_json = FileUtils.read_json(raw_stac_json)
-        if self.__granules_json is None:
-            raise ValueError(f'{raw_stac_json} is not JSON')
-        return self
 
     def __set_props_from_env(self):
         missing_keys = [k for k in [self.STAC_JSON, self.DOWNLOAD_DIR_KEY] if k not in os.environ]
         if len(missing_keys) > 0:
             raise ValueError(f'missing environment keys: {missing_keys}')
-        self.__retrieve_stac_json()
-        self.__download_dir = os.environ.get(self.DOWNLOAD_DIR_KEY)
-        self.__download_dir = self.__download_dir[:-1] if self.__download_dir.endswith('/') else self.__download_dir
+        self._retrieve_stac_json()
+        self._setup_download_dir()
         return self
 
     def __get_downloading_urls(self, granules_result: list):
@@ -81,7 +61,7 @@ def __download_one_granule(self, assets: dict):
         for k, v in assets.items():
             try:
                 LOGGER.debug(f'downloading: {v["href"]}')
-                self.__s3.set_s3_url(v['href']).download(self.__download_dir)
+                self.__s3.set_s3_url(v['href']).download(self._download_dir)
             except Exception as e:
                 LOGGER.exception(f'failed to download {v}')
                 v['cause'] = str(e)
@@ -90,15 +70,13 @@ def __download_one_granule(self, assets: dict):
 
     def download(self, **kwargs) -> list:
         self.__set_props_from_env()
-        LOGGER.debug(f'creating download dir: {self.__download_dir}')
-        FileUtils.mk_dir_p(self.__download_dir)
-        downloading_urls = self.__get_downloading_urls(self.__granules_json)
+        downloading_urls = self.__get_downloading_urls(self._granules_json)
         error_list = []
         for each in downloading_urls:
             LOGGER.debug(f'working on {each}')
             current_error_list = self.__download_one_granule(each)
             error_list.extend(current_error_list)
         if len(error_list) > 0:
-            with open(f'{self.__download_dir}/error.log', 'w') as error_file:
+            with open(f'{self._download_dir}/error.log', 'w') as error_file:
                 error_file.write(json.dumps(error_list, indent=4))
         return downloading_urls

setup.py

@@ -18,7 +18,7 @@
 
 setup(
     name="cumulus_lambda_functions",
-    version="3.5.0",
+    version="3.6.0",
     packages=find_packages(),
     install_requires=install_requires,
     tests_require=['mock', 'nose', 'sphinx', 'sphinx_rtd_theme', 'coverage', 'pystac', 'python-dotenv', 'jsonschema'],