Merge pull request #150 from unity-sds/develop

release/3.8.0

Author: ngachung

CHANGELOG.md

@@ -5,6 +5,18 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [3.8.0] - 2023-05-04
+### Added
+- [#149](https://github.com/unity-sds/unity-data-services/pull/149) feat: writing output content to a file if ENV is provided
+
+## [3.7.1] - 2023-05-04
+### Changed
+- [#148](https://github.com/unity-sds/unity-data-services/pull/148) fix: use cas structure to generate metadata for stac
+
+## [3.7.0] - 2023-04-25
+### Added
+- [#146](https://github.com/unity-sds/unity-data-services/pull/146) feat: Stac metadata extraction 
+
 ## [3.6.1] - 2023-04-24
 ### Changed
 - [#144](https://github.com/unity-sds/unity-data-services/pull/144) fix: downloaded stac to return local absolute path

ci.cd/Makefile

@@ -43,7 +43,8 @@ update_lambda_function_mcp_dev_5:
 	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-cumulus_collections_creation_dapa_facade --publish &>/dev/null
 update_lambda_function_mcp_dev_6:
 	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-metadata_s4pa_generate_cmr --publish &>/dev/null
-
+update_lambda_function_mcp_dev_7:
+	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-metadata_stac_generate_cmr --publish &>/dev/null
 
 mcp_dev: upload_lambda_mcp_dev update_lambda_function_mcp_dev_1 update_lambda_function_mcp_dev_2 update_lambda_function_mcp_dev_4 update_lambda_function_mcp_dev_5
 

cumulus_lambda_functions/docker_entrypoint/__main__.py

@@ -4,24 +4,32 @@
 
 from cumulus_lambda_functions.stage_in_out.catalog_granules_factory import CatalogGranulesFactory
 from cumulus_lambda_functions.stage_in_out.download_granules_factory import DownloadGranulesFactory
-from cumulus_lambda_functions.stage_in_out.download_granules_s3 import DownloadGranulesS3
 from cumulus_lambda_functions.stage_in_out.search_granules_factory import SearchGranulesFactory
+from cumulus_lambda_functions.stage_in_out.stage_in_out_utils import StageInOutUtils
 from cumulus_lambda_functions.stage_in_out.upoad_granules_factory import UploadGranulesFactory
 
 
 def choose_process():
     if argv[1].strip().upper() == 'SEARCH':
         logging.info('starting SEARCH script')
-        return SearchGranulesFactory().get_class(os.getenv('GRANULES_SEARCH_DOMAIN', 'MISSING_GRANULES_SEARCH_DOMAIN')).search()
+        result_str = SearchGranulesFactory().get_class(os.getenv('GRANULES_SEARCH_DOMAIN', 'MISSING_GRANULES_SEARCH_DOMAIN')).search()
+        StageInOutUtils.write_output_to_file(result_str)
+        return result_str
     if argv[1].strip().upper() == 'DOWNLOAD':
         logging.info('starting DOWNLOAD script')
-        return DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download()
+        result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download()
+        StageInOutUtils.write_output_to_file(result_str)
+        return result_str
     if argv[1].strip().upper() == 'UPLOAD':
         logging.info('starting UPLOAD script')
-        return UploadGranulesFactory().get_class(os.getenv('GRANULES_UPLOAD_TYPE', 'MISSING_GRANULES_UPLOAD_TYPE')).upload()
+        result_str = UploadGranulesFactory().get_class(os.getenv('GRANULES_UPLOAD_TYPE', 'MISSING_GRANULES_UPLOAD_TYPE')).upload()
+        StageInOutUtils.write_output_to_file(result_str)
+        return result_str
     if argv[1].strip().upper() == 'CATALOG':
         logging.info('starting CATALOG script')
-        return CatalogGranulesFactory().get_class(os.getenv('GRANULES_CATALOG_TYPE', 'MISSING_GRANULES_CATALOG_TYPE')).catalog()
+        result_str = CatalogGranulesFactory().get_class(os.getenv('GRANULES_CATALOG_TYPE', 'MISSING_GRANULES_CATALOG_TYPE')).catalog()
+        StageInOutUtils.write_output_to_file(result_str)
+        return result_str
     raise ValueError(f'invalid argument: {argv}')
 
 

tests/cumulus_lambda_functions/snpp_level1a_generate_cmr/__init__.py renamed to cumulus_lambda_functions/lib/metadata_extraction/__init__.py

@@ -73,7 +73,7 @@
 """
 from copy import deepcopy
 
-from cumulus_lambda_functions.metadata_cas_generate_cmr.l1a_input_metadata import L1AInputMetadata
+from cumulus_lambda_functions.lib.metadata_extraction.granule_metadata_props import GranuleMetadataProps
 
 SAMPLE_METADATA = {
     "GranuleUR": "MYD13Q1.A2017297.h19v10.006.2017313221202",
@@ -141,11 +141,8 @@
 
 
 class EchoMetadata:
-    def __init__(self, l1a_input_metadata: L1AInputMetadata, granule_id: str, collection_id: str, collection_version: str):
-        self.__l1a_input_metadata = l1a_input_metadata
-        self.__granule_id = granule_id
-        self.__collection_id = collection_id
-        self.__collection_version = collection_version
+    def __init__(self, granules_metadata_props: GranuleMetadataProps):
+        self.__granules_metadata_props = granules_metadata_props
         self.__echo_metadata = deepcopy(SAMPLE_METADATA)
 
     @property
@@ -163,21 +160,24 @@ def echo_metadata(self, val):
 
     def load(self):
         incomplete_metadata = {
-            'GranuleUR': self.__granule_id,
+            'GranuleUR': self.__granules_metadata_props.granule_id,
+            # 'InsertTime': '',  # conditionally add it after dict creation
             # 'LastUpdate': 'TODO',
             'Collection': {
-                'ShortName': self.__collection_id,
-                'VersionId': self.__collection_version,
+                'ShortName': self.__granules_metadata_props.collection_name,
+                'VersionId': self.__granules_metadata_props.collection_version,
             },
             'DataGranule': {
-                'ProductionDateTime': self.__l1a_input_metadata.prod_dt,
+                'ProductionDateTime': self.__granules_metadata_props.prod_dt,
             },
             'Temporal': {
                 'RangeDateTime': {
-                    'BeginningDateTime': self.__l1a_input_metadata.beginning_dt,
-                    'EndingDateTime': self.__l1a_input_metadata.ending_dt,
+                    'BeginningDateTime': self.__granules_metadata_props.beginning_dt,
+                    'EndingDateTime': self.__granules_metadata_props.ending_dt,
                 }
             }
         }
+        if self.__granules_metadata_props.insert_dt is not None:
+            incomplete_metadata['InsertTime'] = self.__granules_metadata_props.insert_dt
         self.__echo_metadata.update(incomplete_metadata)
         return self

cumulus_lambda_functions/metadata_cas_generate_cmr/echo_metadata.py renamed to cumulus_lambda_functions/lib/metadata_extraction/echo_metadata.py

@@ -0,0 +1,100 @@
+class GranuleMetadataProps:
+    def __init__(self):
+        self.__beginning_dt = None
+        self.__ending_dt = None
+        self.__collection_name = None
+        self.__collection_version = None
+        self.__granule_id = None
+        self.__prod_dt = None
+        self.__insert_dt = None
+
+    @property
+    def beginning_dt(self):
+        return self.__beginning_dt
+
+    @beginning_dt.setter
+    def beginning_dt(self, val):
+        """
+        :param val:
+        :return: None
+        """
+        self.__beginning_dt = val
+        return
+
+    @property
+    def ending_dt(self):
+        return self.__ending_dt
+
+    @ending_dt.setter
+    def ending_dt(self, val):
+        """
+        :param val:
+        :return: None
+        """
+        self.__ending_dt = val
+        return
+
+    @property
+    def collection_name(self):
+        return self.__collection_name
+
+    @collection_name.setter
+    def collection_name(self, val):
+        """
+        :param val:
+        :return: None
+        """
+        self.__collection_name = val
+        return
+
+    @property
+    def collection_version(self):
+        return self.__collection_version
+
+    @collection_version.setter
+    def collection_version(self, val):
+        """
+        :param val:
+        :return: None
+        """
+        self.__collection_version = val
+        return
+
+    @property
+    def granule_id(self):
+        return self.__granule_id
+
+    @granule_id.setter
+    def granule_id(self, val):
+        """
+        :param val:
+        :return: None
+        """
+        self.__granule_id = val
+        return
+
+    @property
+    def prod_dt(self):
+        return self.__prod_dt
+
+    @prod_dt.setter
+    def prod_dt(self, val):
+        """
+        :param val:
+        :return: None
+        """
+        self.__prod_dt = val
+        return
+
+    @property
+    def insert_dt(self):
+        return self.__insert_dt
+
+    @insert_dt.setter
+    def insert_dt(self, val):
+        """
+        :param val:
+        :return: None
+        """
+        self.__insert_dt = val
+        return

cumulus_lambda_functions/lib/metadata_extraction/granule_metadata_props.py

@@ -7,8 +7,8 @@
 from cumulus_lambda_functions.lib.aws.aws_s3 import AwsS3
 from cumulus_lambda_functions.lib.json_validator import JsonValidator
 from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator
+from cumulus_lambda_functions.lib.metadata_extraction.echo_metadata import EchoMetadata
 from cumulus_lambda_functions.lib.time_utils import TimeUtils
-from cumulus_lambda_functions.metadata_cas_generate_cmr.echo_metadata import EchoMetadata
 from cumulus_lambda_functions.metadata_cas_generate_cmr.l1a_input_metadata import L1AInputMetadata
 
 LOGGER = LambdaLoggerGenerator.get_logger(__name__, LambdaLoggerGenerator.get_level_from_env())
@@ -349,13 +349,13 @@ def start(self):
         """
         self.__validate_input()
         LOGGER.error(f'input: {self.__event}')
-        pds_metadata = L1AInputMetadata(xmltodict.parse(self.__read_pds_metadata_file())).load()
-        granule_id = self.__event['cma']['event']['meta']['input_granules'][0]['granuleId']
-        collection_name = self.__event['cma']['event']['meta']['collection']['name']
-        collection_version = self.__event['cma']['event']['meta']['collection']['version']
-        echo_metadata = EchoMetadata(pds_metadata, granule_id, collection_name, collection_version).load().echo_metadata
+        granule_metadata_props = L1AInputMetadata(xmltodict.parse(self.__read_pds_metadata_file())).load()
+        granule_metadata_props.granule_id = self.__event['cma']['event']['meta']['input_granules'][0]['granuleId']
+        granule_metadata_props.collection_name = self.__event['cma']['event']['meta']['collection']['name']
+        granule_metadata_props.collection_version = self.__event['cma']['event']['meta']['collection']['version']
+        echo_metadata = EchoMetadata(granule_metadata_props).load().echo_metadata
         echo_metadata_xml_str = xmltodict.unparse(echo_metadata, pretty=True)
-        self.__s3.target_key = os.path.join(os.path.dirname(self.__s3.target_key), f'{granule_id}.cmr.xml')
+        self.__s3.target_key = os.path.join(os.path.dirname(self.__s3.target_key), f'{granule_metadata_props.granule_id}.cmr.xml')
         self.__s3.upload_bytes(echo_metadata_xml_str.encode())
         echo_metadata_md5 = hashlib.md5(echo_metadata_xml_str.encode()).hexdigest()
         returning_dict = deepcopy(self.__event['cma']['event'])
@@ -403,9 +403,9 @@ def start(self):
         returning_dict['payload'] = {
                 "granules": [
                     {
-                        "granuleId": granule_id,
-                        "dataType": collection_name,
-                        "version": collection_version,
+                        "granuleId": granule_metadata_props.granule_id,
+                        "dataType": granule_metadata_props.collection_name,
+                        "version": granule_metadata_props.collection_version,
                         "files": self.__input_file_list + [self.__generate_output_dict(echo_metadata_md5)],
                         # "files": self.__input_file_list,
                         "sync_granule_duration": 20302,

cumulus_lambda_functions/metadata_cas_generate_cmr/generate_cmr.py

@@ -1,8 +1,6 @@
-import json
-
 from cumulus_lambda_functions.lib.json_validator import JsonValidator
 from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator
-
+from cumulus_lambda_functions.lib.metadata_extraction.granule_metadata_props import GranuleMetadataProps
 
 LOGGER = LambdaLoggerGenerator.get_logger(__name__, LambdaLoggerGenerator.get_level_from_env())
 L1A_INPUT_METADATA_SCHEMA = {
@@ -43,67 +41,11 @@ def __init__(self, input_l1a_metadata: dict):
         """
         :param input_l1a_metadata: str - XML string
         """
+        self.__granule_metadata_props = GranuleMetadataProps()
         self.__input_l1a_metadata = input_l1a_metadata
         self.__l1a_metadata_dict = {}
         self.__mandatory_keys = ['EndDateTime', 'ProductName', 'ProductionDateTime', 'StartDateTime']
 
-        self.__beginning_dt = None
-        self.__ending_dt = None
-        self.__prod_dt = None
-        self.__prod_name = None
-
-    @property
-    def beginning_dt(self):
-        return self.__beginning_dt
-
-    @beginning_dt.setter
-    def beginning_dt(self, val):
-        """
-        :param val:
-        :return: None
-        """
-        self.__beginning_dt = val
-        return
-
-    @property
-    def ending_dt(self):
-        return self.__ending_dt
-
-    @ending_dt.setter
-    def ending_dt(self, val):
-        """
-        :param val:
-        :return: None
-        """
-        self.__ending_dt = val
-        return
-
-    @property
-    def prod_dt(self):
-        return self.__prod_dt
-
-    @prod_dt.setter
-    def prod_dt(self, val):
-        """
-        :param val:
-        :return: None
-        """
-        self.__prod_dt = val
-        return
-
-    @property
-    def prod_name(self):
-        return self.__prod_name
-
-    @prod_name.setter
-    def prod_name(self, val):
-        """
-        :param val:
-        :return: None
-        """
-        self.__prod_name = val
-        return
-
     def __validate_pds_metadata(self):
         result = JsonValidator(L1A_INPUT_METADATA_SCHEMA).validate(self.__input_l1a_metadata)
         if result is not None:
@@ -120,8 +62,8 @@ def __load_to_dict(self):
     def load(self):
         self.__validate_pds_metadata()
         self.__load_to_dict()
-        self.beginning_dt = self.__l1a_metadata_dict['StartDateTime']
-        self.ending_dt = self.__l1a_metadata_dict['EndDateTime']
-        self.prod_dt = self.__l1a_metadata_dict['ProductionDateTime']
-        self.prod_name = self.__l1a_metadata_dict['ProductName']
-        return self
+        self.__granule_metadata_props.beginning_dt = self.__l1a_metadata_dict['StartDateTime']
+        self.__granule_metadata_props.ending_dt = self.__l1a_metadata_dict['EndDateTime']
+        self.__granule_metadata_props.prod_dt = self.__l1a_metadata_dict['ProductionDateTime']
+        # self.__granule_metadata_props.prod_name = self.__l1a_metadata_dict['ProductName']  # NOT in used at this moment.
+        return self.__granule_metadata_props