feat: Stac metadata extraction

CHANGELOG.md

@@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [3.7.0] - 2023-04-25
+### Added
+- [#146](https://github.com/unity-sds/unity-data-services/pull/146) feat: Stac metadata extraction 
+
 ## [3.6.1] - 2023-04-24
 ### Changed
 - [#144](https://github.com/unity-sds/unity-data-services/pull/144) fix: downloaded stac to return local absolute path

cumulus_lambda_functions/metadata_cas_generate_cmr/echo_metadata.py → cumulus_lambda_functions/lib/metadata_extraction/echo_metadata.py

@@ -73,7 +73,7 @@
 """
 from copy import deepcopy
 
-from cumulus_lambda_functions.metadata_cas_generate_cmr.l1a_input_metadata import L1AInputMetadata
+from cumulus_lambda_functions.lib.metadata_extraction.granule_metadata_props import GranuleMetadataProps
 
 SAMPLE_METADATA = {
     "GranuleUR": "MYD13Q1.A2017297.h19v10.006.2017313221202",
@@ -141,11 +141,8 @@
 
 
 class EchoMetadata:
-    def __init__(self, l1a_input_metadata: L1AInputMetadata, granule_id: str, collection_id: str, collection_version: str):
-        self.__l1a_input_metadata = l1a_input_metadata
-        self.__granule_id = granule_id
-        self.__collection_id = collection_id
-        self.__collection_version = collection_version
+    def __init__(self, granules_metadata_props: GranuleMetadataProps):
+        self.__granules_metadata_props = granules_metadata_props
         self.__echo_metadata = deepcopy(SAMPLE_METADATA)
 
     @property
@@ -163,21 +160,24 @@ def echo_metadata(self, val):
 
     def load(self):
         incomplete_metadata = {
-            'GranuleUR': self.__granule_id,
+            'GranuleUR': self.__granules_metadata_props.granule_id,
+            # 'InsertTime': '',  # conditionally add it after dict creation
             # 'LastUpdate': 'TODO',
             'Collection': {
-                'ShortName': self.__collection_id,
-                'VersionId': self.__collection_version,
+                'ShortName': self.__granules_metadata_props.collection_name,
+                'VersionId': self.__granules_metadata_props.collection_version,
             },
             'DataGranule': {
-                'ProductionDateTime': self.__l1a_input_metadata.prod_dt,
+                'ProductionDateTime': self.__granules_metadata_props.prod_dt,
             },
             'Temporal': {
                 'RangeDateTime': {
-                    'BeginningDateTime': self.__l1a_input_metadata.beginning_dt,
-                    'EndingDateTime': self.__l1a_input_metadata.ending_dt,
+                    'BeginningDateTime': self.__granules_metadata_props.beginning_dt,
+                    'EndingDateTime': self.__granules_metadata_props.ending_dt,
                 }
             }
         }
+        if self.__granules_metadata_props.insert_dt is not None:
+            incomplete_metadata['InsertTime'] = self.__granules_metadata_props.insert_dt
         self.__echo_metadata.update(incomplete_metadata)
         return self

cumulus_lambda_functions/lib/metadata_extraction/granule_metadata_props.py

@@ -0,0 +1,100 @@
+class GranuleMetadataProps:
+    def __init__(self):
+        self.__beginning_dt = None
+        self.__ending_dt = None
+        self.__collection_name = None
+        self.__collection_version = None
+        self.__granule_id = None
+        self.__prod_dt = None
+        self.__insert_dt = None
+
+    @property
+    def beginning_dt(self):
+        return self.__beginning_dt
+
+    @beginning_dt.setter
+    def beginning_dt(self, val):
+        """
+        :param val:
+        :return: None
+        """
+        self.__beginning_dt = val
+        return
+
+    @property
+    def ending_dt(self):
+        return self.__ending_dt
+
+    @ending_dt.setter
+    def ending_dt(self, val):
+        """
+        :param val:
+        :return: None
+        """
+        self.__ending_dt = val
+        return
+
+    @property
+    def collection_name(self):
+        return self.__collection_name
+
+    @collection_name.setter
+    def collection_name(self, val):
+        """
+        :param val:
+        :return: None
+        """
+        self.__collection_name = val
+        return
+
+    @property
+    def collection_version(self):
+        return self.__collection_version
+
+    @collection_version.setter
+    def collection_version(self, val):
+        """
+        :param val:
+        :return: None
+        """
+        self.__collection_version = val
+        return
+
+    @property
+    def granule_id(self):
+        return self.__granule_id
+
+    @granule_id.setter
+    def granule_id(self, val):
+        """
+        :param val:
+        :return: None
+        """
+        self.__granule_id = val
+        return
+
+    @property
+    def prod_dt(self):
+        return self.__prod_dt
+
+    @prod_dt.setter
+    def prod_dt(self, val):
+        """
+        :param val:
+        :return: None
+        """
+        self.__prod_dt = val
+        return
+
+    @property
+    def insert_dt(self):
+        return self.__insert_dt
+
+    @insert_dt.setter
+    def insert_dt(self, val):
+        """
+        :param val:
+        :return: None
+        """
+        self.__insert_dt = val
+        return

cumulus_lambda_functions/metadata_cas_generate_cmr/generate_cmr.py

@@ -7,8 +7,8 @@
 from cumulus_lambda_functions.lib.aws.aws_s3 import AwsS3
 from cumulus_lambda_functions.lib.json_validator import JsonValidator
 from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator
+from cumulus_lambda_functions.lib.metadata_extraction.echo_metadata import EchoMetadata
 from cumulus_lambda_functions.lib.time_utils import TimeUtils
-from cumulus_lambda_functions.metadata_cas_generate_cmr.echo_metadata import EchoMetadata
 from cumulus_lambda_functions.metadata_cas_generate_cmr.l1a_input_metadata import L1AInputMetadata
 
 LOGGER = LambdaLoggerGenerator.get_logger(__name__, LambdaLoggerGenerator.get_level_from_env())
@@ -349,13 +349,13 @@ def start(self):
         """
         self.__validate_input()
         LOGGER.error(f'input: {self.__event}')
-        pds_metadata = L1AInputMetadata(xmltodict.parse(self.__read_pds_metadata_file())).load()
-        granule_id = self.__event['cma']['event']['meta']['input_granules'][0]['granuleId']
-        collection_name = self.__event['cma']['event']['meta']['collection']['name']
-        collection_version = self.__event['cma']['event']['meta']['collection']['version']
-        echo_metadata = EchoMetadata(pds_metadata, granule_id, collection_name, collection_version).load().echo_metadata
+        granule_metadata_props = L1AInputMetadata(xmltodict.parse(self.__read_pds_metadata_file())).load()
+        granule_metadata_props.granule_id = self.__event['cma']['event']['meta']['input_granules'][0]['granuleId']
+        granule_metadata_props.collection_name = self.__event['cma']['event']['meta']['collection']['name']
+        granule_metadata_props.collection_version = self.__event['cma']['event']['meta']['collection']['version']
+        echo_metadata = EchoMetadata(granule_metadata_props).load().echo_metadata
         echo_metadata_xml_str = xmltodict.unparse(echo_metadata, pretty=True)
-        self.__s3.target_key = os.path.join(os.path.dirname(self.__s3.target_key), f'{granule_id}.cmr.xml')
+        self.__s3.target_key = os.path.join(os.path.dirname(self.__s3.target_key), f'{granule_metadata_props.granule_id}.cmr.xml')
         self.__s3.upload_bytes(echo_metadata_xml_str.encode())
         echo_metadata_md5 = hashlib.md5(echo_metadata_xml_str.encode()).hexdigest()
         returning_dict = deepcopy(self.__event['cma']['event'])
@@ -403,9 +403,9 @@ def start(self):
         returning_dict['payload'] = {
                 "granules": [
                     {
-                        "granuleId": granule_id,
-                        "dataType": collection_name,
-                        "version": collection_version,
+                        "granuleId": granule_metadata_props.granule_id,
+                        "dataType": granule_metadata_props.collection_name,
+                        "version": granule_metadata_props.collection_version,
                         "files": self.__input_file_list + [self.__generate_output_dict(echo_metadata_md5)],
                         # "files": self.__input_file_list,
                         "sync_granule_duration": 20302,

cumulus_lambda_functions/metadata_cas_generate_cmr/l1a_input_metadata.py

@@ -1,8 +1,6 @@
-import json
-
 from cumulus_lambda_functions.lib.json_validator import JsonValidator
 from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator
-
+from cumulus_lambda_functions.lib.metadata_extraction.granule_metadata_props import GranuleMetadataProps
 
 LOGGER = LambdaLoggerGenerator.get_logger(__name__, LambdaLoggerGenerator.get_level_from_env())
 L1A_INPUT_METADATA_SCHEMA = {
@@ -43,67 +41,11 @@ def __init__(self, input_l1a_metadata: dict):
         """
         :param input_l1a_metadata: str - XML string
         """
+        self.__granule_metadata_props = GranuleMetadataProps()
         self.__input_l1a_metadata = input_l1a_metadata
         self.__l1a_metadata_dict = {}
         self.__mandatory_keys = ['EndDateTime', 'ProductName', 'ProductionDateTime', 'StartDateTime']
 
-        self.__beginning_dt = None
-        self.__ending_dt = None
-        self.__prod_dt = None
-        self.__prod_name = None
-
-    @property
-    def beginning_dt(self):
-        return self.__beginning_dt
-
-    @beginning_dt.setter
-    def beginning_dt(self, val):
-        """
-        :param val:
-        :return: None
-        """
-        self.__beginning_dt = val
-        return
-
-    @property
-    def ending_dt(self):
-        return self.__ending_dt
-
-    @ending_dt.setter
-    def ending_dt(self, val):
-        """
-        :param val:
-        :return: None
-        """
-        self.__ending_dt = val
-        return
-
-    @property
-    def prod_dt(self):
-        return self.__prod_dt
-
-    @prod_dt.setter
-    def prod_dt(self, val):
-        """
-        :param val:
-        :return: None
-        """
-        self.__prod_dt = val
-        return
-
-    @property
-    def prod_name(self):
-        return self.__prod_name
-
-    @prod_name.setter
-    def prod_name(self, val):
-        """
-        :param val:
-        :return: None
-        """
-        self.__prod_name = val
-        return
-
     def __validate_pds_metadata(self):
         result = JsonValidator(L1A_INPUT_METADATA_SCHEMA).validate(self.__input_l1a_metadata)
         if result is not None:
@@ -120,8 +62,8 @@ def __load_to_dict(self):
     def load(self):
         self.__validate_pds_metadata()
         self.__load_to_dict()
-        self.beginning_dt = self.__l1a_metadata_dict['StartDateTime']
-        self.ending_dt = self.__l1a_metadata_dict['EndDateTime']
-        self.prod_dt = self.__l1a_metadata_dict['ProductionDateTime']
-        self.prod_name = self.__l1a_metadata_dict['ProductName']
-        return self
+        self.__granule_metadata_props.beginning_dt = self.__l1a_metadata_dict['StartDateTime']
+        self.__granule_metadata_props.ending_dt = self.__l1a_metadata_dict['EndDateTime']
+        self.__granule_metadata_props.prod_dt = self.__l1a_metadata_dict['ProductionDateTime']
+        # self.__granule_metadata_props.prod_name = self.__l1a_metadata_dict['ProductName']  # NOT in used at this moment.
+        return self.__granule_metadata_props