unity-sds · wphyojpl · May 5, 2023 · May 3, 2023 · May 3, 2023 · May 4, 2023
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [3.7.1] - 2023-05-04
+### Changed
+- [#148](https://github.com/unity-sds/unity-data-services/pull/148) fix: use cas structure to generate metadata for stac
+
 ## [3.7.0] - 2023-04-25
 ### Added
 - [#146](https://github.com/unity-sds/unity-data-services/pull/146) feat: Stac metadata extraction 

diff --git a/ci.cd/Makefile b/ci.cd/Makefile
@@ -43,7 +43,8 @@ update_lambda_function_mcp_dev_5:
 	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-cumulus_collections_creation_dapa_facade --publish &>/dev/null
 update_lambda_function_mcp_dev_6:
 	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-metadata_s4pa_generate_cmr --publish &>/dev/null
-
+update_lambda_function_mcp_dev_7:
+	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-metadata_stac_generate_cmr --publish &>/dev/null
 
 mcp_dev: upload_lambda_mcp_dev update_lambda_function_mcp_dev_1 update_lambda_function_mcp_dev_2 update_lambda_function_mcp_dev_4 update_lambda_function_mcp_dev_5
 

diff --git a/cumulus_lambda_functions/metadata_stac_generate_cmr/generate_cmr.py b/cumulus_lambda_functions/metadata_stac_generate_cmr/generate_cmr.py
@@ -1,3 +1,4 @@
+import hashlib
 import json
 import os
 from copy import deepcopy
@@ -9,7 +10,6 @@
 from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator
 from cumulus_lambda_functions.lib.metadata_extraction.echo_metadata import EchoMetadata
 from cumulus_lambda_functions.lib.time_utils import TimeUtils
-from cumulus_lambda_functions.metadata_s4pa_generate_cmr.pds_metadata import PdsMetadata
 from cumulus_lambda_functions.metadata_stac_generate_cmr.stac_input_metadata import StacInputMetadata
 
 LOGGER = LambdaLoggerGenerator.get_logger(__name__, LambdaLoggerGenerator.get_level_from_env())
@@ -47,6 +47,9 @@
                                                         "key": {
                                                             "type": "string"
                                                         },
+                                                        "url_path": {
+                                                            "type": "string"
+                                                        },
                                                         "source": {
                                                             "type": "string"
                                                         },
@@ -61,10 +64,12 @@
                                                         }
                                                     },
                                                     "required": [
-                                                        "bucket",
-                                                        "key",
                                                         "type"
-                                                    ]
+                                                    ],
+                                                    "oneOf": [
+                                                        {"required": ["bucket", "key"]},
+                                                        {"required": ["url_path"]}
+                                                    ],
                                                 }
                                             }
                                         },
@@ -73,16 +78,30 @@
                                             "files"
                                         ]
                                     }
+                                },
+                                "collection": {
+                                    "type": "object",
+                                    "properties": {
+                                        "name": {"type": "string"},
+                                        "version": {"type": "string"},
+                                    },
+                                    "required": ["name", "version"]
                                 }
                             },
                             "required": [
-                                "input_granules"
+                                "input_granules", "collection"
                             ]
                         }
                     },
                     "required": [
                         "meta"
                     ]
+                },
+                "extra_config": {
+                    "required": [],
+                    "properties": {
+                        "add_extra_keys": {"type": "boolean"}
+                    }
                 }
             },
             "required": []
@@ -111,13 +130,20 @@ def __validate_input(self):
 
     def __get_pds_metadata_file(self):
         self.__input_file_list = self.__event['cma']['event']['meta']['input_granules'][0]['files']
+        stac_metadata_file = None
         for each_file in self.__input_file_list:
+            if 'fileName' not in each_file and 'name' in each_file:  # add fileName if there is only name
+                each_file['fileName'] = each_file['name']
+            if 'url_path' in each_file:
+                s3_bucket, s3_key = self.__s3.split_s3_url(each_file['url_path'])
+                each_file['bucket'] = s3_bucket
+                each_file['key'] = s3_key
             LOGGER.debug(f'checking file: {each_file}')
             file_key_upper = each_file['key'].upper().strip()
             LOGGER.debug(f'checking file_key_upper: {file_key_upper} against {self.__file_postfixes}')
             if any([file_key_upper.endswith(k) for k in self.__file_postfixes]):
-                return each_file
-        return None
+                stac_metadata_file = each_file
+        return stac_metadata_file
 
     def __read_pds_metadata_file(self):
         self._pds_file_dict = self.__get_pds_metadata_file()
@@ -127,25 +153,213 @@ def __read_pds_metadata_file(self):
         self.__s3.target_key = self._pds_file_dict['key']
         return self.__s3.read_small_txt_file()
 
+    def __is_adding_extra_keys(self):
+        if 'extra_config' not in self.__event['cma']:
+            return True
+        if 'add_extra_keys' not in self.__event['cma']['extra_config']:
+            return True
+        return self.__event['cma']['extra_config']['add_extra_keys']
+
+    def __generate_output_dict(self, echo_metadata_md5: str):
+        output_dict = {
+            "checksumType": "md5",
+            "checksum": echo_metadata_md5,
+            "type": "metadata",
+
+            "key": self.__s3.target_key,
+            "fileName": os.path.basename(self.__s3.target_key),
+            "bucket": self.__s3.target_bucket,
+            "size": int(self.__s3.get_size()),
+        }
+        if not self.__is_adding_extra_keys():
+            return output_dict
+        output_dict = {**output_dict, **{
+            "path": os.path.dirname(self.__s3.target_key),
+            "name": os.path.basename(self.__s3.target_key),
+            "source_bucket": self.__s3.target_bucket,
+            "url_path": f's3://{self.__s3.target_bucket}/{self.__s3.target_key}',
+        }}
+        return output_dict
+
     def start(self):
+        """
+        sample event
+{
+  "cma": {
+    "task_config": {
+      "bucket": "{$.meta.buckets.internal.name}",
+      "collection": "{$.meta.collection}",
+      "cumulus_message": {
+        "outputs": [
+          {
+            "source": "{$.files}",
+            "destination": "{$.payload}"
+          }
+        ]
+      }
+    },
+    "event": {
+      "cumulus_meta": {
+        "cumulus_version": "11.1.1",
+        "execution_name": "90c9c978-ca5e-47b1-9c4a-3d20c73a4743",
+        "message_source": "sfn",
+        "queueExecutionLimits": {
+          "https://sqs.us-west-2.amazonaws.com/237868187491/uds-dev-cumulus-backgroundProcessing": 5
+        },
+        "state_machine": "arn:aws:states:us-west-2:237868187491:stateMachine:uds-dev-cumulus-IngestGranule",
+        "system_bucket": "uds-dev-cumulus-internal",
+        "workflow_start_time": 1655943753534,
+        "parentExecutionArn": "arn:aws:states:us-west-2:237868187491:execution:uds-dev-cumulus-DiscoverGranules:707b8f70-ac78-4fa8-86f6-b74dcdfed287",
+        "queueUrl": "arn:aws:sqs:us-west-2:237868187491:uds-dev-cumulus-startSF"
+      },
+      "exception": "None",
+      "meta": {
+        "buckets": {
+          "internal": {
+            "name": "uds-dev-cumulus-internal",
+            "type": "internal"
+          },
+          "private": {
+            "name": "uds-dev-cumulus-private",
+            "type": "private"
+          },
+          "protected": {
+            "name": "uds-dev-cumulus-protected",
+            "type": "protected"
+          },
+          "public": {
+            "name": "uds-dev-cumulus-public",
+            "type": "public"
+          },
+          "sps": {
+            "name": "uds-dev-cumulus-sps",
+            "type": "protected"
+          },
+          "staging": {
+            "name": "uds-dev-cumulus-staging",
+            "type": "internal"
+          }
+        },
+        "cmr": {
+          "clientId": "CHANGEME",
+          "cmrEnvironment": "UAT",
+          "cmrLimit": 100,
+          "cmrPageSize": 50,
+          "oauthProvider": "earthdata",
+          "passwordSecretName": "uds-dev-cumulus-message-template-cmr-password20220330223854670000000005",
+          "provider": "CHANGEME",
+          "username": "username"
+        },
+        "collection": {
+          "duplicateHandling": "replace",
+          "process": "snpp.level1",
+          "files": [
+            {
+              "bucket": "protected",
+              "regex": "^SNDR.SNPP.ATMS.L1A.*\\.nc$",
+              "reportToEms": false,
+              "sampleFileName": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc",
+              "type": "data"
+            },
+            {
+              "bucket": "protected",
+              "regex": "^SNDR.SNPP.ATMS.L1A.*\\.nc\\.cas$",
+              "reportToEms": false,
+              "sampleFileName": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc.cas",
+              "type": "metadata"
+            },
+            {
+              "bucket": "protected",
+              "regex": "^SNDR.SNPP.ATMS.L1A.*\\.nc\\.cmr\\.xml$",
+              "reportToEms": false,
+              "sampleFileName": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc.cmr.xml",
+              "type": "metadata"
+            }
+          ],
+          "granuleId": "^SNDR.SNPP.ATMS.L1A.*$",
+          "granuleIdExtraction": "(^SNDR.SNPP.ATMS.L1A.*)(\\.nc|\\.nc\\.cas|\\.nc\\.cmr\\.xml)",
+          "name": "SNDR_SNPP_ATMS_L1A_1",
+          "reportToEms": false,
+          "sampleFileName": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc",
+          "url_path": "{cmrMetadata.Granule.Collection.ShortName}",
+          "version": "1",
+          "updatedAt": 1655943525719,
+          "createdAt": 1655943525719
+        },
+        "process": "snpp.level1",
+        "distribution_endpoint": null,
+        "launchpad": {
+          "api": "launchpadApi",
+          "certificate": "launchpad.pfx",
+          "passphraseSecretName": ""
+        },
+        "provider": {
+          "id": "snpp_l1_s3",
+          "globalConnectionLimit": 1000,
+          "host": "uds-dev-cumulus-staging",
+          "protocol": "s3",
+          "createdAt": 1655943376376,
+          "updatedAt": 1655943376376
+        },
+        "stack": "uds-dev-cumulus",
+        "template": "s3://uds-dev-cumulus-internal/uds-dev-cumulus/workflow_template.json",
+        "workflow_name": "IngestGranule",
+        "workflow_tasks": {
+          "0": {
+            "name": "uds-dev-cumulus-SyncGranule",
+            "version": "$LATEST",
+            "arn": "arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-SyncGranule"
+          }
+        },
+        "staticValue": "aStaticValue",
+        "interpolatedValueStackName": "uds-dev-cumulus",
+        "input_granules": [
+          {
+            "granuleId": "SNDR.SNPP.ATMS.L1A.nominal2.01",
+            "dataType": "SNDR_SNPP_ATMS_L1A_1",
+            "version": "1",
+            "files": [
+              {
+                "size": 9194361,
+                "bucket": "uds-dev-cumulus-internal",
+                "key": "file-staging/uds-dev-cumulus/SNDR_SNPP_ATMS_L1A_1___1/SNDR.SNPP.ATMS.L1A.nominal2.01.nc",
+                "source": "SNDR_SNPP_ATMS_L1A/SNDR.SNPP.ATMS.L1A.nominal2.01.nc",
+                "fileName": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc",
+                "type": "data"
+              },
+              {
+                "size": 2673,
+                "bucket": "uds-dev-cumulus-internal",
+                "key": "file-staging/uds-dev-cumulus/SNDR_SNPP_ATMS_L1A_1___1/SNDR.SNPP.ATMS.L1A.nominal2.01.nc.cas",
+                "source": "SNDR_SNPP_ATMS_L1A/SNDR.SNPP.ATMS.L1A.nominal2.01.nc.cas",
+                "fileName": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc.cas",
+                "type": "metadata"
+              }
+            ],
+            "sync_granule_duration": 694,
+            "createdAt": 1656010982847
+          }
+        ]
+      },
+      "payload": {},
+      "replace": {
+        "Bucket": "uds-dev-cumulus-internal",
+        "Key": "events/172fbbc4-f8ee-4974-8a77-37bc669accb0",
+        "TargetPath": "$.payload"
+      }
+    }
+  }
+}
+        :return:
+        """
         self.__validate_input()
         LOGGER.error(f'input: {self.__event}')
         granules_metadata_props = StacInputMetadata(json.loads(self.__read_pds_metadata_file())).start()
         echo_metadata = EchoMetadata(granules_metadata_props).load().echo_metadata
         echo_metadata_xml_str = xmltodict.unparse(echo_metadata, pretty=True)
         self.__s3.target_key = os.path.join(os.path.dirname(self.__s3.target_key), f'{granules_metadata_props.granule_id}.cmr.xml')
         self.__s3.upload_bytes(echo_metadata_xml_str.encode())
-
-        # put payload
-        # remove replace
-        # add             "task_config": {
-        #                 "inputGranules": "{$.meta.input_granules}",
-        #                 "granuleIdExtraction": "{$.meta.collection.granuleIdExtraction}"
-        #             },
-        # return {
-        #     'files': ['example', 'mock', 'return'],
-        #     'granules': self.__event
-        # }
+        echo_metadata_md5 = hashlib.md5(echo_metadata_xml_str.encode()).hexdigest()
         returning_dict = deepcopy(self.__event['cma']['event'])
         if 'replace' in returning_dict:
             returning_dict.pop('replace')
@@ -194,12 +408,7 @@ def start(self):
                         "granuleId": self.__event['cma']['event']['meta']['input_granules'][0]['granuleId'],
                         "dataType": granules_metadata_props.collection_name,
                         "version": f'{granules_metadata_props.collection_version}',
-                        "files": self.__input_file_list + [{
-                                "key": self.__s3.target_key,
-                                "fileName": os.path.basename(self.__s3.target_key),
-                                "bucket": self.__s3.target_bucket,
-                                "size": int(self.__s3.get_size()),
-                            }],
+                        "files": self.__input_file_list + [self.__generate_output_dict(echo_metadata_md5)],
                         # "files": self.__input_file_list,
                         "sync_granule_duration": 20302,
                         "createdAt": TimeUtils.get_current_unix_milli(),

diff --git a/setup.py b/setup.py
@@ -18,7 +18,7 @@
 
 setup(
     name="cumulus_lambda_functions",
-    version="3.7.0",
+    version="3.7.1",
     packages=find_packages(),
     install_requires=install_requires,
     tests_require=['mock', 'nose', 'sphinx', 'sphinx_rtd_theme', 'coverage', 'pystac', 'python-dotenv', 'jsonschema'],