breaking: allow custom-metadata in granules dapa query

ci.cd/Makefile

@@ -46,6 +46,9 @@ mcp_dev_fastapi: upload_lambda_mcp_dev update_lambda_function_mcp_dev_uds_api
 update_lambda_function_mcp_sbx_7:
 	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-sbx-cumulus-metadata_stac_generate_cmr --publish &>/dev/null
 
-mcp_sbx: upload_lambda_mcp_dev update_lambda_function_mcp_sbx_7
+update_lambda_function_mcp_sbx_8:
+	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-sbx-cumulus-granules_to_es --publish &>/dev/null
+
+mcp_sbx: upload_lambda_mcp_dev update_lambda_function_mcp_sbx_7 update_lambda_function_mcp_sbx_8
 
 mcp_sbx_fastapi: upload_lambda_mcp_dev update_lambda_function_mcp_sbx_uds_api

cognito_readme.md

@@ -1,4 +1,4 @@
-- create a text file: `cognito.jpl.aws.json`
+- create a text file: `cognito.jpl.aws.json`..
 
         {
            "AuthParameters" : {

cumulus_lambda_functions/cumulus_stac/item_transformer.py

@@ -321,10 +321,33 @@ def __get_asset_name(self, input_dict):
         return input_dict['type']
 
     def __get_asset_obj(self, input_dict):
+        """
+      {
+        "bucket": "uds-sbx-cumulus-staging",
+        "checksum": "9817be382b87c48ebe482b9c47d1525a",
+        "checksumType": "md5",
+        "fileName": "test_file01.cmr.xml",
+        "key": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417:test_file01/test_file01.cmr.xml",
+        "size": 1768,
+        "source": "s3://uds-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417:test_file01/test_file01.cmr.xml",
+        "type": "metadata"
+      }
+        "bucket": "am-uds-dev-cumulus-internal",
+        "key": "ATMS_SCIENCE_Group___1/P1570515ATMSSCIENCEAAT16032024518500.PDS",
+        "size": 760,
+        "fileName": "P1570515ATMSSCIENCEAAT16032024518500.PDS",
+        "source": "data/SNPP_ATMS_Level0_T/ATMS_SCIENCE_Group/2016/031//P1570515ATMSSCIENCEAAT16032024518500.PDS",
+        "type": "data"
+
+        :param input_dict:
+        :return:
+        """
+        description_keys = ['size', 'checksumType', 'checksum']
+        descriptions = [f'{k}={input_dict[k]};' for k in description_keys if k in input_dict]
         asset = Asset(
             href=f"s3://{input_dict['bucket']}/{input_dict['key']}",
             title=input_dict['fileName'],
-            description=input_dict['fileName'],
+            description=''.join(descriptions),
         )
         return asset
 
@@ -404,16 +427,57 @@ def to_stac(self, source: dict) -> dict:
         validated_files = [k for k in source['files'] if cumulus_file_validator.validate(k) is None]
         custom_metadata = source['custom_metadata'] if 'custom_metadata' in source else {}
 
+        properties_template = [
+            {
+                'stac_key': 'datetime',
+                'cumulus_key': 'createdAt',
+                'execution': lambda x: f"{TimeUtils.decode_datetime(x['createdAt'], False)}Z"
+            },
+            {
+                'stac_key': 'start_datetime',
+                'cumulus_key': 'beginningDateTime',
+                'execution': lambda x: datetime_to_str(self.get_time_obj(x['beginningDateTime']))
+            },
+            {
+                'stac_key': 'end_datetime',
+                'cumulus_key': 'endingDateTime',
+                'execution': lambda x: datetime_to_str(self.get_time_obj(x['endingDateTime']))
+            },
+            {
+                'stac_key': 'created',
+                'cumulus_key': 'productionDateTime',
+                'execution': lambda x: datetime_to_str(self.get_time_obj(x['productionDateTime']))
+            },
+            {
+                'stac_key': 'updated',
+                'cumulus_key': 'updatedAt',
+                'execution': lambda x: datetime_to_str(TimeUtils().parse_from_unix(source['updatedAt'], True).get_datetime_obj())
+            },
+            {
+                'stac_key': 'status',
+                'cumulus_key': 'status',
+                'execution': lambda x: x['status']
+            },
+            {
+                'stac_key': 'provider',
+                'cumulus_key': 'provider',
+                'execution': lambda x: x['provider']
+            },
+        ]
+        cumulus_properties = {
+            k['stac_key']: k['execution'](source) for k in properties_template if k['cumulus_key'] in source
+        }
         stac_item = Item(
             id=source['granuleId'],
-            bbox=[0.0, 0.0, 0.0, 0.0],
+            bbox=[-180.0, -90.0, 180.0, 90.0],
             properties={
                 **custom_metadata,
+                **cumulus_properties,
                 # "datetime": f"{TimeUtils.decode_datetime(source['createdAt'], False)}Z",
-                "start_datetime": datetime_to_str(self.get_time_obj(source['beginningDateTime'])),
-                "end_datetime": datetime_to_str(self.get_time_obj(source['endingDateTime'])),
-                "created": datetime_to_str(self.get_time_obj(source['productionDateTime'])),
-                "updated": datetime_to_str(TimeUtils().parse_from_unix(source['updatedAt'], True).get_datetime_obj()),
+                # "start_datetime": datetime_to_str(self.get_time_obj(source['beginningDateTime'])),
+                # "end_datetime": datetime_to_str(self.get_time_obj(source['endingDateTime'])),
+                # "created": datetime_to_str(self.get_time_obj(source['productionDateTime'])),
+                # "updated": datetime_to_str(TimeUtils().parse_from_unix(source['updatedAt'], True).get_datetime_obj()),
             },
             collection=source['collectionId'],
             assets={self.__get_asset_name(k): self.__get_asset_obj(k) for k in validated_files},

cumulus_lambda_functions/cumulus_wrapper/query_granules.py

@@ -44,7 +44,7 @@ def with_time_from(self, from_time):
         return self
 
     def with_time_to(self, to_time):
-        self._conditions.append(f'{self.__beginning_time_key}__from={to_time}')
+        self._conditions.append(f'{self.__beginning_time_key}__to={to_time}')
         return self
 
     def with_time(self, input_time):

cumulus_lambda_functions/granules_to_es/granules_index_mapping.py

@@ -0,0 +1,89 @@
+class GranulesIndexMapping:
+    stac_mappings = {
+        "event_time": {"type": "long"},
+        "type": {"type": "keyword"},
+        "stac_version": {"type": "keyword"},
+        "id": {"type": "keyword"},
+        "collection": {"type": "keyword"},
+        "geometry": {"type": "geo_shape"},
+        "bbox": {"type": "geo_shape"},
+        "links": {
+            "type": "object",
+            "properties": {
+                "href": {"type": "keyword"},
+                "rel": {"type": "keyword"},
+                "type": {"type": "keyword"},
+                "title": {"type": "text"}
+            }
+        },
+        "stac_extensions": {"type": "object"},
+        "properties": {
+            "dynamic": "false",
+            "properties": {
+                "provider": {"type": "keyword"},
+                "status": {"type": "keyword"},
+                "datetime": {"type": "date", "format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd'T'HH:mm:ssZ||yyyy-MM-dd'T'HH:mm:ss'Z'||yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ||yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'||yyyy-MM-dd||epoch_millis"},
+                "updated": {"type": "date", "format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd'T'HH:mm:ssZ||yyyy-MM-dd'T'HH:mm:ss'Z'||yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ||yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'||yyyy-MM-dd||epoch_millis"},
+                "start_datetime": {"type": "date", "format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd'T'HH:mm:ssZ||yyyy-MM-dd'T'HH:mm:ss'Z'||yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ||yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'||yyyy-MM-dd||epoch_millis"},
+                "end_datetime": {"type": "date", "format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd'T'HH:mm:ssZ||yyyy-MM-dd'T'HH:mm:ss'Z'||yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ||yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'||yyyy-MM-dd||epoch_millis"},
+                "created": {"type": "date", "format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd'T'HH:mm:ssZ||yyyy-MM-dd'T'HH:mm:ss'Z'||yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ||yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'||yyyy-MM-dd||epoch_millis"},
+            },
+        },
+        "assets": {
+            "type": "nested",
+            "dynamic": "true",
+            "properties": {
+                "dynamic_field": {
+                    "type": "nested",
+                    "dynamic": "true",
+                    "properties": {
+                        "href": {"type": "keyword"},
+                        "title": {"type": "text"},
+                        "description": {"type": "text"}
+                    }
+                }
+            }
+        }
+    }
+    mappings = {
+        "eventTime": {"type": "long"},
+        "collectionId": {"type": "keyword"},
+        "createdAt": {"type": "long"},
+        "duration": {"type": "float"},
+        "error": {
+            "type": "object",
+            "properties": {
+                "Cause": {"type": "text"},
+                "Error": {"type": "text"}
+            }
+        },
+        "execution": {"type": "text"},
+        "granuleId": {"type": "keyword"},
+        "processingEndDateTime": {"type": "date"},
+        "processingStartDateTime": {"type": "date"},
+        "productVolume": {"type": "keyword"},
+        "provider": {"type": "keyword"},
+        "published": {"type": "boolean"},
+        "status": {"type": "keyword"},
+        "timestamp": {"type": "long"},
+        "timeToArchive": {"type": "float"},
+        "timeToPreprocess": {"type": "float"},
+        "updatedAt": {"type": "long"},
+        "files": {
+            "type": "object",
+            "properties": {
+                "bucket": {"type": "keyword"},
+                "checksum": {"type": "keyword"},
+                "checksumType": {"type": "keyword"},
+                "fileName": {"type": "keyword"},
+                "key": {"type": "keyword"},
+                "size": {"type": "integer"},
+                "source": {"type": "text"},
+                "type":{"type": "keyword"}
+            }
+        },
+        "beginningDateTime": {"type": "date"},
+        "endingDateTime": {"type": "date"},
+        "lastUpdateDateTime": {"type": "date"},
+        "productionDateTime": {"type": "date"},
+    }

cumulus_lambda_functions/granules_to_es/granules_indexer.py

@@ -0,0 +1,89 @@
+import json
+import os
+
+from cumulus_lambda_functions.cumulus_stac.item_transformer import ItemTransformer
+
+from cumulus_lambda_functions.lib.uds_db.uds_collections import UdsCollections
+
+from cumulus_lambda_functions.metadata_stac_generate_cmr.stac_input_metadata import StacInputMetadata
+
+from cumulus_lambda_functions.lib.aws.aws_s3 import AwsS3
+
+from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator
+
+from cumulus_lambda_functions.lib.json_validator import JsonValidator
+
+from cumulus_lambda_functions.lib.aws.aws_message_transformers import AwsMessageTransformers
+from cumulus_lambda_functions.uds_api.dapa.granules_db_index import GranulesDbIndex
+
+LOGGER = LambdaLoggerGenerator.get_logger(__name__, LambdaLoggerGenerator.get_level_from_env())
+
+
+class GranulesIndexer:
+    CUMULUS_SCHEMA = {
+        'type': 'object',
+        'required': ['event', 'record'],
+        'properties': {
+            'event': {'type': 'string'},
+            'record': {'type': 'object'},
+        }
+    }
+
+    def __init__(self, event) -> None:
+        self.__event = event
+        self.__cumulus_record = {}
+        self.__file_postfixes = os.getenv('FILE_POSTFIX', 'STAC.JSON')
+        self.__file_postfixes = [k.upper().strip() for k in self.__file_postfixes.split(',')]
+        self.__input_file_list = []
+        self.__s3 = AwsS3()
+
+
+    def __get_pds_metadata_file(self):
+        self.__input_file_list = self.__cumulus_record['files']
+        stac_metadata_file = None
+        for each_file in self.__input_file_list:
+            if 'fileName' not in each_file and 'name' in each_file:  # add fileName if there is only name
+                each_file['fileName'] = each_file['name']
+            if 'url_path' in each_file:
+                s3_bucket, s3_key = self.__s3.split_s3_url(each_file['url_path'])
+                each_file['bucket'] = s3_bucket
+                each_file['key'] = s3_key
+            LOGGER.debug(f'checking file: {each_file}')
+            file_key_upper = each_file['key'].upper().strip()
+            LOGGER.debug(f'checking file_key_upper: {file_key_upper} against {self.__file_postfixes}')
+            if any([file_key_upper.endswith(k) for k in self.__file_postfixes]):
+                stac_metadata_file = each_file
+        return stac_metadata_file
+
+    def __read_pds_metadata_file(self):
+        pds_file_dict = self.__get_pds_metadata_file()
+        if pds_file_dict is None:
+            raise ValueError('missing PDS metadata file')
+        self.__s3.target_bucket = pds_file_dict['bucket']
+        self.__s3.target_key = pds_file_dict['key']
+        return self.__s3.read_small_txt_file()
+
+    def start(self):
+        incoming_msg = AwsMessageTransformers().sqs_sns(self.__event)
+        result = JsonValidator(self.CUMULUS_SCHEMA).validate(incoming_msg)
+        if result is not None:
+            raise ValueError(f'input json has CUMULUS validation errors: {result}')
+        self.__cumulus_record = incoming_msg['record']
+        if len(self.__cumulus_record['files']) < 1:
+            # TODO ingest updating stage?
+            return
+        stac_input_meta = StacInputMetadata(json.loads(self.__read_pds_metadata_file()))
+        granules_metadata_props = stac_input_meta.start()
+        self.__cumulus_record['custom_metadata'] = stac_input_meta.custom_properties
+        stac_item = ItemTransformer().to_stac(self.__cumulus_record)
+        if 'bbox' in stac_item:
+            stac_item['bbox'] = GranulesDbIndex.to_es_bbox(stac_item['bbox'])
+        collection_identifier = UdsCollections.decode_identifier(self.__cumulus_record['collectionId'])
+        LOGGER.debug(f'stac_item: {stac_item}')
+        GranulesDbIndex().add_entry(collection_identifier.tenant,
+                                    collection_identifier.venue,
+                                    stac_item,
+                                    self.__cumulus_record['granuleId']
+                                    )
+        LOGGER.debug(f'added to GranulesDbIndex')
+        return self

cumulus_lambda_functions/granules_to_es/lambda_function.py

@@ -0,0 +1,15 @@
+from cumulus_lambda_functions.granules_to_es.granules_indexer import GranulesIndexer
+from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator
+
+
+def lambda_handler(event, context):
+    """
+{'cma': {'task_config': {'bucket': '{$.meta.buckets.internal.name}', 'collection': '{$.meta.collection}', 'cumulus_message': {'outputs': [{'source': '{$.files}', 'destination': '{$.payload}'}]}}, 'event': {'cumulus_meta': {'cumulus_version': '10.0.1', 'execution_name': 'c6d885dc-b4b2-4eb0-b22e-b6f58a7a0870', 'message_source': 'sfn', 'queueExecutionLimits': {'https://sqs.us-west-2.amazonaws.com/884500545225/am-uds-dev-cumulus-backgroundProcessing': 5}, 'state_machine': 'arn:aws:states:us-west-2:884500545225:stateMachine:am-uds-dev-cumulus-IngestGranule', 'system_bucket': 'am-uds-dev-cumulus-internal', 'workflow_start_time': 1646785175509, 'parentExecutionArn': 'arn:aws:states:us-west-2:884500545225:execution:am-uds-dev-cumulus-DiscoverGranules:885483b4-ba55-4db1-b197-661e1e595a45', 'queueUrl': 'arn:aws:sqs:us-west-2:884500545225:am-uds-dev-cumulus-startSF'}, 'exception': 'None', 'meta': {'buckets': {'internal': {'name': 'am-uds-dev-cumulus-internal', 'type': 'internal'}, 'protected': {'name': 'am-uds-dev-cumulus-protected', 'type': 'protected'}}, 'cmr': {'clientId': 'CHANGEME', 'cmrEnvironment': 'UAT', 'cmrLimit': 100, 'cmrPageSize': 50, 'oauthProvider': 'earthdata', 'passwordSecretName': 'am-uds-dev-cumulus-message-template-cmr-password20220216072916956000000002', 'provider': 'CHANGEME', 'username': 'username'}, 'collection': {'name': 'ATMS_SCIENCE_Group_2011', 'version': '001', 'process': 'modis', 'granuleId': '^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}0$', 'granuleIdExtraction': '(P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}0).+', 'sampleFileName': 'P1570515ATMSSCIENCEAXT11344000000001.PDS', 'duplicateHandling': 'replace', 'url_path': '{cmrMetadata.Granule.Collection.ShortName}___{cmrMetadata.Granule.Collection.VersionId}', 'provider_path': '/data/SNPP_ATMS_Level0_T/ATMS_SCIENCE_Group/2011/', 'files': [{'bucket': 'internal', 'regex': '^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}00\\.PDS$', 'sampleFileName': 'P1570515ATMSSCIENCEAXT11344000000000.PDS', 'type': 'data'}, {'bucket': 'internal', 'regex': '^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}01\\.PDS$', 'sampleFileName': 'P1570515ATMSSCIENCEAXT11344000000001.PDS', 'type': 'metadata'}, {'bucket': 'internal', 'regex': '^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}01\\.PDS\\.xml$', 'sampleFileName': 'P1570515ATMSSCIENCEAXT11344000000001.PDS.xml', 'type': 'metadata'}, {'bucket': 'internal', 'regex': '^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}0\\.cmr\\.xml$', 'sampleFileName': 'P1570515ATMSSCIENCEAXT11344000000001.PDS.xml', 'type': 'metadata'}], 'updatedAt': 1646326197526, 'createdAt': 1646258167624}, 'distribution_endpoint': 's3://am-uds-dev-cumulus-internal/', 'launchpad': {'api': 'launchpadApi', 'certificate': 'launchpad.pfx', 'passphraseSecretName': ''}, 'provider': {'password': 'AQICAHhSagsGDAl5tQWM010IEvxKgj2LcsNub5v5FHoRpOjXcQHFbE4iMnF/W0Y/NrsYvrfHAAAAajBoBgkqhkiG9w0BBwagWzBZAgEAMFQGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMLaH13SdxPXREjXLtAgEQgCfA+lEu2c/xLTGwJsbtKlXJbKDy4pwV+rS3BnJqgBoLLMQZqOdoFhk=', 'host': 'snppl0.gesdisc.eosdis.nasa.gov', 'updatedAt': 1646244053419, 'protocol': 'https', 'createdAt': 1646244053419, 'encrypted': True, 'username': 'AQICAHhSagsGDAl5tQWM010IEvxKgj2LcsNub5v5FHoRpOjXcQGRoY5EBMpvvyMASUowBM61AAAAYzBhBgkqhkiG9w0BBwagVDBSAgEAME0GCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQM9OhRHwTuxiz74q4UAgEQgCDEHOhsVG6+LqXfnlw+Z3Wg9MDOCd9/K5/X5j3tPJYkaA==', 'allowedRedirects': ['https://urs.earthdata.nasa.gov', 'urs.earthdata.nasa.gov'], 'id': 'snpp_provider_02', 'globalConnectionLimit': 10}, 'stack': 'am-uds-dev-cumulus', 'template': 's3://am-uds-dev-cumulus-internal/am-uds-dev-cumulus/workflow_template.json', 'workflow_name': 'IngestGranule', 'workflow_tasks': {'SyncGranule': {'name': 'am-uds-dev-cumulus-SyncGranule', 'version': '$LATEST', 'arn': 'arn:aws:lambda:us-west-2:884500545225:function:am-uds-dev-cumulus-SyncGranule'}}, 'staticValue': 'aStaticValue', 'interpolatedValueStackName': 'am-uds-dev-cumulus', 'input_granules': [{'granuleId': 'P1570515ATMSSCIENCEAXT1134912000000', 'dataType': 'ATMS_SCIENCE_Group_2011', 'version': '001', 'files': [{'bucket': 'am-uds-dev-cumulus-internal', 'key': 'file-staging/am-uds-dev-cumulus/ATMS_SCIENCE_Group_2011___001/P1570515ATMSSCIENCEAXT11349120000000.PDS', 'source': 'data/SNPP_ATMS_Level0_T/ATMS_SCIENCE_Group/2011/349//P1570515ATMSSCIENCEAXT11349120000000.PDS', 'fileName': 'P1570515ATMSSCIENCEAXT11349120000000.PDS', 'type': 'data', 'size': 744}, {'bucket': 'am-uds-dev-cumulus-internal', 'key': 'file-staging/am-uds-dev-cumulus/ATMS_SCIENCE_Group_2011___001/P1570515ATMSSCIENCEAXT11349120000001.PDS', 'source': 'data/SNPP_ATMS_Level0_T/ATMS_SCIENCE_Group/2011/349//P1570515ATMSSCIENCEAXT11349120000001.PDS', 'fileName': 'P1570515ATMSSCIENCEAXT11349120000001.PDS', 'type': 'metadata', 'size': 18084600}, {'bucket': 'am-uds-dev-cumulus-internal', 'key': 'file-staging/am-uds-dev-cumulus/ATMS_SCIENCE_Group_2011___001/P1570515ATMSSCIENCEAXT11349120000001.PDS.xml', 'source': 'data/SNPP_ATMS_Level0_T/ATMS_SCIENCE_Group/2011/349//P1570515ATMSSCIENCEAXT11349120000001.PDS.xml', 'fileName': 'P1570515ATMSSCIENCEAXT11349120000001.PDS.xml', 'type': 'metadata', 'size': 9526}], 'sync_granule_duration': 9822, 'createdAt': 1647386972717}], 'process': 'modis'}, 'payload': {}, 'replace': {'Bucket': 'am-uds-dev-cumulus-internal', 'Key': 'events/5d8edf37-0a18-4af5-a76f-7c2091cdd1e2', 'TargetPath': '$.payload'}}}}
+    :param event:
+    :param context:
+    :return:
+    """
+    LambdaLoggerGenerator.remove_default_handlers()
+    # TODO implement
+    GranulesIndexer(event).start()
+    return {}