breaking: configurable file postfixes for PDS metadata extraction (#124)

wphyojpl · web-flow · commit 5cc8c98dba35 · 2023-04-05T16:44:44.000-07:00
* feat: configurable file postfixes for PDS metadata extraction

* fix: some bugfix + update manual deployment update

* breaking: rename lvl0 and lvl1 to s4pa and cas

* fix: rename output variables

* chore: update contributing
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [3.0.0] - 2023-03-27
+### Breaking
+- [#124](https://github.com/unity-sds/unity-data-services/pull/124) configurable file postfixes for PDS metadata extraction + rename function names which will break previous terraforms
+
 ## [2.0.0] - 2023-01-23
 ### Breaking
 - [#120](https://github.com/unity-sds/unity-data-services/pull/120) breakup upload and download dockers into search + download & upload + catalog
diff --git a/ci.cd/Makefile b/ci.cd/Makefile
@@ -41,6 +41,9 @@ update_lambda_function_mcp_dev_4:
 	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-cumulus_collections_creation_dapa --publish &>/dev/null
 update_lambda_function_mcp_dev_5:
 	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-cumulus_collections_creation_dapa_facade --publish &>/dev/null
+update_lambda_function_mcp_dev_6:
+	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-metadata_s4pa_generate_cmr --publish &>/dev/null
+
 
 mcp_dev: upload_lambda_mcp_dev update_lambda_function_mcp_dev_1 update_lambda_function_mcp_dev_2 update_lambda_function_mcp_dev_4 update_lambda_function_mcp_dev_5
 
diff --git a/cumulus_lambda_functions/metadata_cas_generate_cmr/__init__.py b/cumulus_lambda_functions/metadata_cas_generate_cmr/__init__.py
diff --git a/cumulus_lambda_functions/metadata_cas_generate_cmr/echo_metadata.py b/cumulus_lambda_functions/metadata_cas_generate_cmr/echo_metadata.py
@@ -73,7 +73,7 @@
 """
 from copy import deepcopy
 
-from cumulus_lambda_functions.snpp_level1a_generate_cmr.l1a_input_metadata import L1AInputMetadata
+from cumulus_lambda_functions.metadata_cas_generate_cmr.l1a_input_metadata import L1AInputMetadata
 
 SAMPLE_METADATA = {
     "GranuleUR": "MYD13Q1.A2017297.h19v10.006.2017313221202",
diff --git a/cumulus_lambda_functions/metadata_cas_generate_cmr/generate_cmr.py b/cumulus_lambda_functions/metadata_cas_generate_cmr/generate_cmr.py
@@ -8,8 +8,8 @@
 from cumulus_lambda_functions.lib.json_validator import JsonValidator
 from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator
 from cumulus_lambda_functions.lib.time_utils import TimeUtils
-from cumulus_lambda_functions.snpp_level1a_generate_cmr.echo_metadata import EchoMetadata
-from cumulus_lambda_functions.snpp_level1a_generate_cmr.l1a_input_metadata import L1AInputMetadata
+from cumulus_lambda_functions.metadata_cas_generate_cmr.echo_metadata import EchoMetadata
+from cumulus_lambda_functions.metadata_cas_generate_cmr.l1a_input_metadata import L1AInputMetadata
 
 LOGGER = LambdaLoggerGenerator.get_logger(__name__, LambdaLoggerGenerator.get_level_from_env())
 
diff --git a/cumulus_lambda_functions/metadata_cas_generate_cmr/l1a_input_metadata.py b/cumulus_lambda_functions/metadata_cas_generate_cmr/l1a_input_metadata.py
diff --git a/cumulus_lambda_functions/metadata_cas_generate_cmr/lambda_function.py b/cumulus_lambda_functions/metadata_cas_generate_cmr/lambda_function.py
@@ -1,7 +1,7 @@
 import json
 
 from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator
-from cumulus_lambda_functions.snpp_level1a_generate_cmr.generate_cmr import GenerateCmr
+from cumulus_lambda_functions.metadata_cas_generate_cmr.generate_cmr import GenerateCmr
 
 
 def lambda_handler(event, context):
diff --git a/cumulus_lambda_functions/metadata_s4pa_generate_cmr/__init__.py b/cumulus_lambda_functions/metadata_s4pa_generate_cmr/__init__.py
diff --git a/cumulus_lambda_functions/metadata_s4pa_generate_cmr/__main__.py b/cumulus_lambda_functions/metadata_s4pa_generate_cmr/__main__.py
diff --git a/cumulus_lambda_functions/metadata_s4pa_generate_cmr/echo_metadata.py b/cumulus_lambda_functions/metadata_s4pa_generate_cmr/echo_metadata.py
@@ -73,7 +73,7 @@
 """
 from copy import deepcopy
 
-from cumulus_lambda_functions.snpp_lvl0_generate_cmr.pds_metadata import PdsMetadata
+from cumulus_lambda_functions.metadata_s4pa_generate_cmr.pds_metadata import PdsMetadata
 
 SAMPLE_METADATA = {
     "GranuleUR": "MYD13Q1.A2017297.h19v10.006.2017313221202",
diff --git a/cumulus_lambda_functions/metadata_s4pa_generate_cmr/generate_cmr.py b/cumulus_lambda_functions/metadata_s4pa_generate_cmr/generate_cmr.py
@@ -7,8 +7,8 @@
 from cumulus_lambda_functions.lib.json_validator import JsonValidator
 from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator
 from cumulus_lambda_functions.lib.time_utils import TimeUtils
-from cumulus_lambda_functions.snpp_lvl0_generate_cmr.echo_metadata import EchoMetadata
-from cumulus_lambda_functions.snpp_lvl0_generate_cmr.pds_metadata import PdsMetadata
+from cumulus_lambda_functions.metadata_s4pa_generate_cmr.echo_metadata import EchoMetadata
+from cumulus_lambda_functions.metadata_s4pa_generate_cmr.pds_metadata import PdsMetadata
 
 LOGGER = LambdaLoggerGenerator.get_logger(__name__, LambdaLoggerGenerator.get_level_from_env())
 
@@ -97,6 +97,8 @@ def __init__(self, event):
         self.__event = event
         self.__s3 = AwsS3()
         self._pds_file_dict = None
+        self.__file_postfixes = os.getenv('FILE_POSTFIX', '1.PDS.XML, NC.XML')
+        self.__file_postfixes = [k.upper().strip() for k in self.__file_postfixes.split(',')]
         self.__input_file_list = []
 
     def __validate_input(self):
@@ -109,7 +111,9 @@ def __get_pds_metadata_file(self):
         self.__input_file_list = self.__event['cma']['event']['meta']['input_granules'][0]['files']
         for each_file in self.__input_file_list:
             LOGGER.debug(f'checking file: {each_file}')
-            if each_file['key'].upper().endswith('1.PDS.XML'):
+            file_key_upper = each_file['key'].upper().strip()
+            LOGGER.debug(f'checking file_key_upper: {file_key_upper} against {self.__file_postfixes}')
+            if any([file_key_upper.endswith(k) for k in self.__file_postfixes]):
                 return each_file
         return None
 
diff --git a/cumulus_lambda_functions/metadata_s4pa_generate_cmr/lambda_function.py b/cumulus_lambda_functions/metadata_s4pa_generate_cmr/lambda_function.py
@@ -1,7 +1,7 @@
 import json
 
 from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator
-from cumulus_lambda_functions.snpp_lvl0_generate_cmr.generate_cmr import GenerateCmr
+from cumulus_lambda_functions.metadata_s4pa_generate_cmr.generate_cmr import GenerateCmr
 
 
 def lambda_handler(event, context):
diff --git a/cumulus_lambda_functions/metadata_s4pa_generate_cmr/pds_metadata.py b/cumulus_lambda_functions/metadata_s4pa_generate_cmr/pds_metadata.py
diff --git a/setup.py b/setup.py
@@ -17,7 +17,7 @@
 
 setup(
     name="cumulus_lambda_functions",
-    version="2.0.0",
+    version="3.0.0",
     packages=find_packages(),
     install_requires=install_requires,
     tests_require=['mock', 'nose', 'sphinx', 'sphinx_rtd_theme', 'coverage', 'pystac', 'python-dotenv', 'jsonschema'],
diff --git a/tests/cumulus_lambda_functions/snpp_level1a_generate_cmr/test_l1a_input_metadata.py b/tests/cumulus_lambda_functions/snpp_level1a_generate_cmr/test_l1a_input_metadata.py
@@ -1,7 +1,7 @@
 import unittest
 import xmltodict
 
-from cumulus_lambda_functions.snpp_level1a_generate_cmr.l1a_input_metadata import L1AInputMetadata
+from cumulus_lambda_functions.metadata_cas_generate_cmr.l1a_input_metadata import L1AInputMetadata
 
 
 class TestL1aInputMetadata(unittest.TestCase):
diff --git a/tests/cumulus_lambda_functions/snpp_lvl0_generate_cmr/test_echo_metadata.py b/tests/cumulus_lambda_functions/snpp_lvl0_generate_cmr/test_echo_metadata.py
@@ -2,8 +2,8 @@
 
 import xmltodict
 
-from cumulus_lambda_functions.snpp_lvl0_generate_cmr.echo_metadata import EchoMetadata
-from cumulus_lambda_functions.snpp_lvl0_generate_cmr.pds_metadata import PdsMetadata
+from cumulus_lambda_functions.metadata_s4pa_generate_cmr.echo_metadata import EchoMetadata
+from cumulus_lambda_functions.metadata_s4pa_generate_cmr.pds_metadata import PdsMetadata
 
 
 class TestEchoMetadata(unittest.TestCase):
diff --git a/tf-module/unity-cumulus/main.tf b/tf-module/unity-cumulus/main.tf
@@ -33,16 +33,17 @@ data "aws_iam_policy_document" "lambda_assume_role_policy" {
   }
 }
 
-resource "aws_lambda_function" "snpp_lvl0_generate_cmr" {
+resource "aws_lambda_function" "metadata_s4pa_generate_cmr" {
   filename      = local.lambda_file_name
-  function_name = "${var.prefix}-snpp_lvl0_generate_cmr"
+  function_name = "${var.prefix}-metadata_s4pa_generate_cmr"
   role          = var.lambda_processing_role_arn
-  handler       = "cumulus_lambda_functions.snpp_lvl0_generate_cmr.lambda_function.lambda_handler"
+  handler       = "cumulus_lambda_functions.metadata_s4pa_generate_cmr.lambda_function.lambda_handler"
   runtime       = "python3.9"
   timeout       = 300
   environment {
     variables = {
       LOG_LEVEL = var.log_level
+      FILE_POSTFIX = var.metadata_s4pa_file_postfix
     }
   }
 
@@ -53,11 +54,11 @@ resource "aws_lambda_function" "snpp_lvl0_generate_cmr" {
   tags = var.tags
 }
 
-resource "aws_lambda_function" "snpp_lvl1_generate_cmr" {
+resource "aws_lambda_function" "metadata_cas_generate_cmr" {
   filename      = local.lambda_file_name
-  function_name = "${var.prefix}-snpp_lvl1_generate_cmr"
+  function_name = "${var.prefix}-metadata_cas_generate_cmr"
   role          = var.lambda_processing_role_arn
-  handler       = "cumulus_lambda_functions.snpp_level1a_generate_cmr.lambda_function.lambda_handler"
+  handler       = "cumulus_lambda_functions.metadata_cas_generate_cmr.lambda_function.lambda_handler"
   runtime       = "python3.9"
   timeout       = 300
   environment {
diff --git a/tf-module/unity-cumulus/outputs.tf b/tf-module/unity-cumulus/outputs.tf
@@ -1,5 +1,5 @@
-output "snpp_lvl0_generate_cmr_arn" {
-  value = aws_lambda_function.snpp_lvl0_generate_cmr.arn
+output "metadata_s4pa_generate_cmr_arn" {
+  value = aws_lambda_function.metadata_s4pa_generate_cmr.arn
 }
 
 output "cumulus_granules_dapa_arn" {
@@ -10,8 +10,8 @@ output "cumulus_collections_dapa_arn" {
   value = aws_lambda_function.cumulus_collections_dapa.arn
 }
 
-output "snpp_lvl1_generate_cmr_arn" {
-    value = aws_lambda_function.snpp_lvl1_generate_cmr.arn
+output "metadata_cas_generate_cmr_arn" {
+    value = aws_lambda_function.metadata_cas_generate_cmr.arn
 }
 
 output "cumulus_collections_ingest_cnm_dapa_arn" {
diff --git a/tf-module/unity-cumulus/variables.tf b/tf-module/unity-cumulus/variables.tf
@@ -3,6 +3,10 @@ variable "log_level" {
   default = "20"
   description = "Lambda Log Level. Follow Python3 log level numbers info=20, warning=30, etc..."
 }
+variable "metadata_s4pa_file_postfix" {
+  type = string
+  description = "Comma separated File Postfix for PDS XML metadata files"
+}
 variable "prefix" {
   type = string
 }

Original file line number	Diff line number	Diff line change
`@@ -1,5 +1,5 @@`
`1`		`-output "snpp_lvl0_generate_cmr_arn" {`
`2`		`- value = aws_lambda_function.snpp_lvl0_generate_cmr.arn`
	`1`	`+output "metadata_s4pa_generate_cmr_arn" {`
	`2`	`+ value = aws_lambda_function.metadata_s4pa_generate_cmr.arn`
`3`	`3`	`}`
`4`	`4`
`5`	`5`	`output "cumulus_granules_dapa_arn" {`
`@@ -10,8 +10,8 @@ output "cumulus_collections_dapa_arn" {`
`10`	`10`	`value = aws_lambda_function.cumulus_collections_dapa.arn`
`11`	`11`	`}`
`12`	`12`
`13`		`-output "snpp_lvl1_generate_cmr_arn" {`
`14`		`- value = aws_lambda_function.snpp_lvl1_generate_cmr.arn`
	`13`	`+output "metadata_cas_generate_cmr_arn" {`
	`14`	`+ value = aws_lambda_function.metadata_cas_generate_cmr.arn`
`15`	`15`	`}`
`16`	`16`
`17`	`17`	`output "cumulus_collections_ingest_cnm_dapa_arn" {`
Original file line number	Diff line number	Diff line change
`@@ -3,6 +3,10 @@ variable "log_level" {`
`3`	`3`	`default = "20"`
`4`	`4`	`description = "Lambda Log Level. Follow Python3 log level numbers info=20, warning=30, etc..."`
`5`	`5`	`}`
	`6`	`+variable "metadata_s4pa_file_postfix" {`
	`7`	`+ type = string`
	`8`	`+ description = "Comma separated File Postfix for PDS XML metadata files"`
	`9`	`+}`
`6`	`10`	`variable "prefix" {`
`7`	`11`	`type = string`
`8`	`12`	`}`