feat: Custom metadata mechanism

.gitignore

@@ -7,4 +7,6 @@ local*
 *egg-info*
 dist
 __pycache__
-.env
+.env
+.terraform
+

CHANGELOG.md

@@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [5.5.0] - 2023-09-28
+### Added
+- [#201](https://github.com/unity-sds/unity-data-services/pull/201) feat: Custom metadata mechanism
+
+## [5.4.0] - 2023-08-16
+### Added
+- [#95](https://github.com/unity-sds/unity-data-services/pull/95) feat: elasticsearch branch
+
 ## [5.3.3] - 2023-09-18
 ### Changed
 - [#204](https://github.com/unity-sds/unity-data-services/pull/204) chore: clean.up - remove old codes

ci.cd/Makefile

@@ -26,8 +26,26 @@ update_lambda_function_mcp_dev_6:
 	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-metadata_s4pa_generate_cmr --publish &>/dev/null
 update_lambda_function_mcp_dev_7:
 	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-metadata_stac_generate_cmr --publish &>/dev/null
+update_lambda_function_mcp_dev_auth_list:
+	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-cumulus_auth_list --publish &>/dev/null
+update_lambda_function_mcp_dev_auth_add:
+	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-cumulus_auth_add --publish &>/dev/null
+update_lambda_function_mcp_dev_auth_delete:
+	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-cumulus_auth_delete --publish &>/dev/null
 
 update_lambda_function_mcp_dev_uds_api:
 	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-uds_api_1 --publish &>/dev/null
 
+update_lambda_function_mcp_sbx_uds_api:
+	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-sbx-cumulus-uds_api_1 --publish &>/dev/null
+
+mcp_dev: upload_lambda_mcp_dev update_lambda_function_mcp_dev_auth_list update_lambda_function_mcp_dev_auth_add update_lambda_function_mcp_dev_auth_delete
+
 mcp_dev_fastapi: upload_lambda_mcp_dev update_lambda_function_mcp_dev_uds_api
+
+update_lambda_function_mcp_sbx_7:
+	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-sbx-cumulus-metadata_stac_generate_cmr --publish &>/dev/null
+
+mcp_sbx: upload_lambda_mcp_dev update_lambda_function_mcp_sbx_7
+
+mcp_sbx_fastapi: upload_lambda_mcp_dev update_lambda_function_mcp_sbx_uds_api

cumulus_lambda_functions/cumulus_auth_crud/auth_crud.py

@@ -0,0 +1,181 @@
+import json
+import os
+
+from cumulus_lambda_functions.lib.authorization.uds_authorizer_abstract import UDSAuthorizorAbstract
+from cumulus_lambda_functions.lib.authorization.uds_authorizer_factory import UDSAuthorizerFactory
+from cumulus_lambda_functions.lib.json_validator import JsonValidator
+
+from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator
+from cumulus_lambda_functions.lib.uds_db.db_constants import DBConstants
+from cumulus_lambda_functions.lib.utils.lambda_api_gateway_utils import LambdaApiGatewayUtils
+
+LOGGER = LambdaLoggerGenerator.get_logger(__name__, LambdaLoggerGenerator.get_level_from_env())
+
+
+delete_schema = {
+    'type': 'object',
+    'required': ['tenant', 'venue', 'group_name'],
+    'properties': {
+        'tenant': {'type': 'string'},
+        'venue': {'type': 'string'},
+        'group_name': {'type': 'string'},
+    }
+}
+
+list_schema = {
+    'type': 'object',
+    'properties': {
+        'tenant': {'type': 'string'},
+        'venue': {'type': 'string'},
+        'group_names': {
+            'type': 'array',
+            'items': {'type': 'string'},
+            'minItems': 1,
+        },
+    }
+}
+
+add_schema = {
+    'type': 'object',
+    'required': ['tenant', 'venue', 'group_name', 'actions', 'resources'],
+    'properties': {
+        'tenant': {'type': 'string'},
+        'venue': {'type': 'string'},
+        'group_name': {'type': 'string'},
+        'actions': {
+            'type': 'array',
+            'items': {
+                'type': 'string',
+                'enum': [DBConstants.create, DBConstants.delete, DBConstants.update, DBConstants.read]
+            },
+            'minItems': 1,
+        },
+        'resources': {
+            'type': 'array',
+            'items': {'type': 'string'},
+            'minItems': 1,
+        },
+    }
+
+}
+
+
+class AuthCrud:
+    def __init__(self, event):
+        required_env = ['ES_URL', 'ADMIN_COMMA_SEP_GROUPS']
+        if not all([k in os.environ for k in required_env]):
+            raise EnvironmentError(f'one or more missing env: {required_env}')
+        self.__admin_groups = [k.strip() for k in os.getenv('ADMIN_COMMA_SEP_GROUPS').split(',')]
+        self.__event = event
+        self.__request_body = {}
+        self.__es_url = os.getenv('ES_URL')
+        self.__es_port = int(os.getenv('ES_PORT', '443'))
+        self.__authorizer: UDSAuthorizorAbstract = UDSAuthorizerFactory() \
+            .get_instance(UDSAuthorizerFactory.cognito,
+                          es_url=self.__es_url,
+                          es_port=self.__es_port
+                          )
+        self.__lambda_utils = LambdaApiGatewayUtils(self.__event, 10)
+
+    def __is_admin(self):
+        auth_info = self.__lambda_utils.get_authorization_info()
+        belonged_admin_groups = list(set(self.__admin_groups) & set(auth_info['ldap_groups']))
+        return len(belonged_admin_groups) > 0
+
+    def __load_request_body(self):
+        if 'body' in self.__event and self.__event['body'] is not None:
+            self.__request_body = json.loads(self.__event['body'])
+        return
+
+    def list_all_record(self):
+        if not self.__is_admin():
+            return {
+                'statusCode': 403,
+                'body': f'user is not in admin groups: {self.__admin_groups}'
+            }
+        self.__load_request_body()
+        all_records = self.__authorizer.list_groups(
+            tenant=self.__request_body['tenant'] if 'tenant' in self.__request_body else None,
+            venue=self.__request_body['venue'] if 'venue' in self.__request_body else None,
+            ldap_group_names=self.__request_body['group_names'] if 'group_names' in self.__request_body else None,
+        )
+        return {
+                'statusCode': 200,
+                'body': json.dumps(all_records)
+            }
+
+    def add_new_record(self):
+        if not self.__is_admin():
+            return {
+                'statusCode': 403,
+                'body': f'user is not in admin groups: {self.__admin_groups}'
+            }
+        self.__load_request_body()
+        body_validator_result = JsonValidator(add_schema).validate(self.__request_body)
+        if body_validator_result is not None:
+            LOGGER.error(f'invalid add body: {body_validator_result}. request_body: {self.__request_body}')
+            return {
+                'statusCode': 500,
+                'body': f'invalid add body: {body_validator_result}. request_body: {self.__request_body}'
+            }
+        self.__authorizer.add_authorized_group(
+            action=self.__request_body['actions'],
+            resource=self.__request_body['resources'],
+            tenant=self.__request_body['tenant'],
+            venue=self.__request_body['venue'],
+            ldap_group_name=self.__request_body['group_name'],
+        )
+        return {
+            'statusCode': 200,
+            'body': 'inserted'
+        }
+
+    def update_record(self):
+        if not self.__is_admin():
+            return {
+                'statusCode': 403,
+                'body': f'user is not in admin groups: {self.__admin_groups}'
+            }
+        self.__load_request_body()
+        body_validator_result = JsonValidator(add_schema).validate(self.__request_body)
+        if body_validator_result is not None:
+            LOGGER.error(f'invalid update body: {body_validator_result}. request_body: {self.__request_body}')
+            return {
+                'statusCode': 500,
+                'body': f'invalid update body: {body_validator_result}. request_body: {self.__request_body}'
+            }
+        self.__authorizer.update_authorized_group(
+            action=self.__request_body['actions'],
+            resource=self.__request_body['resources'],
+            tenant=self.__request_body['tenant'],
+            venue=self.__request_body['venue'],
+            ldap_group_name=self.__request_body['group_name'],
+        )
+        return {
+            'statusCode': 200,
+            'body': 'updated'
+        }
+
+    def delete_record(self):
+        if not self.__is_admin():
+            return {
+                'statusCode': 403,
+                'body': f'user is not in admin groups: {self.__admin_groups}'
+            }
+        self.__load_request_body()
+        body_validator_result = JsonValidator(delete_schema).validate(self.__request_body)
+        if body_validator_result is not None:
+            LOGGER.error(f'invalid delete body: {body_validator_result}. request_body: {self.__request_body}')
+            return {
+                'statusCode': 500,
+                'body': f'invalid delete body: {body_validator_result}. request_body: {self.__request_body}'
+            }
+        self.__authorizer.delete_authorized_group(
+            tenant=self.__request_body.get('tenant'),
+            venue=self.__request_body.get('venue'),
+            ldap_group_name=self.__request_body.get('group_name'),
+        )
+        return {
+            'statusCode': 200,
+            'body': 'deleted'
+        }

cumulus_lambda_functions/cumulus_auth_crud/lambda_function.py

@@ -0,0 +1,22 @@
+from cumulus_lambda_functions.cumulus_auth_crud.auth_crud import AuthCrud
+from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator
+
+
+def auth_list(event, context):
+    LambdaLoggerGenerator.remove_default_handlers()
+    return AuthCrud(event).list_all_record()
+
+
+def auth_add(event, context):
+    LambdaLoggerGenerator.remove_default_handlers()
+    return AuthCrud(event).add_new_record()
+
+
+def auth_update(event, context):
+    LambdaLoggerGenerator.remove_default_handlers()
+    return AuthCrud(event).update_record()
+
+
+def auth_delete(event, context):
+    LambdaLoggerGenerator.remove_default_handlers()
+    return AuthCrud(event).delete_record()

cumulus_lambda_functions/cumulus_es_setup/es_mappings.py

@@ -0,0 +1,50 @@
+alias_pointer = {
+    "actions" : [
+        {"add" : {"index" : "authorization_mappings_v1", "alias" : "authorization_mappings"}},
+        {"add" : {"index" : "unity_collections_v1", "alias" : "unity_collections"}},
+        {"add": {"index": "unity_granules_v1", "alias": "unity_granules_read"}},
+        {"add": {"index": "unity_granules_v1", "alias": "unity_granules_write"}}
+    ]
+}
+unity_granules_v1 = {
+    "settings": {
+        "number_of_shards": 3,
+        "number_of_replicas": 2
+    },
+    "mappings": {
+        "properties": {
+            "granule_id": {"type": "keyword"}
+        }
+    }
+}
+authorization_mappings_v1 = {
+  "settings" : {
+        "number_of_shards" : 3,
+        "number_of_replicas" : 2
+    },
+  "mappings": {
+    "properties": {
+      "action": {"type": "keyword"},
+      "collection_map": {"type": "keyword"},
+      "user_group": {"type": "keyword"},
+      "tenant": {"type": "keyword"},
+      "tenant_venue": {"type": "keyword"}
+    }
+  }
+}
+
+unity_collections_v1 = {
+  "settings" : {
+        "number_of_shards" : 3,
+        "number_of_replicas" : 2
+    },
+  "mappings": {
+    "properties": {
+      "collection_id": {"type": "keyword"},
+      "bbox": {"type": "geo_shape"},
+      "granule_count": {"type":  "integer"},
+      "start_time": {"type":  "long"},
+      "end_time": {"type":  "long"}
+    }
+  }
+}

cumulus_lambda_functions/cumulus_es_setup/es_setup.py

@@ -0,0 +1,46 @@
+import os
+
+from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator
+
+from cumulus_lambda_functions.lib.aws.es_abstract import ESAbstract
+
+from cumulus_lambda_functions.lib.aws.es_factory import ESFactory
+
+from cumulus_lambda_functions.lib.uds_db.db_constants import DBConstants
+from cumulus_lambda_functions.cumulus_es_setup import es_mappings
+LOGGER = LambdaLoggerGenerator.get_logger(__name__, LambdaLoggerGenerator.get_level_from_env())
+
+
+class SetupESIndexAlias:
+    def __init__(self):
+        required_env = ['ES_URL']
+        if not all([k in os.environ for k in required_env]):
+            raise EnvironmentError(f'one or more missing env: {required_env}')
+        self.__es: ESAbstract = ESFactory().get_instance('AWS',
+                                                         index=DBConstants.collections_index,
+                                                         base_url=os.getenv('ES_URL'),
+                                                         port=int(os.getenv('ES_PORT', '443'))
+                                                         )
+
+    def get_index_mapping(self, index_name: str):
+        if not hasattr(es_mappings, index_name):
+            raise ValueError(f'missing index_name: {index_name}')
+        index_json = getattr(es_mappings, index_name)
+        return index_json
+
+    def start(self):
+        if not hasattr(es_mappings, 'alias_pointer'):
+            raise ValueError(f'missing alias_pointer')
+        alias_json = getattr(es_mappings, 'alias_pointer')
+        alias_json = [k['add'] for k in alias_json['actions']]
+        for each_action in alias_json:
+            try:
+                current_index = each_action['index']
+                current_alias = each_action['alias']
+                LOGGER.debug(f'working on {current_index}')
+                index_json = self.get_index_mapping(current_index)
+                self.__es.create_index(current_index, index_json)
+                self.__es.create_alias(current_index, current_alias)
+            except:
+                LOGGER.exception(f'failed to create index / alias for: {each_action}')
+        return self