release/5.3.1

CHANGELOG.md

@@ -5,6 +5,18 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [5.3.1] - 2023-08-16
+### Changed
+- [#194](https://github.com/unity-sds/unity-data-services/pull/194) fix: Cataloging large number asynchronously by batch + download is stuck when there are large number of files
+
+## [5.3.0] - 2023-08-07
+### Changed
+- [#190](https://github.com/unity-sds/unity-data-services/pull/190) feat: Using Fastapi for all API endpoints
+
+## [5.2.3] - 2023-08-07
+### Changed
+- [#193](https://github.com/unity-sds/unity-data-services/pull/193) fix: add collection folder in s3 upload
+
 ## [5.2.2] - 2023-07-21
 ### Changed
 - [#188](https://github.com/unity-sds/unity-data-services/pull/188) fix: Update stage out task to read STAC items from STAC catalog

README.md

@@ -109,7 +109,7 @@ Questions about our project? Please see our: [FAQ]([INSERT LINK TO FAQ / DISCUSS
 
 <!-- example FAQ inline format>
 1. Question 1
-   - Answer to question 1
+   - Answer to question 1.
 2. Question 2
    - Answer to question 2
 -->

ci.cd/Makefile

@@ -6,7 +6,7 @@ export VERSION ?= latest
 all: build_lambda upload_lambda update_lambda_function build_docker
 local: build_lambda upload_lambda update_lambda_function_1 update_lambda_function_2 update_lambda_function_3
 build_docker:
-	docker build -t "$(IMAGE_PREFIX)/$(NAME):$(VERSION)" -f docker/Dockerfile .
+	docker build --no-cache -t "$(IMAGE_PREFIX)/$(NAME):$(VERSION)" -f docker/Dockerfile_download_granules.jpl .
 
 zip_docker:
 	docker save "$(IMAGE_PREFIX)/$(NAME):$(VERSION)" | gzip > "$(NAME)__$(VERSION).tar.gz"
@@ -46,6 +46,10 @@ update_lambda_function_mcp_dev_6:
 update_lambda_function_mcp_dev_7:
 	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-metadata_stac_generate_cmr --publish &>/dev/null
 
+update_lambda_function_mcp_dev_uds_api:
+	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-uds_api_1 --publish &>/dev/null
+
+mcp_dev_fastapi: upload_lambda_mcp_dev update_lambda_function_mcp_dev_uds_api
 mcp_dev: upload_lambda_mcp_dev update_lambda_function_mcp_dev_1 update_lambda_function_mcp_dev_2 update_lambda_function_mcp_dev_4 update_lambda_function_mcp_dev_5
 
 

cumulus_lambda_functions/cumulus_collections_dapa/cumulus_create_collection_dapa.py

@@ -13,6 +13,7 @@
 
 class CumulusCreateCollectionDapa:
     def __init__(self, event):
+        print(event)
         required_env = ['CUMULUS_LAMBDA_PREFIX', 'CUMULUS_WORKFLOW_SQS_URL']
         if not all([k in os.environ for k in required_env]):
             raise EnvironmentError(f'one or more missing env: {required_env}')

cumulus_lambda_functions/cumulus_dapa_client/dapa_client.py

@@ -6,6 +6,7 @@
 
 from cumulus_lambda_functions.lib.cognito_login.cognito_token_retriever import CognitoTokenRetriever
 from cumulus_lambda_functions.lib.constants import Constants
+from cumulus_lambda_functions.uds_api.web_service_constants import WebServiceConstants
 
 LOGGER = logging.getLogger(__name__)
 
@@ -15,8 +16,9 @@ def __init__(self):
         self.__token_retriever = CognitoTokenRetriever()
         self.__token = None
         self.__dapa_base_api = None
-        self.__get_dapa_base_api()
         self.__verify_ssl = True
+        self.__api_base_prefix = WebServiceConstants.API_PREFIX
+        self.__get_dapa_base_api()
 
     def with_verify_ssl(self, verify_ssl: bool):
         self.__verify_ssl = verify_ssl
@@ -27,6 +29,7 @@ def __get_dapa_base_api(self):
             raise ValueError(f'missing key: {Constants.DAPA_API_KEY}')
         self.__dapa_base_api = os.environ.get(Constants.DAPA_API_KEY)
         self.__dapa_base_api = self.__dapa_base_api[:-1] if self.__dapa_base_api.endswith('/') else self.__dapa_base_api
+        self.__api_base_prefix = os.environ.get(Constants.DAPA_API_PREIFX_KEY) if Constants.DAPA_API_PREIFX_KEY in os.environ else self.__api_base_prefix
         return self
 
     def __get_token(self):
@@ -46,7 +49,7 @@ def get_collection(self, collection_id: str):
         LOGGER.debug(f'getting collection details for: {collection_id}')
         self.__get_token()
         header = {'Authorization': f'Bearer {self.__token}'}
-        dapa_collection_url = f'{self.__dapa_base_api}/am-uds-dapa/collections?limit=1000'
+        dapa_collection_url = f'{self.__dapa_base_api}/{self.__api_base_prefix}/collections?limit=1000'
         response = requests.get(url=dapa_collection_url, headers=header, verify=self.__verify_ssl)
         if response.status_code > 400:
             raise RuntimeError(
@@ -92,7 +95,7 @@ def get_granules(self, collection_id='*', limit=1000, offset=0, date_from='', da
         :param date_to:
         :return:
         """
-        dapa_granules_api = f'{self.__dapa_base_api}/am-uds-dapa/collections/{collection_id}/items?limit={limit}&offset={offset}'
+        dapa_granules_api = f'{self.__dapa_base_api}/{self.__api_base_prefix}/collections/{collection_id}/items?limit={limit}&offset={offset}'
         if date_from != '' or date_to != '':
             dapa_granules_api = f"{dapa_granules_api}&datetime={date_from if date_from != '' else '..'}/{date_to if date_to != '' else '..'}"
         if filter is not None:
@@ -111,7 +114,7 @@ def get_granules(self, collection_id='*', limit=1000, offset=0, date_from='', da
         return granules_result
 
     def ingest_granules_w_cnm(self, cnm_ingest_body: dict) -> str:
-        dapa_ingest_cnm_api = f'{self.__dapa_base_api}/am-uds-dapa/collections/'
+        dapa_ingest_cnm_api = f'{self.__dapa_base_api}/{self.__api_base_prefix}/collections/'
         LOGGER.debug(f'getting granules for: {dapa_ingest_cnm_api}')
         self.__get_token()
         header = {
@@ -123,5 +126,5 @@ def ingest_granules_w_cnm(self, cnm_ingest_body: dict) -> str:
         if response.status_code > 400:
             raise RuntimeError(
                 f'querying granules ingestion ends in error. status_code: {response.status_code}. url: {dapa_ingest_cnm_api}. details: {response.text}')
-        granules_result = response.text
+        granules_result = json.loads(response.text)
         return granules_result

cumulus_lambda_functions/cumulus_stac/unity_collection_stac.py

@@ -70,6 +70,4 @@ def start(self):
                                      )
         stac_collection.add_links(self.__files)
         new_collection = stac_collection.to_dict(include_self_link=False)
-        if 'links' in new_collection and len(new_collection['links']) > 0 and new_collection['links'][0]['rel'] == 'root':
-            new_collection['links'][0]['href'] = './collection.json'
         return new_collection

cumulus_lambda_functions/lib/constants.py

@@ -17,3 +17,4 @@ class Constants:
     BASE64_STR = 'BASE64'
     PARAM_STORE = 'PARAM_STORE'
     DAPA_API_KEY = 'DAPA_API'
+    DAPA_API_PREIFX_KEY = 'DAPA_API_PREIFX_KEY'