fix: S3 upload structure (add collection folder) (#193)

* fix: retry 5 times + wait 45 sec for DAAC

* feat: make re-try configurable

* feat: adding fastapi (in-progress)

* fix: get collection creation working (in-progress)

* feat: add collection folder in s3 upload

* fix: update note

* fix: update wrong PR number

* fix: update missed file during merge

* fix: remove old test case

Author: wphyojpl

CHANGELOG.md

@@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [5.2.3] - 2023-08-07
+### Changed
+- [#193](https://github.com/unity-sds/unity-data-services/pull/193) fix: add collection folder in s3 upload
+
 ## [5.2.2] - 2023-07-21
 ### Changed
 - [#188](https://github.com/unity-sds/unity-data-services/pull/188) fix: Update stage out task to read STAC items from STAC catalog

cumulus_lambda_functions/stage_in_out/catalog_granules_unity.py

@@ -32,7 +32,7 @@ def __set_props_from_env(self):
         self.__provider_id = os.environ.get(self.PROVIDER_ID_KEY)
         self.__verify_ssl = os.environ.get(self.VERIFY_SSL_KEY, 'TRUE').strip().upper() == 'TRUE'
         self.__delaying_second = int(os.environ.get(self.DELAY_SECOND, '30'))
-        self.__repeating_times = int(os.environ.get(self.REPEAT_TIMES, '30'))
+        self.__repeating_times = int(os.environ.get(self.REPEAT_TIMES, '5'))
         self.__verify_ssl = os.environ.get(self.VERIFY_SSL_KEY, 'TRUE').strip().upper() == 'TRUE'
         return self
 

cumulus_lambda_functions/stage_in_out/upload_granules_by_complete_catalog_s3.py

@@ -62,7 +62,7 @@ def upload(self, **kwargs) -> str:
                 uploading_current_granule_stac = None
                 for asset_type, asset_href in current_assets.items():
                     LOGGER.debug(f'uploading {asset_type}, {asset_href}')
-                    s3_url = self.__s3.upload(asset_href, self.__staging_bucket, f'{self.__collection_id}:{current_granule_id}', self.__delete_files)
+                    s3_url = self.__s3.upload(asset_href, self.__staging_bucket, f'{self.__collection_id}/{self.__collection_id}:{current_granule_id}', self.__delete_files)
                     if asset_href == each_child:
                         uploading_current_granule_stac = s3_url
                     updating_assets[asset_type] = s3_url

cumulus_lambda_functions/uds_api/dapa/collections_dapa_creation.py

@@ -142,4 +142,4 @@ def start(self, current_url: URL):
             'body': json.dumps({
                 'message': 'processing'
             })
-        }
+        }

cumulus_lambda_functions/uds_api/web_service.py

@@ -31,4 +31,4 @@ async def root(request: Request):
 
 if __name__ == '__main__':
     uvicorn.run("web_service:app", port=8005, log_level="info", reload=True)
-    print("running")
+    print("running")

setup.py

@@ -22,7 +22,7 @@
 
 setup(
     name="cumulus_lambda_functions",
-    version="5.3.0",
+    version="5.2.3",
     packages=find_packages(),
     install_requires=install_requires,
     tests_require=['mock', 'nose', 'sphinx', 'sphinx_rtd_theme', 'coverage', 'pystac', 'python-dotenv', 'jsonschema'],

tests/integration_tests/test_docker_entry.py

@@ -1790,10 +1790,10 @@ def test_03_upload_complete_catalog(self):
             self.assertTrue('metadata__cas' in upload_result['assets'], 'missing assets#metadata__cas')
             self.assertTrue('href' in upload_result['assets']['metadata__cas'], 'missing assets#metadata__cas#href')
             self.assertTrue(
-                upload_result['assets']['metadata__cas']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/'))
+                upload_result['assets']['metadata__cas']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/{os.environ["COLLECTION_ID"]}/'))
             self.assertTrue('data' in upload_result['assets'], 'missing assets#data')
             self.assertTrue('href' in upload_result['assets']['data'], 'missing assets#data#href')
-            self.assertTrue(upload_result['assets']['data']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/'))
+            self.assertTrue(upload_result['assets']['data']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/{os.environ["COLLECTION_ID"]}/'))
             self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file')
             """
             Example output: 
@@ -1808,11 +1808,11 @@ def test_03_upload_complete_catalog(self):
                                 'updated': '2022-03-23T15:48:21.578000Z', 'datetime': '1970-01-01T00:00:00Z'},
                     'geometry': {'type': 'Point', 'coordinates': [0.0, 0.0]}, 'links': [], 
                     'assets': {'data': {
-                        'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc',
+                        'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc',
                         'title': 'main data'}, 'metadata__cas': {
-                        'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.cas',
+                        'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.cas',
                         'title': 'metadata cas'}, 'metadata__stac': {
-                        'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.stac.json',
+                        'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.stac.json',
                         'title': 'metadata stac'}}, 
                     'bbox': [0.0, 0.0, 0.0, 0.0], 
                     'stac_extensions': [],
@@ -1887,11 +1887,11 @@ def test_04_catalog_from_file_item_collection(self):
                             'created': '2016-02-01T02:45:59.639000Z', 'updated': '2022-03-23T15:48:21.578000Z',
                             'datetime': '1970-01-01T00:00:00Z'}, 'geometry': {'type': 'Point', 'coordinates': [0.0, 0.0]},
              'links': [], 'assets': {
-                'data': {'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc',
+                'data': {'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc',
                          'title': 'main data'}, 'metadata__cas': {
-                    'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.cas',
+                    'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.cas',
                     'title': 'metadata cas'}, 'metadata__stac': {
-                    'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.stac.json',
+                    'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.stac.json',
                     'title': 'metadata stac'}}, 'bbox': [0.0, 0.0, 0.0, 0.0], 'stac_extensions': [],
              'collection': 'NEW_COLLECTION_EXAMPLE_L1B___9'}]}
         os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username'
@@ -1903,6 +1903,8 @@ def test_04_catalog_from_file_item_collection(self):
         os.environ['VERIFY_SSL'] = 'FALSE'
         os.environ['PROVIDER_ID'] = 'SNPP'
         os.environ['GRANULES_CATALOG_TYPE'] = 'UNITY'
+        os.environ['DELAY_SECOND'] = '35'
+        os.environ['REPEAT_TIMES'] = '3'
         if len(argv) > 1:
             argv.pop(-1)
         argv.append('CATALOG')