Merge pull request #166 from unity-sds/develop

release/5.0.1

Author: ngachung

CHANGELOG.md

@@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [5.0.1] - 2023-06-21
+### Added
+- [#165](https://github.com/unity-sds/unity-data-services/pull/165) fix: convert all outputs into json str
+
 ## [5.0.0] - 2023-06-13
 ### Added
 - [#163](https://github.com/unity-sds/unity-data-services/pull/163) breaking: new upload implementation for complete catalog (no connection to DAPA)

cumulus_lambda_functions/stage_in_out/catalog_granules_unity.py

@@ -1,3 +1,5 @@
+import json
+
 from cumulus_lambda_functions.cumulus_dapa_client.dapa_client import DapaClient
 from cumulus_lambda_functions.stage_in_out.catalog_granules_abstract import CatalogGranulesAbstract
 import logging
@@ -36,4 +38,4 @@ def catalog(self, **kwargs):
         dapa_client = DapaClient().with_verify_ssl(self.__verify_ssl)
         LOGGER.debug(f'dapa_body_granules: {dapa_body}')
         dapa_ingest_result = dapa_client.ingest_granules_w_cnm(dapa_body)
-        return dapa_ingest_result
+        return json.dumps(dapa_ingest_result)

cumulus_lambda_functions/stage_in_out/download_granules_abstract.py

@@ -64,7 +64,7 @@ def _retrieve_stac_json(self):
         self._granules_json = ItemCollection.from_dict(json_stac)
         return self
 
-    def download(self, **kwargs) -> dict:
+    def download(self, **kwargs) -> str:
         self._set_props_from_env()
         LOGGER.debug(f'creating download dir: {self._download_dir}')
         if len(self._granules_json.items) < 1:
@@ -88,4 +88,4 @@ def download(self, **kwargs) -> dict:
         if len(error_list) > 0:
             with open(f'{self._download_dir}/error.log', 'w') as error_file:
                 error_file.write(json.dumps(error_list, indent=4))
-        return granules_json_dict
+        return json.dumps(granules_json_dict)

cumulus_lambda_functions/stage_in_out/upload_granules_by_complete_catalog_s3.py

@@ -42,7 +42,7 @@ def __set_props_from_env(self):
         self.__delete_files = os.environ.get(self.DELETE_FILES_KEY, 'FALSE').strip().upper() == 'TRUE'
         return self
 
-    def upload(self, **kwargs) -> dict:
+    def upload(self, **kwargs) -> str:
         self.__set_props_from_env()
         child_links = self.__gc.get_child_link_hrefs(os.environ.get(self.CATALOG_FILE))
         errors = []
@@ -81,4 +81,4 @@ def upload(self, **kwargs) -> dict:
             LOGGER.error(f'some errors while uploading granules: {errors}')
         LOGGER.debug(f'dapa_body_granules: {dapa_body_granules}')
         uploaded_item_collections = ItemCollection(items=dapa_body_granules)
-        return uploaded_item_collections.to_dict(False)
+        return json.dumps(uploaded_item_collections.to_dict(False))

setup.py

@@ -18,7 +18,7 @@
 
 setup(
     name="cumulus_lambda_functions",
-    version="5.0.0",
+    version="5.0.1",
     packages=find_packages(),
     install_requires=install_requires,
     tests_require=['mock', 'nose', 'sphinx', 'sphinx_rtd_theme', 'coverage', 'pystac', 'python-dotenv', 'jsonschema'],

tests/integration_tests/test_docker_entry.py

@@ -43,7 +43,7 @@ def test_01_search_part_01(self):
             # self.assertTrue(isinstance(search_result, list), f'search_result is not list: {search_result}')
             self.assertEqual(len(item_collections.items), 4000, f'wrong length')
             search_result = set([k.id for k in item_collections.items])
-            self.assertEqual(len(search_result),4000, f'wrong length. not unique')
+            self.assertEqual(len(search_result), 4000, f'wrong length. not unique')
             self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file')
             self.assertEqual(sorted(json.dumps(FileUtils.read_json(os.environ['OUTPUT_FILE']))), sorted(search_result_str), f'not identical result')
         return
@@ -226,7 +226,8 @@ def test_02_download(self):
         with tempfile.TemporaryDirectory() as tmp_dir_name:
             os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json')
             os.environ['DOWNLOAD_DIR'] = tmp_dir_name
-            download_result = choose_process()
+            download_result_str = choose_process()
+            download_result = json.loads(download_result_str)
             self.assertTrue('features' in download_result, f'missing features in download_result')
             self.assertEqual(len(download_result['features']) + 2, len(glob(os.path.join(tmp_dir_name, '*'))), f'downloaded file does not match')
             error_file = os.path.join(tmp_dir_name, 'error.log')
@@ -256,7 +257,8 @@ def test_02_download__daac(self):
         with tempfile.TemporaryDirectory() as tmp_dir_name:
             os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json')
             os.environ['DOWNLOAD_DIR'] = tmp_dir_name
-            download_result = choose_process()
+            download_result_str = choose_process()
+            download_result = json.loads(download_result_str)
             self.assertTrue('features' in download_result, f'missing features in download_result')
             self.assertEqual(len(download_result['features']) + 2, len(glob(os.path.join(tmp_dir_name, '*'))), f'downloaded file does not match')
             error_file = os.path.join(tmp_dir_name, 'error.log')
@@ -291,7 +293,8 @@ def test_02_download__daac__from_file(self):
             FileUtils.write_json(granule_json_file, granule_json)
             os.environ['STAC_JSON'] = granule_json_file
             os.environ['DOWNLOAD_DIR'] = downloading_dir
-            download_result = choose_process()
+            download_result_str = choose_process()
+            download_result = json.loads(download_result_str)
             self.assertTrue('features' in download_result, f'missing features in download_result')
             self.assertEqual(len(download_result['features']) + 1, len(glob(os.path.join(downloading_dir, '*'))), f'downloaded file does not match')
             error_file = os.path.join(downloading_dir, 'error.log')
@@ -320,7 +323,8 @@ def test_02_download__daac_error(self):  # TODO update this later
             os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json')
             # TODO this is downloading a login page HTML
             os.environ['DOWNLOAD_DIR'] = tmp_dir_name
-            download_result = choose_process()
+            download_result_str = choose_process()
+            download_result = json.loads(download_result_str)
             self.assertTrue(isinstance(download_result, list), f'download_result is not list: {download_result}')
             self.assertEqual(sum([len(k) for k in download_result]), len(glob(os.path.join(tmp_dir_name, '*'))), f'downloaded file does not match')
             error_file = os.path.join(tmp_dir_name, 'error.log')
@@ -344,7 +348,8 @@ def test_02_download__from_file(self):
             FileUtils.write_json(granule_json_file, granule_json)
             os.environ['STAC_JSON'] = granule_json_file
             os.environ['DOWNLOAD_DIR'] = downloading_dir
-            download_result = choose_process()
+            download_result_str = choose_process()
+            download_result = json.loads(download_result_str)
             self.assertTrue('features' in download_result, f'missing features in download_result')
             self.assertEqual(len(download_result['features']) + 1, len(glob(os.path.join(downloading_dir, '*'))), f'downloaded file does not match')
             error_file = os.path.join(downloading_dir, 'error.log')
@@ -373,7 +378,8 @@ def test_02_download__from_http(self):
             FileUtils.write_json(granule_json_file, granule_json)
             os.environ['STAC_JSON'] = granule_json_file
             os.environ['DOWNLOAD_DIR'] = downloading_dir
-            download_result = choose_process()
+            download_result_str = choose_process()
+            download_result = json.loads(download_result_str)
             self.assertTrue('features' in download_result, f'missing features in download_result')
             self.assertEqual(len(download_result['features']) + 1, len(glob(os.path.join(downloading_dir, '*'))), f'downloaded file does not match')
             error_file = os.path.join(tmp_dir_name, 'error.log')
@@ -836,7 +842,8 @@ def test_03_upload_complete_catalog(self):
             with open(os.environ['CATALOG_FILE'], 'w') as ff:
                 ff.write(json.dumps(catalog.to_dict(False, False)))
 
-            upload_result = choose_process()
+            upload_result_str = choose_process()
+            upload_result = json.loads(upload_result_str)
             print(upload_result)
             self.assertTrue('features' in upload_result, 'missing features')
             self.assertEqual(1, len(upload_result['features']), 'wrong length of upload_result features')
@@ -892,7 +899,8 @@ def test_04_catalog(self):
         argv.append('CATALOG')
         with tempfile.TemporaryDirectory() as tmp_dir_name:
             os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json')
-            catalog_result = choose_process()
+            catalog_result_str = choose_process()
+            catalog_result = json.loads(catalog_result_str)
             self.assertEqual('registered', catalog_result, 'wrong status')
             self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file')
         return
@@ -916,7 +924,8 @@ def test_04_catalog_from_file(self):
             FileUtils.write_json(input_file_path, upload_result)
             os.environ['UPLOADED_FILES_JSON'] = input_file_path
             os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json')
-            catalog_result = choose_process()
+            catalog_result_str = choose_process()
+            catalog_result = json.loads(catalog_result_str)
             self.assertEqual('registered', catalog_result, 'wrong status')
             self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file')
         return
@@ -940,7 +949,8 @@ def test_04_catalog_from_file_item_collection(self):
             FileUtils.write_json(input_file_path, upload_result)
             os.environ['UPLOADED_FILES_JSON'] = input_file_path
             os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json')
-            catalog_result = choose_process()
+            catalog_result_str = choose_process()
+            catalog_result = json.loads(catalog_result_str)
             self.assertEqual('registered', catalog_result, 'wrong status')
             self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file')
         return