Merge pull request #120 from unity-sds/split-stage-in-out

breaking: split stage in stage out

Author: ngachung

.gitignore

@@ -1,5 +1,6 @@
 .DS_Store
 .idea
+.eggs
 build
 scratch*
 local*

CHANGELOG.md

@@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [2.0.0] - 2023-01-23
+### Breaking
+- [#120](https://github.com/unity-sds/unity-data-services/pull/120) breakup upload and download dockers into search + download & upload + catalog
+
 ## [1.10.1] - 2023-01-23
 ### Fixed
 - [#112](https://github.com/unity-sds/unity-data-services/pull/112) update dockerfile base images

cumulus_lambda_functions/cumulus_download_granules/__main__.py

@@ -2,23 +2,29 @@
 import os
 from sys import argv
 
-from cumulus_lambda_functions.cumulus_download_granules.download_granules import DownloadGranules
-from cumulus_lambda_functions.cumulus_upload_granules.upload_granules import UploadGranules
+from cumulus_lambda_functions.stage_in_out.catalog_granules_factory import CatalogGranulesFactory
+from cumulus_lambda_functions.stage_in_out.download_granules_s3 import DownloadGranulesS3
+from cumulus_lambda_functions.stage_in_out.search_granules_factory import SearchGranulesFactory
+from cumulus_lambda_functions.stage_in_out.upoad_granules_factory import UploadGranulesFactory
 
 
 def choose_process():
+    if argv[1].strip().upper() == 'SEARCH':
+        logging.info('starting SEARCH script')
+        return SearchGranulesFactory().get_class(os.getenv('GRANULES_SEARCH_DOMAIN', 'MISSING_GRANULES_SEARCH_DOMAIN')).search()
     if argv[1].strip().upper() == 'DOWNLOAD':
         logging.info('starting DOWNLOAD script')
-        DownloadGranules().start()
-    elif argv[1].strip().upper() == 'UPLOAD':
+        return DownloadGranulesS3().download()
+    if argv[1].strip().upper() == 'UPLOAD':
         logging.info('starting UPLOAD script')
-        logging.info(UploadGranules().start())
-    else:
-        raise ValueError(f'invalid argument: {argv}')
-    return
+        return UploadGranulesFactory().get_class(os.getenv('GRANULES_UPLOAD_TYPE', 'MISSING_GRANULES_UPLOAD_TYPE')).upload()
+    if argv[1].strip().upper() == 'CATALOG':
+        logging.info('starting CATALOG script')
+        return CatalogGranulesFactory().get_class(os.getenv('GRANULES_CATALOG_TYPE', 'MISSING_GRANULES_CATALOG_TYPE')).catalog()
+    raise ValueError(f'invalid argument: {argv}')
 
 
 if __name__ == '__main__':
     logging.basicConfig(level=int(os.environ.get('LOG_LEVEL', '10')),
                         format="%(asctime)s [%(levelname)s] [%(name)s::%(lineno)d] %(message)s")
-    choose_process()
+    print(choose_process())

cumulus_lambda_functions/cumulus_upload_granules/__init__.py

@@ -0,0 +1,7 @@
+from abc import ABC, abstractmethod
+
+
+class CatalogGranulesAbstract(ABC):
+    @abstractmethod
+    def catalog(self, **kwargs):
+        raise NotImplementedError()

cumulus_lambda_functions/docker_entrypoint/__main__.py

@@ -0,0 +1,8 @@
+class CatalogGranulesFactory:
+    UNITY = 'UNITY'
+
+    def get_class(self, upload_type):
+        if upload_type == CatalogGranulesFactory.UNITY:
+            from cumulus_lambda_functions.stage_in_out.catalog_granules_unity import CatalogGranulesUnity
+            return CatalogGranulesUnity()
+        raise ValueError(f'unknown search_type: {upload_type}')

cumulus_lambda_functions/cumulus_download_granules/__init__.py renamed to cumulus_lambda_functions/stage_in_out/__init__.py

@@ -0,0 +1,39 @@
+from cumulus_lambda_functions.cumulus_dapa_client.dapa_client import DapaClient
+from cumulus_lambda_functions.stage_in_out.catalog_granules_abstract import CatalogGranulesAbstract
+import json
+import logging
+import os
+
+LOGGER = logging.getLogger(__name__)
+
+
+class CatalogGranulesUnity(CatalogGranulesAbstract):
+    PROVIDER_ID_KEY = 'PROVIDER_ID'
+    UPLOADED_FILES_JSON = 'UPLOADED_FILES_JSON'
+    VERIFY_SSL_KEY = 'VERIFY_SSL'
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.__provider_id = ''
+        self.__uploaded_files_json = None
+        self.__verify_ssl = True
+
+    def __set_props_from_env(self):
+        missing_keys = [k for k in [self.UPLOADED_FILES_JSON, self.PROVIDER_ID_KEY] if k not in os.environ]
+        if len(missing_keys) > 0:
+            raise ValueError(f'missing environment keys: {missing_keys}')
+        self.__provider_id = os.environ.get(self.PROVIDER_ID_KEY)
+        self.__uploaded_files_json = json.loads(os.environ.get(self.UPLOADED_FILES_JSON))
+        self.__verify_ssl = os.environ.get(self.VERIFY_SSL_KEY, 'TRUE').strip().upper() == 'TRUE'
+        return self
+
+    def catalog(self, **kwargs):
+        self.__set_props_from_env()
+        dapa_body = {
+            "provider_id": self.__provider_id,
+            "features": self.__uploaded_files_json
+        }
+        dapa_client = DapaClient().with_verify_ssl(self.__verify_ssl)
+        LOGGER.debug(f'dapa_body_granules: {dapa_body}')
+        dapa_ingest_result = dapa_client.ingest_granules_w_cnm(dapa_body)
+        return dapa_ingest_result

cumulus_lambda_functions/stage_in_out/catalog_granules_abstract.py

@@ -0,0 +1,7 @@
+from abc import ABC, abstractmethod
+
+
+class DownloadGranulesAbstract(ABC):
+    @abstractmethod
+    def download(self, **kwargs) -> list:
+        raise NotImplementedError()