Merge pull request #27 from unity-sds/push_ecr

Add ability to push into an AWS ECR registry using environment credentials

Author: mcduffie

requirements.txt

@@ -1,2 +1,3 @@
-app_pack_generator>=1.0.0
+boto3
+app_pack_generator>=1.1.0
 unity-sds-client>=0.2.0

unity_app_generator/__main__.py

@@ -54,16 +54,26 @@ def init(args):
 def build_docker(args):
     state_dir = check_state_directory(state_directory_path(args))
 
-    app_gen = UnityApplicationGenerator(state_dir, use_owner=args.use_owner)
+    app_gen = UnityApplicationGenerator(state_dir,
+                                        use_namespace=args.image_namespace,
+                                        use_repository=args.image_repository,
+                                        use_tag=args.image_tag)
 
     app_gen.create_docker_image()
 
 def push_docker(args):
     state_dir = check_state_directory(state_directory_path(args))
 
     app_gen = UnityApplicationGenerator(state_dir)
-    
-    app_gen.push_to_docker_registry(args.container_registry, image_tag=args.image_tag)
+
+    app_gen.push_to_docker_registry(args.container_registry)
+
+def push_ecr(args):
+    state_dir = check_state_directory(state_directory_path(args))
+
+    app_gen = UnityApplicationGenerator(state_dir)
+
+    app_gen.push_to_aws_ecr()
 
 def notebook_parameters(args):
 
@@ -94,6 +104,9 @@ def main():
     parser.add_argument("--state_directory",
         help=f"An alternative location to store the application state other than {STATE_DIRECTORY}")
 
+    parser.add_argument("--verbose", "-v", action="store_true", default=False,
+        help=f"Enable verbose logging")
+
     # init
     subparsers = parser.add_subparsers(required=True)
 
@@ -116,8 +129,14 @@ def main():
     parser_build_docker = subparsers.add_parser('build_docker',
         help=f"Build a Docker image from the initialized application directory")
 
-    parser_build_docker.add_argument("--no_owner", dest="use_owner", action="store_false", default=True,
-        help="Disable using the owner of the Git repository in the Docker image tag")
+    parser_build_docker.add_argument("-n", "--image_namespace", 
+        help="Docker image namespace to use instead of the automatically generated one from the Git repository owner. An empty string removes the namespace from the image reference.")
+
+    parser_build_docker.add_argument("-r", "--image_repository", 
+        help="Docker image repository to use instead of the automatically generated one from the Git repository name.")
+
+    parser_build_docker.add_argument("-t", "--image_tag", 
+        help="Docker image tag to use instead of the automatically generated one from the Git commit id")
 
     parser_build_docker.set_defaults(func=build_docker)
 
@@ -129,11 +148,15 @@ def main():
     parser_push_docker.add_argument("container_registry", 
         help="URL or Dockerhub username of a Docker registry for pushing of the built image")
 
-    parser_push_docker.add_argument("-t", "--image_tag", 
-        help="Docker image tag to push into container registry if already built without using the build_docker subcommand")
-
     parser_push_docker.set_defaults(func=push_docker)
 
+    # push_ecr
+
+    parser_push_ecr = subparsers.add_parser('push_ecr',
+        help=f"Push a Docker image from the initialized application directory to an AWS Elastic Container Registry (ECR)")
+
+    parser_push_ecr.set_defaults(func=push_ecr)
+
     # notebook_parameters
 
     parser_parameters = subparsers.add_parser('parameters',
@@ -174,7 +197,10 @@ def main():
 
     args = parser.parse_args()
 
-    logging.basicConfig(level=logging.DEBUG)
+    if args.verbose:
+        logging.basicConfig(level=logging.DEBUG)
+    else:
+        logging.basicConfig(level=logging.INFO)
 
     try:
         args.func(args)
@@ -185,4 +211,4 @@ def main():
     #app_gen.push_to_application_registry(None)
 
 if __name__ == '__main__':
-    main()
+    main()

unity_app_generator/ecr_helper.py

@@ -0,0 +1,71 @@
+import base64
+
+import boto3
+from botocore.exceptions import ClientError
+
+import logging
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from app_pack_generator import DockerUtil
+
+logger = logging.getLogger(__name__)
+
+class ECRHelper(object):
+
+    def __init__(self, docker_util : 'DockerUtil'):
+
+        self.docker_util = docker_util
+        self.ecr_client = boto3.client("ecr")
+
+    def create_repository(self):
+
+        if self.docker_util.image_namespace is not None and self.docker_util.image_namespace != "":
+            aws_repo_name = f"{self.docker_util.image_namespace}/{self.docker_util.image_repository}"
+        else:
+            aws_repo_name = self.docker_util.image_repository
+
+        logger.info(f"Creating AWS ECR repository named: {aws_repo_name}")
+        
+        try:
+            response = self.ecr_client.create_repository(
+                repositoryName=aws_repo_name,
+            )
+
+            return response["repository"]['repositoryUri']
+
+        except ClientError as err:
+            if err.response["Error"]["Code"] == "RepositoryAlreadyExistsException":
+                logger.debug(f"Repository {aws_repo_name} already exists.")
+                response = self.ecr_client.describe_repositories(
+                    repositoryNames=[aws_repo_name]
+                )
+                
+                return response['repositories'][0]['repositoryUri']
+            else:
+                logger.error(
+                    "Error creating repository %s. Here's why %s",
+                    repository_name,
+                    err.response["Error"]["Message"],
+                )
+                raise
+
+    def docker_login(self):
+
+        logger.info("Logging into Docker using ECR credentials")
+
+        token = self.ecr_client.get_authorization_token()
+        username, password = base64.b64decode(token['authorizationData'][0]['authorizationToken']).decode().split(':')
+
+        # Remove the protocol prefix for this to work
+        # https://github.com/docker/docker-py/issues/2256
+        registry = token['authorizationData'][0]['proxyEndpoint'].replace("https://", "")
+
+        response = self.docker_util.docker_client.login(
+            username=username,
+            password=password,
+            registry=registry
+        )
+
+        return registry

unity_app_generator/generator.py

@@ -5,6 +5,7 @@
 from glob import glob
 
 from .state import ApplicationState
+from .ecr_helper import ECRHelper
 
 from app_pack_generator import GitManager, DockerUtil, ApplicationNotebook
 from app_pack_generator import ProcessCWL, DataStagingCWL, Descriptor
@@ -18,7 +19,8 @@ class ApplicationGenerationError(Exception):
 
 class UnityApplicationGenerator(object):
 
-    def __init__(self, state_directory, source_repository=None, destination_directory=None, checkout=None, use_owner=True):
+    def __init__(self, state_directory, source_repository=None, destination_directory=None, checkout=None, 
+                 use_namespace=None, use_repository=None, use_tag=None):
 
         if not ApplicationState.exists(state_directory):
             self.repo_info = self._localize_source(source_repository, destination_directory, checkout)
@@ -27,34 +29,54 @@ def __init__(self, state_directory, source_repository=None, destination_director
             self.app_state = ApplicationState(state_directory)
             self.repo_info = self._localize_source(self.app_state.source_repository, self.app_state.app_base_path, checkout)
 
-        self.docker_util = DockerUtil(self.repo_info, do_prune=False, use_owner=use_owner)
+        # Use value from command line to override values saved into app state
+        image_namespace = use_namespace if use_namespace is not None else self.app_state.docker_image_namespace
+        image_repository = use_repository if use_repository is not None else self.app_state.docker_image_repository
+        image_tag = use_tag if use_tag is not None else self.app_state.docker_image_tag
+
+        self.docker_util = DockerUtil(self.repo_info, do_prune=False,
+                                      use_namespace=image_namespace,
+                                      use_repository=image_repository,
+                                      use_tag=image_tag)
 
     def _localize_source(self, source, dest, checkout):
 
         # Check out original repository
         git_mgr = GitManager(source, dest)
 
         if checkout is not None:
-            logger.debug(f"Checking out {checkout} in {repo_dir}")
+            logger.info(f"Checking out {checkout} in {repo_dir}")
             git_mgr.checkout(checkout)
 
         return git_mgr
 
     def create_docker_image(self):
+ 
+        # These come either from the commandline or are generated by Docker util
+        self.app_state.docker_image_namespace = self.docker_util.image_namespace
+        self.app_state.docker_image_repository = self.docker_util.image_repository
+        self.app_state.docker_image_tag = self.docker_util.image_tag
 
         # Create Docker image
-        self.app_state.docker_image_tag = self.docker_util.repo2docker()
-
-    def push_to_docker_registry(self, docker_registry, image_tag=None):
+        self.app_state.docker_image_reference = self.docker_util.build_image()
 
-        if image_tag is not None:
-            self.app_state.docker_image_tag = image_tag
-        
-        if self.app_state.docker_image_tag is None:
-            raise ApplicationGenerationError("Cannot push Docker image to registry without a valid tag. Run the Docker build command or supply an image tag as an argument.")
+    def push_to_docker_registry(self, docker_registry):
 
         # Push to remote repository
-        self.app_state.docker_url = self.docker_util.push_image(docker_registry, self.app_state.docker_image_tag)
+        self.app_state.docker_url = self.docker_util.push_image(docker_registry, self.app_state.docker_image_reference)
+
+    def push_to_aws_ecr(self):
+
+        ecr_helper = ECRHelper(self.docker_util)
+
+        # Create an ECR registry if it doesn't already exist
+        ecr_helper.create_repository()
+
+        # Log in to ECR via Docker
+        registry_url = ecr_helper.docker_login()
+
+        # Push docker image into ECR
+        self.push_to_docker_registry(registry_url)
 
     def _generate_dockstore_cwl(self, cwl_output_path, target_cwl_filename):
 
@@ -77,12 +99,12 @@ def _generate_dockstore_cwl(self, cwl_output_path, target_cwl_filename):
 
     def create_cwl(self, cwl_output_path=None, docker_url=None, monolithic=False):
 
-        # Fall through using docker_image_tag if docker_url does not exist because no push has occurred
+        # Fall through using docker_image_reference if docker_url does not exist because no push has occurred
         # Or if docker_url is supplied as an argument use that
         if docker_url is None and self.app_state.docker_url is not None:
             docker_url = self.app_state.docker_url
-        elif docker_url is None and self.app_state.docker_image_tag is not None:
-            docker_url = self.app_state.docker_image_tag
+        elif docker_url is None and self.app_state.docker_image_reference is not None:
+            docker_url = self.app_state.docker_image_reference
         elif docker_url is None:
             raise ApplicationGenerationError("Cannot create CWL files when Docker image tag or URL has not yet been registered through building and/or pushing Docker image")
 

unity_app_generator/state.py

@@ -17,7 +17,10 @@ class ApplicationState(object):
         "cwl_output_path": "",
 
         "source_repository": None,
+        "docker_image_namespace": None,
+        "docker_image_repository": None,
         "docker_image_tag": None,
+        "docker_image_reference": None,
         "docker_url": None,
         "app_registry_id": None,
     }

unity_app_generator/version.py

@@ -1 +1 @@
-__version__="1.0.0"
+__version__="1.1.0"