Add clipper deployment support

.gitignore

@@ -115,3 +115,6 @@ yarn.lock
 
 # docs
 built-docs
+
+# MacOS X
+.DS_Store

README.md

@@ -108,8 +108,8 @@ directory. More tutorials and examples coming soon!
 Deployment guides:
 - [Serverless deployment with AWS Lambda](https://github.com/bentoml/BentoML/blob/master/examples/deploy-with-serverless)
 - [API server deployment with AWS SageMaker](https://github.com/bentoml/BentoML/blob/master/examples/deploy-with-sagemaker)
+- [API server deployment with Clipper](https://github.com/bentoml/BentoML/blob/master/example/deploy-with-clipper/deploy-iris-classifier-to-clipper.ipynb)
 - [(WIP) API server deployment on Kubernetes](https://github.com/bentoml/BentoML/tree/master/examples/deploy-with-kubernetes)
-- [(WIP) API server deployment with Clipper](https://github.com/bentoml/BentoML/pull/151)
 
 
 We collect example notebook page views to help us improve this project.

bentoml/__init__.py

@@ -31,6 +31,7 @@
 from bentoml.archive import save, load
 
 from bentoml.utils.log import configure_logging
+from bentoml import deployment
 
 configure_logging()
 
@@ -46,4 +47,5 @@
     "handlers",
     "metrics",
     "BentoService",
+    "deployment",
 ]

bentoml/deployment/clipper/__init__.py

@@ -0,0 +1,137 @@
+# Copyright 2019 Atalaya Tech, Inc.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import shutil
+import re
+
+import docker
+
+from bentoml.archive import load
+from bentoml.handlers import ImageHandler
+from bentoml.deployment.utils import (
+    generate_bentoml_deployment_snapshot_path,
+    process_docker_api_line,
+)
+from bentoml.deployment.clipper.templates import (
+    DEFAULT_CLIPPER_ENTRY,
+    DOCKERFILE_CLIPPER,
+)
+from bentoml.exceptions import BentoMLException
+
+
+def generate_clipper_compatiable_string(item):
+    """Generate clipper compatiable string. It must be a valid DNS-1123.
+    It must consist of lower case alphanumeric characters, '-' or '.',
+    and must start and end with an alphanumeric character
+
+    :param item: String
+    :return: string
+    """
+
+    pattern = re.compile("[^a-zA-Z0-9-]")
+    result = re.sub(pattern, "-", item)
+    return result.lower()
+
+
+def deploy_bentoml(
+    clipper_conn,
+    archive_path,
+    api_name,
+    input_type="strings",
+    model_name=None,
+    labels=["bentoml"],
+):
+    """Deploy bentoml bundle to clipper cluster
+
+    Args:
+        clipper_conn(clipper_admin.ClipperConnection): Clipper connection instance
+        archive_path(str): Path to the bentoml service archive.
+        api_name(str): name of the api that will be used as prediction function for
+            clipper cluster
+        input_type(str): Input type that clipper accept. The default input_type for
+            image handler is `bytes`, for other handlers is `strings`. Availabel input_type
+            are `integers`, `floats`, `doubles`, `bytes`, or `strings`
+        model_name(str): Model's name for clipper cluster
+        labels(:obj:`list(str)`, optional): labels for clipper model
+
+    Returns:
+        tuple: Model name and model version that deployed to clipper
+
+    """
+    bento_service = load(archive_path)
+    apis = bento_service.get_service_apis()
+
+    if api_name:
+        api = next(item for item in apis if item.name == api_name)
+    elif len(apis) == 1:
+        api = apis[0]
+    else:
+        raise BentoMLException(
+            "Please specify api-name, when more than one API is present in the archive"
+        )
+    model_name = model_name or generate_clipper_compatiable_string(
+        bento_service.name + "-" + api.name
+    )
+    version = generate_clipper_compatiable_string(bento_service.version)
+
+    if isinstance(api.handler, ImageHandler):
+        input_type = "bytes"
+
+    try:
+        clipper_conn.start_clipper()
+    except docker.errors.APIError:
+        clipper_conn.connect()
+    except Exception:
+        raise BentoMLException("Can't start or connect with clipper cluster")
+
+    snapshot_path = generate_bentoml_deployment_snapshot_path(
+        bento_service.name, bento_service.version, "clipper"
+    )
+
+    entry_py_content = DEFAULT_CLIPPER_ENTRY.format(
+        api_name=api.name, input_type=input_type
+    )
+    model_path = os.path.join(snapshot_path, "bento")
+    shutil.copytree(archive_path, model_path)
+
+    with open(os.path.join(snapshot_path, "clipper_entry.py"), "w") as f:
+        f.write(entry_py_content)
+
+    docker_content = DOCKERFILE_CLIPPER.format(
+        model_name=model_name, model_version=version
+    )
+    with open(os.path.join(snapshot_path, "Dockerfile-clipper"), "w") as f:
+        f.write(docker_content)
+
+    docker_api = docker.APIClient()
+    image_tag = bento_service.name.lower() + "-clipper:" + bento_service.version
+    for line in docker_api.build(
+        path=snapshot_path, dockerfile="Dockerfile-clipper", tag=image_tag
+    ):
+        process_docker_api_line(line)
+
+    clipper_conn.deploy_model(
+        name=model_name,
+        version=version,
+        input_type=input_type,
+        image=image_tag,
+        labels=labels,
+    )
+
+    return model_name, version

bentoml/deployment/clipper/templates.py

@@ -0,0 +1,127 @@
+# Copyright 2019 Atalaya Tech, Inc.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+
+DEFAULT_CLIPPER_ENTRY = """\
+from __future__ import print_function
+
+import rpc # this is clipper's rpc.py module
+import os
+import sys
+
+from bentoml import load
+
+IMPORT_ERROR_RETURN_CODE = 3
+
+bento_service = load('/container/bento')
+apis = bento_service.get_service_apis()
+
+api = next(item for item in apis if item.name == '{api_name}')
+if not api:
+    raise BentoMLException("Can't find api with name %s" % {api_name})
+
+class BentoClipperContainer(rpc.ModelContainerBase):
+    def __init__(self):
+        self.input_type = '{input_type}'
+
+    def predict_ints(self, inputs):
+        preds = api.handle_clipper_ints(inputs)
+        return [str(p) for p in preds]
+
+    def predict_floats(self, inputs):
+        preds = api.handle_clipper_floats(inputs)
+        return [str(p) for p in preds]
+
+    def predict_doubles(self, inputs):
+        preds = api.handle_clipper_doubles(inputs)
+        return [str(p) for p in preds]
+
+    def predict_bytes(self, inputs):
+        preds = api.handle_clipper_bytes(inputs)
+        return [str(p) for p in preds]
+
+    def predict_strings(self, inputs):
+        preds = api.handle_clipper_strings(inputs)
+        return [str(p) for p in preds]
+
+
+if __name__ == "__main__":
+    print("Starting Bento service Clipper Containter")
+    rpc_service = rpc.RPCService()
+
+    try:
+        model = BentoClipperContainer()
+        sys.stdout.flush()
+        sys.stderr.flush()
+    except ImportError:
+        sys.exit(IMPORT_ERROR_RETURN_CODE)
+
+    rpc_service.start(model)
+"""
+
+DOCKERFILE_CLIPPER = """\
+FROM clipper/python36-closure-container:0.3
+
+# Install miniconda3 for python. Copied from
+# https://github.com/ContinuumIO/docker-images/blob/master/miniconda3/debian/Dockerfile
+ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
+ENV PATH /opt/conda/bin:$PATH
+
+RUN set -x \
+    && apt-get update --fix-missing \
+    && apt-get install -y wget bzip2 ca-certificates curl git libpq-dev build-essential \
+                          libzmq3-dev libzmq5 libzmq5-dev redis-server libsodium18 \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.6.14-Linux-x86_64.sh -O ~/miniconda.sh && \
+    /bin/bash ~/miniconda.sh -b -p /opt/conda && \
+    rm ~/miniconda.sh && \
+    /opt/conda/bin/conda clean -tipsy && \
+    ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \
+    echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \
+    echo "conda activate base" >> ~/.bashrc
+
+
+# update conda and setup environment and pre-install common ML libraries to speed up docker build
+RUN conda update conda -y \
+      && conda install pip numpy scipy \
+      && pip install six bentoml
+
+
+# copy over model files
+COPY . /container
+WORKDIR /container
+
+# update conda base env
+RUN conda env update -n base -f /container/bento/environment.yml
+RUN pip install -r /container/bento/requirements.txt
+
+# Install packages for clipper
+RUN pip install clipper_admin pyzmq==17.0.* prometheus_client==0.1.* pyyaml>=4.2b1 jsonschema==2.6.* redis==2.10.* psutil==5.4.* flask==0.12.2
+
+# run user defined setup script
+RUN if [ -f /container/bento/setup.sh ]; then /bin/bash -c /container/bento/setup.sh; fi
+
+ENV CLIPPER_MODEL_NAME={model_name}
+ENV CLIPPER_MODEL_VERSION={model_version}
+
+# Stop running entry point from base image
+ENTRYPOINT []
+# Run BentoML bundle for clipper
+CMD ["python", "/container/clipper_entry.py"]
+"""  # noqa: E501

bentoml/deployment/sagemaker/__init__.py

@@ -29,7 +29,10 @@
 import docker
 
 from bentoml.deployment.base_deployment import Deployment
-from bentoml.deployment.utils import generate_bentoml_deployment_snapshot_path
+from bentoml.deployment.utils import (
+    generate_bentoml_deployment_snapshot_path,
+    process_docker_api_line,
+)
 from bentoml.utils.whichcraft import which
 from bentoml.exceptions import BentoMLException
 from bentoml.deployment.sagemaker.templates import (
@@ -56,26 +59,6 @@ def strip_scheme(url):
     return parsed.geturl().replace(scheme, "", 1)
 
 
-def process_docker_api_line(payload):
-    """ Process the output from API stream, throw an Exception if there is an error """
-    # Sometimes Docker sends to "{}\n" blocks together...
-    for segment in payload.decode("utf-8").split("\n"):
-        line = segment.strip()
-        if line:
-            try:
-                line_payload = json.loads(line)
-            except ValueError as e:
-                print("Could not decipher payload from API: " + e)
-            if line_payload:
-                if "errorDetail" in line_payload:
-                    error = line_payload["errorDetail"]
-                    sys.stderr.write(error["message"])
-                    raise RuntimeError("Error on build - code %s" % error["code"])
-                elif "stream" in line_payload:
-                    # TODO: move this to logger.info
-                    sys.stdout.write(line_payload["stream"])
-
-
 def generate_aws_compatible_string(item):
     pattern = re.compile("[^a-zA-Z0-9-]|_")
     return re.sub(pattern, "-", item)

bentoml/deployment/serverless/aws_lambda_template.py

@@ -90,7 +90,6 @@ def generate_serverless_configuration_for_aws(
     }
 
     serverless_config["custom"] = custom_config
-    # package_config = {"include": ["handler.py", service_name + "/**"]}
 
     yaml.dump(serverless_config, Path(config_path))
     return

bentoml/deployment/utils.py

@@ -17,11 +17,14 @@
 from __future__ import print_function
 
 import os
+import json
+import logging
 
 from datetime import datetime
 
 from bentoml.config import BENTOML_HOME
 
+logger = logging.getLogger(__name__)
 
 def generate_bentoml_deployment_snapshot_path(service_name, service_version, platform):
     return os.path.join(
@@ -32,3 +35,22 @@ def generate_bentoml_deployment_snapshot_path(service_name, service_version, pla
         service_version,
         datetime.now().isoformat(),
     )
+
+
+def process_docker_api_line(payload):
+    """ Process the output from API stream, throw an Exception if there is an error """
+    # Sometimes Docker sends to "{}\n" blocks together...
+    for segment in payload.decode("utf-8").split("\n"):
+        line = segment.strip()
+        if line:
+            try:
+                line_payload = json.loads(line)
+            except ValueError as e:
+                print("Could not decipher payload from Docker API: " + e)
+            if line_payload:
+                if "errorDetail" in line_payload:
+                    error = line_payload["errorDetail"]
+                    logger.error(error['message'])
+                    raise RuntimeError("Error on build - code %s" % error["code"])
+                elif "stream" in line_payload:
+                    logger.info(line_payload['stream'])