OpenVINO Model Server deployer

contrib/components/openvino/ovms-deployer/README.md

@@ -0,0 +1,32 @@
+# Deployer of OpenVINO Model Server
+
+This component triggers deployment of [OpenVINO Model Server](https://github.com/IntelAI/OpenVINO-model-server) in Kubernetes.
+
+It applies the passed component parameters on jinja template and applied deployment and server records.
+
+
+
+```bash
+./deploy.sh
+     --model-export-path
+     --cluster-name
+     --namespace
+     --server-name
+     --replicas
+     --batch-size
+     --model-version-policy
+     --log-level
+```
+
+
+## building docker image
+
+
+```bash
+docker build --build-arg http_proxy=$http_proxy --build-arg https_proxy=$https_proxy .
+```
+
+## testing the image locally
+
+
+```

contrib/components/openvino/ovms-deployer/containers/Dockerfile

@@ -0,0 +1,44 @@
+FROM intelpython/intelpython3_core
+
+RUN apt-get update -q && apt-get upgrade -y && \
+    apt-get install -y -qq --no-install-recommends \
+      apt-transport-https \
+      ca-certificates \
+      git \
+      gnupg \
+      lsb-release \
+      unzip \
+      wget && \
+    wget -O /opt/ks_0.12.0_linux_amd64.tar.gz \
+      https://github.com/ksonnet/ksonnet/releases/download/v0.12.0/ks_0.12.0_linux_amd64.tar.gz && \
+    tar -C /opt -xzf /opt/ks_0.12.0_linux_amd64.tar.gz && \
+    cp /opt/ks_0.12.0_linux_amd64/ks /bin/. && \
+    rm -f /opt/ks_0.12.0_linux_amd64.tar.gz && \
+    wget -O /bin/kubectl \
+      https://storage.googleapis.com/kubernetes-release/release/v1.11.2/bin/linux/amd64/kubectl && \
+    chmod u+x /bin/kubectl && \
+    wget -O /opt/kubernetes_v1.11.2 \
+      https://github.com/kubernetes/kubernetes/archive/v1.11.2.tar.gz && \
+    mkdir -p /src && \
+    tar -C /src -xzf /opt/kubernetes_v1.11.2 && \
+    rm -rf /opt/kubernetes_v1.11.2 && \
+    wget -O /opt/google-apt-key.gpg \
+      https://packages.cloud.google.com/apt/doc/apt-key.gpg && \
+    apt-key add /opt/google-apt-key.gpg && \
+    export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \
+    echo "deb https://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" >> \
+      /etc/apt/sources.list.d/google-cloud-sdk.list && \
+    apt-get update -q && \
+    apt-get install -y -qq --no-install-recommends google-cloud-sdk && \
+    gcloud config set component_manager/disable_update_check true
+
+RUN conda install -y opencv && conda clean -a -y
+ADD requirements.txt /deploy/
+WORKDIR /deploy
+RUN pip install -r requirements.txt
+ADD apply_template.py deploy.sh evaluate.py ovms.j2 classes.py /deploy/
+ENTRYPOINT ["./deploy.sh"]
+
+
+
+

contrib/components/openvino/ovms-deployer/containers/apply_template.py

@@ -0,0 +1,14 @@
+from jinja2 import Template
+import os
+
+
+f = open("ovms.j2","r")
+ovms_template = f.read()
+t = Template(ovms_template)
+ovms_k8s = t.render(os.environ)
+f.close
+f = open("ovms.yaml", "w")
+f.write(ovms_k8s)
+f.close
+
+print(ovms_k8s)

contrib/components/openvino/ovms-deployer/containers/deploy.sh

@@ -0,0 +1,154 @@
+#!/bin/bash -e
+
+set -x
+
+KUBERNETES_NAMESPACE="${KUBERNETES_NAMESPACE:-kubeflow}"
+SERVER_NAME="${SERVER_NAME:-model-server}"
+SERVER_ENDPOINT_OUTPUT_FILE="${SERVER_ENDPOINT_OUTPUT_FILE:-/tmp/server_endpoint/data}"
+
+while (($#)); do
+   case $1 in
+     "--model-export-path")
+       shift
+       export MODEL_EXPORT_PATH="$1"
+       shift
+       ;;
+     "--cluster-name")
+       shift
+       CLUSTER_NAME="$1"
+       shift
+       ;;
+     "--namespace")
+       shift
+       KUBERNETES_NAMESPACE="$1"
+       shift
+       ;;
+     "--server-name")
+       shift
+       SERVER_NAME="$1"
+       shift
+       ;;
+     "--replicas")
+       shift
+       export REPLICAS="$1"
+       shift
+       ;;
+     "--batch-size")
+       shift
+       export BATCH_SIZE="$1"
+       shift
+       ;;
+     "--model-version-policy")
+       shift
+       export MODEL_VERSION_POLICY="$1"
+       shift
+       ;;
+     "--log-level")
+       shift
+       export LOG_LEVEL="$1"
+       shift
+       ;;
+     "--server-endpoint-output-file")
+       shift
+       SERVER_ENDPOINT_OUTPUT_FILE = "$1"
+       shift
+       ;;
+     *)
+       echo "Unknown argument: '$1'"
+       exit 1
+       ;;
+   esac
+done
+
+if [ -z "${MODEL_EXPORT_PATH}" ]; then
+  echo "You must specify a path to the saved model"
+  exit 1
+fi
+
+echo "Deploying the model '${MODEL_EXPORT_PATH}'"
+
+if [ -z "${CLUSTER_NAME}" ]; then
+  CLUSTER_NAME=$(wget -q -O- --header="Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/instance/attributes/cluster-name)
+fi
+
+# Ensure the server name is not more than 63 characters.
+export SERVER_NAME="${SERVER_NAME:0:63}"
+# Trim any trailing hyphens from the server name.
+while [[ "${SERVER_NAME:(-1)}" == "-" ]]; do SERVER_NAME="${SERVER_NAME::-1}"; done
+
+echo "Deploying ${SERVER_NAME} to the cluster ${CLUSTER_NAME}"
+
+# Connect kubectl to the local cluster
+kubectl config set-cluster "${CLUSTER_NAME}" --server=https://kubernetes.default --certificate-authority=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+kubectl config set-credentials pipeline --token "$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
+kubectl config set-context kubeflow --cluster "${CLUSTER_NAME}" --user pipeline
+kubectl config use-context kubeflow
+
+echo "Generating service and deployment yaml files"
+python apply_template.py
+
+kubectl apply -f ovms.yaml
+
+sleep 10
+echo "Waiting for the TF Serving deployment to have at least one available replica..."
+timeout="1000"
+start_time=`date +%s`
+while [[ $(kubectl get deploy --namespace "${KUBERNETES_NAMESPACE}" --selector=app="ovms-${SERVER_NAME}" --output=jsonpath='{.items[0].status.availableReplicas}') < "1" ]]; do
+  current_time=`date +%s`
+  elapsed_time=$(expr $current_time + 1 - $start_time)
+  if [[ $elapsed_time -gt $timeout ]];then
+    echo "timeout"
+    exit 1
+  fi
+  sleep 5
+done
+
+echo "Obtaining the pod name..."
+start_time=`date +%s`
+pod_name=""
+while [[ $pod_name == "" ]];do
+  pod_name=$(kubectl get pods --namespace "${KUBERNETES_NAMESPACE}" --selector=app="ovms-${SERVER_NAME}" --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}')
+  current_time=`date +%s`
+  elapsed_time=$(expr $current_time + 1 - $start_time)
+  if [[ $elapsed_time -gt $timeout ]];then
+    echo "timeout"
+    exit 1
+  fi
+  sleep 2
+done
+echo "Pod name is: " $pod_name
+
+# Wait for the pod container to start running
+echo "Waiting for the TF Serving pod to start running..."
+start_time=`date +%s`
+exit_code="1"
+while [[ $exit_code != "0" ]];do
+  kubectl get po ${pod_name} --namespace "${KUBERNETES_NAMESPACE}" -o jsonpath='{.status.containerStatuses[0].state.running}'
+  exit_code=$?
+  current_time=`date +%s`
+  elapsed_time=$(expr $current_time + 1 - $start_time)
+  if [[ $elapsed_time -gt $timeout ]];then
+    echo "timeout"
+    exit 1
+  fi
+  sleep 2
+done
+
+start_time=`date +%s`
+while [ -z "$(kubectl get po ${pod_name} --namespace "${KUBERNETES_NAMESPACE}" -o jsonpath='{.status.containerStatuses[0].state.running}')" ]; do
+  current_time=`date +%s`
+  elapsed_time=$(expr $current_time + 1 - $start_time)
+  if [[ $elapsed_time -gt $timeout ]];then
+    echo "timeout"
+    exit 1
+  fi
+  sleep 5
+done
+
+# Wait a little while and then grab the logs of the running server
+sleep 10
+echo "Logs from the TF Serving pod:"
+kubectl logs ${pod_name} --namespace "${KUBERNETES_NAMESPACE}"
+
+mkdir -p "$(dirname "$SERVER_ENDPOINT_OUTPUT_FILE")"
+echo "ovms-${SERVER_NAME}:80" > "$SERVER_ENDPOINT_OUTPUT_FILE"

contrib/components/openvino/ovms-deployer/containers/evaluate.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python
+import grpc
+import numpy as np
+import tensorflow.contrib.util as tf_contrib_util
+import datetime
+import argparse
+from tensorflow_serving.apis import predict_pb2
+from tensorflow_serving.apis import prediction_service_pb2_grpc
+from urllib.parse import urlparse
+import requests
+import cv2
+import os
+import json
+import classes
+
+
+def crop_resize(img,cropx,cropy):
+    y,x,c = img.shape
+    if y < cropy:
+        img = cv2.resize(img, (x, cropy))
+        y = cropy
+    if x < cropx:
+        img = cv2.resize(img, (cropx,y))
+        x = cropx
+    startx = x//2-(cropx//2)
+    starty = y//2-(cropy//2)
+    return img[starty:starty+cropy,startx:startx+cropx,:]
+
+def get_file_content(source_path):
+    parsed_path = urlparse(source_path)
+    if parsed_path.scheme == "http" or parsed_path.scheme == "https":
+        try:
+            response = requests.get(source_path, stream=True)
+            content = response.content
+        except requests.exceptions.RequestException as e:
+            print(e)
+            content = None
+    elif parsed_path.scheme == "":
+        if os.path.isfile(source_path):
+            with open(input_images) as f:
+                content = f.readlines()
+            f.close
+        else:
+            print("file " + source_path + "is not accessible")
+            content = None
+    return content
+
+def getJpeg(path, size, path_prefix):
+    print(os.path.join(path_prefix,path))
+    content = get_file_content(os.path.join(path_prefix,path))
+
+    if content:
+        try:
+            img = np.frombuffer(content, dtype=np.uint8)
+            img = cv2.imdecode(img, cv2.IMREAD_COLOR)  # BGR format
+            # retrived array has BGR format and 0-255 normalization
+            # add image preprocessing if needed by the model
+            img = crop_resize(img, size, size)
+            img = img.astype('float32')
+            img = img.transpose(2,0,1).reshape(1,3,size,size)
+            print(path, img.shape, "; data range:",np.amin(img),":",np.amax(img))
+        except e:
+            print("Can not read the image file", e)
+            img = None
+    else:
+        print("Can not open ", os.path(path_prefix,path))
+        img = None
+    return img
+
+parser = argparse.ArgumentParser(description='Sends requests to OVMS and TF Serving using images in numpy format')
+parser.add_argument('--images_list', required=False, default='input_images.txt', help='Path to a file with a list of labeled images. It should include in every line a path to the image file and a numerical label separate by space.')
+parser.add_argument('--grpc_endpoint',required=False, default='localhost:9000',  help='Specify endpoint of grpc service. default:localhost:9000')
+parser.add_argument('--input_name',required=False, default='input', help='Specify input tensor name. default: input')
+parser.add_argument('--output_name',required=False, default='resnet_v1_50/predictions/Reshape_1', help='Specify output name. default: output')
+parser.add_argument('--model_name', default='resnet', help='Define model name, must be same as is in service. default: resnet',
+                    dest='model_name')
+parser.add_argument('--size',required=False, default=224, type=int, help='The size of the image in the model')
+parser.add_argument('--image_path_prefix',required=False, default="", type=str, help='Path prefix to be added to every image in the list')
+args = vars(parser.parse_args())
+
+channel = grpc.insecure_channel(args['grpc_endpoint'])
+stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)
+input_images = args.get('images_list')
+size = args.get('size')
+
+input_list_content = get_file_content(input_images)
+if input_list_content is None:
+    print("Can not open input images file", input_images)
+    exit(1)
+else:
+    lines = input_list_content.decode().split("\n")
+print(lines)
+print('Start processing:')
+print('\tModel name: {}'.format(args.get('model_name')))
+print('\tImages list file: {}'.format(args.get('images_list')))
+
+i = 0
+matched = 0
+processing_times = np.zeros((0),int)
+imgs = np.zeros((0,3,size, size), np.dtype('<f'))
+lbs = np.zeros((0), int)
+
+for line in lines:
+    path, label = line.strip().split(" ")
+    img = getJpeg(path, size, args.get('image_path_prefix'))
+    if img is not None:
+        request = predict_pb2.PredictRequest()
+        request.model_spec.name = args.get('model_name')
+        request.inputs[args['input_name']].CopyFrom(tf_contrib_util.make_tensor_proto(img, shape=(img.shape)))
+        start_time = datetime.datetime.now()
+        result = stub.Predict(request, 10.0) # result includes a dictionary with all model outputs
+        end_time = datetime.datetime.now()
+        if args['output_name'] not in result.outputs:
+            print("Invalid output name", args['output_name'])
+            print("Available outputs:")
+            for Y in result.outputs:
+                print(Y)
+            exit(1)
+        duration = (end_time - start_time).total_seconds() * 1000
+        processing_times = np.append(processing_times,np.array([int(duration)]))
+        output = tf_contrib_util.make_ndarray(result.outputs[args['output_name']])
+        nu = np.array(output)
+        # for object classification models show imagenet class
+        print('Processing time: {:.2f} ms; speed {:.2f} fps'.format(round(duration), 2),
+          round(1000 / duration, 2)
+          )
+        ma = np.argmax(nu)
+        if int(label) == ma:
+            matched += 1
+        i += 1
+        print("Detected: {} - {} ; Should be: {} - {}".format(ma,classes.imagenet_classes[int(ma)],label,classes.imagenet_classes[int(label)]))
+
+accuracy = matched/i
+latency = np.average(processing_times)
+metrics = {'metrics': [{'name': 'accuracy-score','numberValue':  accuracy,'format': "PERCENTAGE"},
+                       {'name': 'latency','numberValue':  latency,'format': "RAW"}]}
+with open('/mlpipeline-metrics.json', 'w') as f:
+    json.dump(metrics, f)
+f.close
+print("\nOverall accuracy=",matched/i*100,"%")
+print("Average latency=",latency,"ms")