add mediapipe model

Author: yashbonde

posemodel/.nboxignore

@@ -0,0 +1,4 @@
+__pycache__/
+.git/
+.vscode/
+venv/

posemodel/README.md

@@ -0,0 +1,47 @@
+# Mediapipe Model
+
+In this example we are going to deploy a (MediaPipe)[https://mediapipe.dev/] model for landmark detection. There are many ways to call a deployed a CV model:
+- directly transfering uint8 array: as you would think this creates the heaviest message, this is the worst. Do it just to realise how bad it is.
+- transferring the bytes of the image: this is an industry standard approach where the client would send in a base64 encoded image bytes and the server would reconstruct the image, this is a good case when your server is in an unsafe environment
+- sending in a URL and server would fetch it: this is good when your server is in a safe environment and you know what are the actual contents of the URL
+
+**Note on opencv**: `mediapipe` has a dependency on `opencv` and installing `opencv` is a bit tricky since it depends directly on the system packages. So we the trick the system by installing it right from inside our script even before `mediapipe` is imported, this may seem like a hack but it is **99% solution that works 99% of times**.
+
+## Serve
+
+The class is defined in `model.py` file and to serve this model run:
+```
+nbx serve upload model:MediaPipeModel 'mediapipe_model'
+```
+
+The way `nbox.Operator` works is that it would take all the functions that you have in a class and create an endpoint against it, in this case:
+
+- `predict` would be served at `method_predict_rest/`, this takes in a raw array and returns predictions
+- `predict_b64` would be served at `method_predict_b64_rest/`, this takes in a base64 encoded image
+- `predict_url` would be served at `method_predict_url_rest/`, this takes in a URL
+
+The developer is free from writing API endpoints, managing the complexity of on-wire protocols, they simply write functions that can take in any input (for REST it needs to be JSON serialisable).
+
+## Use
+
+The model is now deployed on an API endpoint that looks like this: `https://api.nimblebox.ai/cdlmonrl/`, you can go to the Deploy → 'mediapipe_model' → Settings and get your access key, it would look like this: `nbxdeploy_AZqcVWuVm0pC4k567EaUjOCOulZiQ3YdLEQJNnrR`. The file `predict.py` contains more detailed tests for the API endpoint. Here's from my run:
+
+```
+Time taken for array (avg. 10 calls): 9.3824s
+Time taken for b64 (avg. 20 calls): 1.2814s
+Time taken for url (avg. 50 calls): 0.4145s
+```
+
+## Advanced
+
+The `nbox.Operator` is designed to wrap any arbitrary python class or function to become part of a distributed compute fabric. When you have deployed a model on NBX-Deploy you can connect directly via an `Operator` with `.from_serving` classmethod like:
+
+```
+mediapipe = Operator.from_serving("https://api.nimblebox.ai/cdlmonrl/", "<token>")
+out = mediapipe.predict_url(url)
+```
+
+To test it run file:
+```
+python3 advanced.py
+```

posemodel/advanced.py

@@ -0,0 +1,6 @@
+from nbox import Operator
+url = "https://i0.wp.com/post.healthline.com/wp-content/uploads/2020/01/Runner-training-on-running-track-1296x728-header-1296x728.jpg?w=1155&h=1528"
+
+mediapipe = Operator.from_serving("https://api.nimblebox.ai/cdlmonrl/", "<token>")
+out = mediapipe.predict_url(url)
+print(out)

posemodel/model.py

@@ -0,0 +1,54 @@
+import subprocess
+from typing import List
+subprocess.call(['/model/venv/bin/python3', '-m', 'pip', 'install', 'opencv-python-headless'])
+
+import requests
+from io import BytesIO
+import itertools
+import numpy as np
+import mediapipe as mp
+
+import base64
+from PIL import Image
+
+from nbox import operator
+
+@operator()
+class MediaPipeModel():
+  def __init__(self):
+    mp_pose = mp.solutions.pose
+    self.pose = mp_pose.Pose(
+      static_image_mode=True, model_complexity=2, min_detection_confidence=0.8
+    )
+
+  def predict(self, image_array):
+    mp_pose = mp.solutions.pose
+    landmarks = [
+      "LEFT_ANKLE","LEFT_EAR","LEFT_ELBOW","LEFT_EYE","LEFT_EYE_INNER","LEFT_EYE_OUTER","LEFT_FOOT_INDEX",
+      "LEFT_HEEL","LEFT_HIP","LEFT_INDEX","LEFT_KNEE","LEFT_PINKY","LEFT_SHOULDER","LEFT_THUMB","LEFT_WRIST",
+      "MOUTH_LEFT","MOUTH_RIGHT","NOSE","RIGHT_ANKLE","RIGHT_EAR","RIGHT_ELBOW","RIGHT_EYE","RIGHT_EYE_INNER",
+      "RIGHT_EYE_OUTER","RIGHT_FOOT_INDEX","RIGHT_HEEL","RIGHT_HIP","RIGHT_INDEX","RIGHT_KNEE","RIGHT_PINKY",
+      "RIGHT_SHOULDER","RIGHT_THUMB","RIGHT_WRIST",
+    ]
+    coordinates = ["x", "y", "z", "visibility"]
+
+    data = {}
+    image = np.array(image_array).astype(np.uint8)
+    image_height, image_width, _ = image.shape
+    data["image_width"] = image_width
+    data["image_height"] = image_height
+    results = self.pose.process(image)
+    if results.pose_landmarks:
+      for l, c in itertools.product(landmarks, coordinates):
+        data[f"{l}_{c}"] = results.pose_landmarks.landmark[mp_pose.PoseLandmark[l]].__getattribute__(c)
+    return {"pred": data}
+
+  def predict_b64(self, image_b64: str, shape: List[int]):
+    img = Image.frombytes("RGB", shape, base64.b64decode(image_b64))
+    image_array = np.array(img, dtype = np.uint8).reshape(*shape, 3)
+    return self.predict(image_array)
+
+  def predict_url(self, url: str):
+    r = requests.get(url)
+    img = Image.open(BytesIO(r.content))
+    return self.predict(np.array(img))

posemodel/predict.py

@@ -0,0 +1,73 @@
+import time
+import requests
+import numpy as np
+from PIL import Image
+from base64 import b64encode
+from io import BytesIO
+
+r = requests.get(
+  "https://i0.wp.com/post.healthline.com/wp-content/uploads/2020/01/Runner-training-on-running-track-1296x728-header-1296x728.jpg?w=1155&h=1528"
+)
+img = Image.open(BytesIO(r.content))
+_shape = np.array(img).shape[:2]
+
+times = []
+n = 10
+print("Testing predict_rest")
+for _ in range(n):
+  st = time.time()
+  r = requests.post(
+    "https://api.nimblebox.ai/cdlmonrl//method_predict_rest",
+    headers = {"NBX-KEY": "<token>"},
+    json = {
+      "image_array": np.array(img).tolist()
+    }
+  )
+  r.raise_for_status()
+  et = time.time()
+  times.append(et - st)
+
+_mt = np.mean(times)
+print(f"Time taken for array (avg. {n} calls): {_mt:0.4f}s")
+# print(r.json())
+
+
+times = []
+n = 20
+print("Testing predict_b64")
+for _ in range(n):
+  st = time.time()
+  r = requests.post(
+    "https://api.nimblebox.ai/cdlmonrl//method_predict_b64_rest",
+    headers = {"NBX-KEY": "<token>"},
+    json = {
+      "image_b64": b64encode(img.tobytes()).decode("utf-8"),
+      "shape": _shape
+    }
+  )
+  r.raise_for_status()
+  et = time.time()
+  times.append(et - st)
+
+_mt = np.mean(times)
+print(f"Time taken for b64 (avg. {n} calls): {_mt:0.4f}s")
+# print(r.json())
+
+times = []
+n = 50
+print("Testing predict_url")
+for _ in range(n):
+  st = time.time()
+  r = requests.post(
+    "https://api.nimblebox.ai/cdlmonrl//method_predict_url_rest",
+    headers = {"NBX-KEY": "<token>"},
+    json = {
+      "url": "https://i0.wp.com/post.healthline.com/wp-content/uploads/2020/01/Runner-training-on-running-track-1296x728-header-1296x728.jpg?w=1155&h=1528"
+    }
+  )
+  r.raise_for_status()
+  et = time.time()
+  times.append(et - st)
+
+_mt = np.mean(times)
+print(f"Time taken for url (avg. {n} calls): {_mt:0.4f}s")

posemodel/requirements.txt

@@ -0,0 +1,9 @@
+nbox[serving]==0.10.6
+numpy
+mediapipe
+
+# installing opencv-python-headless through requirement can cause one of the following errors:
+# AttributeError: partially initialized module 'cv2' has no attribute '_registerMatType' (most likely due to a circular import)
+# ImportError: libGL.so.1: cannot open shared object file: No such file or directory
+
+# pillow