CaaS - test APIs and E2E tests (#2191)

Author: RobertLucian

Makefile

@@ -124,8 +124,7 @@ async-gateway-update:
 	@./dev/registry.sh update-single async-gateway
 	@kubectl delete pods -l cortex.dev/async=gateway --namespace=default
 
-# Docker images
-
+# docker images
 images-all:
 	@./dev/registry.sh update all
 images-all-skip-push:
@@ -136,15 +135,8 @@ images-dev:
 images-dev-skip-push:
 	@./dev/registry.sh update dev --skip-push
 
-images-api:
-	@./dev/registry.sh update api
-images-api-skip-push:
-	@./dev/registry.sh update api --skip-push
-
 images-manager-skip-push:
 	@./dev/registry.sh update-single manager --skip-push
-images-iris:
-	@./dev/registry.sh update-single python-handler-cpu
 
 registry-create:
 	@./dev/registry.sh create
@@ -179,6 +171,12 @@ test-go:
 test-python:
 	@./build/test.sh python
 
+
+# build test api images
+# the DOCKER_PASSWORD and DOCKER_USERNAME vars to the quay repo are required
+build-and-push-test-images:
+	@./test/utils/build-and-push-images.sh quay.io
+
 # run e2e tests on an existing cluster
 # read test/e2e/README.md for instructions first
 test-e2e:

build/build-image.sh

@@ -26,16 +26,4 @@ image=$1
 if [ "$image" == "inferentia" ]; then
   aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 790709498068.dkr.ecr.us-west-2.amazonaws.com
 fi
-
-build_args=""
-
-if [ "${image}" == "python-handler-gpu" ]; then
-  cuda=("10.0" "10.1" "10.1" "10.2" "10.2" "11.0" "11.1")
-  cudnn=("7" "7" "8" "7" "8" "8" "8")
-  for i in ${!cudnn[@]}; do
-    build_args="${build_args} --build-arg CUDA_VERSION=${cuda[$i]} --build-arg CUDNN=${cudnn[$i]}"
-    docker build "$ROOT" -f $ROOT/images/$image/Dockerfile $build_args -t quay.io/cortexlabs/${image}:${CORTEX_VERSION}-cuda${cuda[$i]}-cudnn${cudnn[$i]} -t cortexlabs/${image}:${CORTEX_VERSION}-cuda${cuda[$i]}-cudnn${cudnn[$i]}
-  done
-else
-  docker build "$ROOT" -f $ROOT/images/$image/Dockerfile $build_args -t quay.io/cortexlabs/${image}:${CORTEX_VERSION} -t cortexlabs/${image}:${CORTEX_VERSION}
-fi
+docker build "$ROOT" -f $ROOT/images/$image/Dockerfile -t quay.io/cortexlabs/${image}:${CORTEX_VERSION} -t cortexlabs/${image}:${CORTEX_VERSION}

build/images.sh

@@ -19,13 +19,6 @@
 
 set -euo pipefail
 
-api_images=(
-  "python-handler-cpu"
-  "python-handler-gpu"
-  "tensorflow-handler"
-  "python-handler-inf"
-)
-
 dev_images=(
   "manager"
   "proxy"
@@ -61,7 +54,6 @@ non_dev_images=(
 )
 
 all_images=(
-  "${api_images[@]}"
   "${dev_images[@]}"
   "${non_dev_images[@]}"
 )

build/push-image.sh

@@ -23,13 +23,4 @@ host=$1
 image=$2
 
 echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
-
-if [ "$image" == "python-handler-gpu" ]; then
-  cuda=("10.0" "10.1" "10.1" "10.2" "10.2" "11.0" "11.1")
-  cudnn=("7" "7" "8" "7" "8" "8" "8")
-  for i in ${!cudnn[@]}; do
-    docker push $host/cortexlabs/${image}:${CORTEX_VERSION}-cuda${cuda[$i]}-cudnn${cudnn[$i]}
-  done
-else
-  docker push $host/cortexlabs/${image}:${CORTEX_VERSION}
-fi
+docker push $host/cortexlabs/${image}:${CORTEX_VERSION}

dev/registry.sh

@@ -108,15 +108,13 @@ function build() {
   local tag=$2
   local dir="${ROOT}/images/${image}"
 
-  build_args=""
-
   tag_args=""
   if [ -n "$AWS_ACCOUNT_ID" ] && [ -n "$AWS_REGION" ]; then
     tag_args+=" -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/cortexlabs/$image:$tag"
   fi
 
   blue_echo "Building $image:$tag..."
-  docker build $ROOT -f $dir/Dockerfile -t cortexlabs/$image:$tag $tag_args $build_args
+  docker build $ROOT -f $dir/Dockerfile -t cortexlabs/$image:$tag $tag_args
   green_echo "Built $image:$tag\n"
 }
 
@@ -150,10 +148,6 @@ function build_and_push() {
   set -euo pipefail  # necessary since this is called in a new shell by parallel
 
   tag=$CORTEX_VERSION
-  if [ "${image}" == "python-handler-gpu" ]; then
-    tag="${CORTEX_VERSION}-cuda10.2-cudnn8"
-  fi
-
   build $image $tag
   push $image $tag
 }
@@ -240,8 +234,6 @@ elif [ "$cmd" = "update" ]; then
     images_to_build+=( "${dev_images[@]}" )
   fi
 
-  images_to_build+=( "${api_images[@]}" )
-
   if [[ " ${images_to_build[@]} " =~ " operator " ]]; then
     cache_builder operator
   fi

test/apis/README.md

@@ -0,0 +1,9 @@
+*.dockerfile
+README.md
+sample.json
+expectations.yaml
+*.pyc
+*.pyo
+*.pyd
+__pycache__
+.pytest_cache

test/apis/async/iris-classifier/cortex.yaml

@@ -0,0 +1,16 @@
+- name: text-generator
+  kind: AsyncAPI
+  pod:
+    port: 9000
+    containers:
+    - name: api
+      image: quay.io/cortexlabs-test/async-text-generator-cpu:latest
+      readiness_probe:
+        http_get:
+          path: "/healthz"
+          port: 9000
+      compute:
+        cpu: 1
+        mem: 2.5G
+  autoscaling:
+    max_concurrency: 1

test/apis/async/iris-classifier/expectations.yaml

@@ -0,0 +1,19 @@
+- name: text-generator
+  kind: AsyncAPI
+  pod:
+    port: 9000
+    containers:
+    - name: api
+      image: quay.io/cortexlabs-test/async-text-generator-gpu:latest
+      env:
+        TARGET_DEVICE: "cuda"
+      readiness_probe:
+        http_get:
+          path: "/healthz"
+          port: 9000
+      compute:
+        cpu: 1
+        gpu: 1
+        mem: 512M
+  autoscaling:
+    max_concurrency: 1

test/apis/async/iris-classifier/handler.py

@@ -3,8 +3,7 @@ response:
   json_schema:
     type: "object"
     properties:
-      label:
+      prediction:
         type: "string"
-        const: "setosa"
     required:
-      - "label"
+      - "prediction"

test/apis/async/iris-classifier/requirements.txt

@@ -0,0 +1,42 @@
+import os
+
+from fastapi import FastAPI, Response, status
+from pydantic import BaseModel
+from transformers import GPT2Tokenizer, GPT2LMHeadModel
+
+
+class Request(BaseModel):
+    text: str
+
+
+state = {
+    "model_ready": False,
+    "tokenizer": None,
+    "model": None,
+}
+device = os.getenv("TARGET_DEVICE", "cpu")
+app = FastAPI()
+
+
+@app.on_event("startup")
+def startup():
+    global state
+    state["tokenizer"] = GPT2Tokenizer.from_pretrained("gpt2")
+    state["model"] = GPT2LMHeadModel.from_pretrained("gpt2").to(device)
+    state["model_ready"] = True
+
+
+@app.get("/healthz")
+def healthz(response: Response):
+    if not state["model_ready"]:
+        response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE
+
+
+@app.post("/")
+def text_generator(request: Request):
+    input_length = len(request.text.split())
+    tokens = state["tokenizer"].encode(request.text, return_tensors="pt").to(device)
+    prediction = state["model"].generate(tokens, max_length=input_length + 20, do_sample=True)
+    return {
+        "prediction": state["tokenizer"].decode(prediction[0]),
+    }

test/apis/async/iris-classifier/sample.json

@@ -0,0 +1,3 @@
+{
+    "text": "machine learning is"
+}