Add YoloV4 inference server

.gitignore

@@ -13,6 +13,7 @@ env
 .testvenv
 python_env
 .venv
+venv/
 
 # persistent storage volume
 persistent_volume
@@ -41,3 +42,4 @@ db.sqlite3
 # unignore
 !requirements.txt
 !requirements-dev.txt
+!requirements-test.txt

scripts/add_spdx_header.py

@@ -32,6 +32,7 @@ def add_spdx_header(file_path):
         repo_root / "tt-metal-llama3-70b",
         repo_root / "vllm-tt-metal-llama3-70b",
         repo_root / "tt-metal-mistral-7b",
+        repo_root / "tt-metal-yolov4",
         repo_root / "tests",
         repo_root / "utils",
         repo_root / "evals",

tt-metal-yolov4/.env.default

@@ -0,0 +1,4 @@
+TT_METAL_DOCKERFILE_VERSION=v0.53.0-rc34
+TT_METAL_COMMIT_SHA_OR_TAG=65d246482b3fd821d383a4aa2814f1de5392f417
+TT_METAL_COMMIT_DOCKER_TAG=65d246482b3f  # 12-character version of TT_METAL_COMMIT_SHA_OR_TAG
+IMAGE_VERSION=v0.0.1

tt-metal-yolov4/README.md

@@ -0,0 +1,41 @@
+# TT Metalium YOLOv4 Inference API
+
+This implementation supports YOLOv4 execution on Grayskull and Worhmole.
+
+
+## Table of Contents
+- [Run server](#run-server)
+- [Development](#development)
+- [Tests](#tests)
+
+
+## Run server
+To run the YOLOv4 inference server, run the following command from the project root at `tt-inference-server`:
+```bash
+cd tt-inference-server
+docker compose --env-file tt-metal-yolov4/.env.default -f tt-metal-yolov4/docker-compose.yaml up --build
+```
+
+This will start the default Docker container with the entrypoint command set to `server/run_uvicorn.sh`. The next section describes how to override the container's default command with an interractive shell via `bash`.
+
+
+## Development
+Inside the container you can then start the server with:
+```bash
+docker compose --env-file tt-metal-yolov4/.env.default -f tt-metal-yolov4/docker-compose.yaml run --rm --build inference_server /bin/bash
+```
+
+Inside the container, run `cd ~/app/server` to navigate to the server implementation.
+
+
+## Tests
+Tests can be found in `tests/`. The tests have their own dependencies found in `requirements-test.txt`.
+
+To load test the server, we use `locust` to simulate a single client sending an infinite-FPS video stream to the server for 1 minute.
+This yields a server performance ceiling of ~25FPS. First, ensure the server is running (see [how to run the server](#run-server)). Then in a different shell with the base dev `venv` activated:
+```bash
+cd tt-metal-yolov4
+pip install -r requirements-test.txt
+cd tests/
+locust --config locust_config.conf
+```

tt-metal-yolov4/docker-compose.yaml

@@ -0,0 +1,26 @@
+services:
+  inference_server:
+    image: ghcr.io/tenstorrent/tt-inference-server/tt-metal-yolov4-src-base:${IMAGE_VERSION}-tt-metal-${TT_METAL_COMMIT_DOCKER_TAG}
+    build:
+      context: .
+      dockerfile: yolov4.src.Dockerfile
+      args:
+        TT_METAL_DOCKERFILE_VERSION: ${TT_METAL_DOCKERFILE_VERSION}
+        TT_METAL_COMMIT_SHA_OR_TAG: ${TT_METAL_COMMIT_SHA_OR_TAG}
+    container_name: inference_server
+    ports:
+      - "7000:7000"
+    devices:
+      - "/dev/tenstorrent:/dev/tenstorrent"
+    volumes:
+      - "/dev/hugepages-1G/:/dev/hugepages-1G:rw"
+    shm_size: "32G"
+    cap_add:
+      - ALL
+    stdin_open: true
+    tty: true
+    # this is redundant as docker compose automatically uses the .env file as its in the same directory
+    # but this explicitly demonstrates its usage
+    env_file:
+      - .env.default
+    restart: no

tt-metal-yolov4/requirements-test.txt

@@ -0,0 +1,2 @@
+pillow==10.3.0
+locust==2.25.0

tt-metal-yolov4/requirements.txt

@@ -0,0 +1,6 @@
+# inference server requirements
+fastapi==0.85.1
+uvicorn==0.19.0
+python-multipart==0.0.5
+
+-f https://download.pytorch.org/whl/cpu/torch_stable.html

tt-metal-yolov4/server/coco.names

@@ -0,0 +1,80 @@
+person
+bicycle
+car
+motorbike
+aeroplane
+bus
+train
+truck
+boat
+traffic light
+fire hydrant
+stop sign
+parking meter
+bench
+bird
+cat
+dog
+horse
+sheep
+cow
+elephant
+bear
+zebra
+giraffe
+backpack
+umbrella
+handbag
+tie
+suitcase
+frisbee
+skis
+snowboard
+sports ball
+kite
+baseball bat
+baseball glove
+skateboard
+surfboard
+tennis racket
+bottle
+wine glass
+cup
+fork
+knife
+spoon
+bowl
+banana
+apple
+sandwich
+orange
+broccoli
+carrot
+hot dog
+pizza
+donut
+cake
+chair
+sofa
+pottedplant
+bed
+diningtable
+toilet
+tvmonitor
+laptop
+mouse
+remote
+keyboard
+cell phone
+microwave
+oven
+toaster
+sink
+refrigerator
+book
+clock
+vase
+scissors
+teddy bear
+hair drier
+toothbrush