Tested apis; Added benchmark results to README

README.md

@@ -100,7 +100,19 @@ the user has full control of what they want to run on the pool.
 
 --- 
 
-### Prod results / Benchmarks
+### Benchmarks
 
-For more examples see /examples
+- APIs Fake Load: sync (examples/sync_api.py) VS pool based (examples/ml_pool_api.py)
 
+1. 1 uvicorn worker, 10 concurrent clients, 50 requests / client, 10M CPU burn cycles (imitates model scoring)
+
+```
+sync - 338 seconds
+ml_pool - 84 seconds (11 workers)
+```
+
+2. 1 uvicorn worker, 20 concurrent clients, 50 requests / client, 10M CPU burn cycles
+```
+sync - 657 seconds (1.5 requests / s)
+ml_pool - 143 seconds (11 workers) (7 requests/s)
+```

examples/load_test_api.py

@@ -0,0 +1,39 @@
+import sys
+
+sys.path.append("..")
+
+import requests
+import threading
+
+from ml_pool.utils import timer
+
+
+URL = "http://127.0.0.1:8000/iris"
+CLIENTS = 20
+REQUESTS_PER_CLIENT = 50
+
+
+def client(index, features):
+    for i in range(REQUESTS_PER_CLIENT):
+        response = requests.post(url=URL, json={"features": features})
+        print(
+            f"Client {index} got {i} / {REQUESTS_PER_CLIENT} "
+            f"response {response.json()}"
+        )
+
+
+@timer
+def main():
+    threads = [
+        threading.Thread(target=client, args=(i, [6.2, 2.2, 4.5, 1.5]))
+        for i in range(CLIENTS)
+    ]
+    for thread in threads:
+        thread.start()
+
+    for thread in threads:
+        thread.join()
+
+
+if __name__ == "__main__":
+    main()

examples/ml_pool_api.py

@@ -0,0 +1,65 @@
+import sys
+
+sys.path.append("..")
+
+from functools import partial
+
+from fastapi import FastAPI
+import pydantic
+import xgboost
+import numpy as np
+import uvicorn
+
+from ml_pool import MLPool
+from ml_pool.logger import get_logger
+
+
+logger = get_logger("api")
+
+app = FastAPI()
+
+
+def load_model(model_path: str):
+    model = xgboost.Booster()
+    model.load_model(model_path)
+    return model
+
+
+def score_model(model, features):
+    # Imitates a heavy model that takes time to score + feature engineering
+    # could also be unloaded to the worker pool
+    sum_ = 0
+    for i in range(10_000_000):
+        sum_ += 1
+
+    features = xgboost.DMatrix([features])
+    return np.argmax(model.predict(features))
+
+
+class Request(pydantic.BaseModel):
+    features: list[float]
+
+
+class Response(pydantic.BaseModel):
+    prediction: int
+
+
+@app.get("/")
+def health_check():
+    return {"Message": "Up and running"}
+
+
+@app.post("/iris")
+def score(request: Request) -> Response:
+    logger.info(f"Got request for features: {request}")
+    job_id = pool.schedule_model_scoring(features=request.features)
+    result = pool.get_scoring_result(job_id, wait_if_not_available=True)
+    return Response(prediction=result)
+
+
+if __name__ == "__main__":
+    with MLPool(
+        load_model_func=partial(load_model, "iris_xgb.json"),
+        score_model_func=score_model,
+    ) as pool:
+        uvicorn.run(app, workers=1)

examples/sync_api.py

@@ -0,0 +1,61 @@
+import sys
+
+sys.path.append("..")
+
+from fastapi import FastAPI
+import pydantic
+import xgboost
+import numpy as np
+import uvicorn
+
+from ml_pool.logger import get_logger
+
+
+logger = get_logger("api")
+
+app = FastAPI()
+
+
+def load_model(model_path: str):
+    model = xgboost.Booster()
+    model.load_model(model_path)
+    logger.info("Model loaded")
+    return model
+
+
+model = load_model("iris_xgb.json")
+
+
+def score_model(model, features):
+    # Imitates a heavy model that takes time to score + feature engineering
+    # could also be unloaded to the worker pool
+    sum_ = 0
+    for i in range(10_000_000):
+        sum_ += 1
+
+    features = xgboost.DMatrix([features])
+    return np.argmax(model.predict(features))
+
+
+class Request(pydantic.BaseModel):
+    features: list[float]
+
+
+class Response(pydantic.BaseModel):
+    prediction: int
+
+
+@app.get("/")
+def health_check():
+    return {"Message": "Up and running"}
+
+
+@app.post("/iris")
+def score(request: Request) -> Response:
+    logger.info(f"Got request for features: {request}")
+    result = score_model(model, request.features)
+    return Response(prediction=result)
+
+
+if __name__ == "__main__":
+    uvicorn.run(app, workers=1)

ml_pool/config.py

@@ -2,7 +2,7 @@
 
 
 class Config:
-    LOGGER_VERBOSE = False
+    LOGGER_VERBOSE = True
     LOGGER_FORMAT = (
         "%(name)s %(process)d %(levelname)s %(lineno)s: %(message)s"
     )