update model inference.

Author: fedml-alex

python/fedml/cli/model_deployment/device_model_deployment.py

@@ -4,7 +4,7 @@
 import platform
 import time
 
-import psutil
+import numpy as np
 import requests
 import tritonclient.http as http_client
 
@@ -116,7 +116,7 @@ def should_exit_logs(cmd_type, cmd_process_id, model_name, inference_engine, inf
                 get_model_info(model_name, inference_engine, inference_port)
             logging.info("Log test for deploying model successfully, inference url: {}, "
                          "model metadata: {}, model config: {}".format(
-                inference_output_url, model_metadata, model_config))
+                          inference_output_url, model_metadata, model_config))
             if inference_output_url != "":
                 return True
         except Exception as e:
@@ -152,7 +152,7 @@ def log_deployment_result(cmd_container_name, cmd_type, cmd_process_id, inferenc
                 logging.info("{}".format(added_logs))
             last_err_logs = err_str
 
-        time.sleep(5)
+        time.sleep(3)
 
         if should_exit_logs(cmd_type, cmd_process_id, inference_model_name, inference_engine, inference_http_port):
             break
@@ -185,11 +185,11 @@ def get_model_info(model_name, inference_engine, inference_http_port, infer_host
         infer_url_host = infer_host
     else:
         infer_url_host = local_ip
-    inference_output_url = "{}:{}/{}/models/{}/versions/{}/infer".format(infer_url_host,
-                                                                         inference_http_port,
-                                                                         ClientConstants.INFERENCE_INFERENCE_SERVER_VERSION,
-                                                                         inference_model_name,
-                                                                         model_version)
+    inference_output_url = "http://{}:{}/{}/models/{}/versions/{}/infer".format(infer_url_host,
+                                                                                inference_http_port,
+                                                                                ClientConstants.INFERENCE_INFERENCE_SERVER_VERSION,
+                                                                                inference_model_name,
+                                                                                model_version)
 
     return inference_output_url, model_version, model_metadata, model_config
 
@@ -213,14 +213,18 @@ def run_http_inference_with_lib_http_api(model_name, inference_http_port, batch_
     model_metadata = triton_client.get_model_metadata(model_name=inference_model_name, model_version=model_version)
     model_config = triton_client.get_model_config(model_name=inference_model_name, model_version=model_version)
 
-    inference_output_sample = {}
+    print("model metadata {}".format(model_metadata))
+    inference_response_list = list()
     inference_input_list = model_metadata["inputs"]
     infer_item_count = 0
     inference_query_list = []
+
+    input_data_np = np.asarray(inference_input_data_list * batch_size, dtype=object)
+
     for infer_input_item in inference_input_list:
         query_item = http_client.InferInput(name=infer_input_item["name"],
-                                            shape=(batch_size,), datatype=infer_input_item["data_type"])
-        query_item.set_data_from_numpy(inference_input_data_list[infer_item_count])
+                                            shape=(batch_size,), datatype=infer_input_item["datatype"])
+        query_item.set_data_from_numpy(input_data_np)
         inference_query_list.append(query_item)
         infer_item_count += 1
 
@@ -238,9 +242,13 @@ def run_http_inference_with_lib_http_api(model_name, inference_http_port, batch_
     )
 
     for infer_output_item in inference_output_list:
-        inference_output_sample[infer_output_item["name"]] = response.as_numpy(infer_output_item["name"])
+        response_item = response.get_output(infer_output_item["name"])
+        inference_response_list.append(response_item)
+        print("response item {}".format(response_item))
 
-    return inference_output_sample
+    inference_response_dict = {"outputs": inference_response_list}
+    print("return {}".format(inference_response_dict))
+    return inference_response_dict
 
 
 def run_http_inference_with_raw_http_request(self, inference_input_json, inference_input_data_list):

python/fedml/cli/model_deployment/device_model_inference.py

@@ -2,10 +2,10 @@
 from urllib.parse import urlparse
 
 from fastapi import FastAPI, Request
-from fedml.cli.model_deployment.device_model_deployment import run_http_inference_with_lib_http_api
+from fedml.cli.model_deployment.device_model_deployment import run_http_inference_with_lib_http_api, \
+    run_http_inference_with_raw_http_request
 from fedml.cli.model_deployment.device_client_constants import ClientConstants
 from fedml.cli.model_deployment.device_server_constants import ServerConstants
-from fedml.cli.model_deployment.device_server_runner import FedMLServerRunner
 from fedml.cli.model_deployment.device_model_monitor import FedMLModelMetrics
 from fedml.cli.model_deployment.device_model_cache import FedMLModelCache
 
@@ -49,8 +49,11 @@ async def predict(request: Request):
     # Send inference request to idle device
     inference_response = {}
     if inference_output_url != "":
+        input_data = input_json.get("data", "SampleData")
+        input_data_list = list()
+        input_data_list.append(str(input_data))
         inference_response = send_inference_request(idle_device, model_name, inference_host,
-                                                    inference_output_url, input_json, input_json)
+                                                    inference_output_url, input_json, input_data_list)
 
     model_metrics.calc_metrics(model_id, model_name, infer_end_point_id, inference_output_url)
 
@@ -79,9 +82,11 @@ def found_idle_inference_device(end_point_id):
     return idle_device, model_id, model_name, inference_host, inference_output_url
 
 
-def send_inference_request(device, model_name, inference_host, inference_url, json_req, bin_data=None):
+def send_inference_request(device, model_name, inference_host, inference_url, json_req, input_data_list=None):
     inference_response = run_http_inference_with_lib_http_api(model_name,
-                                                              ClientConstants.INFERENCE_HTTP_PORT, 1, bin_data,
+                                                              ClientConstants.INFERENCE_HTTP_PORT,
+                                                              1,
+                                                              input_data_list,
                                                               inference_host)
     return inference_response