Merge pull request #685 from alex-liang-kh/dev/v0.7.0

Dev/v0.7.0

Author: fedml-alex

python/fedml/cli/cli.py

@@ -717,6 +717,10 @@ def device():
 @click.option(
     "--master", "-m", default=None, is_flag=True, help="login as master device in the federated inference cluster.",
 )
+@click.option(
+    "--infer_host", "-ih", type=str, default="127.0.0.1",
+    help="used this ip address or domain name as inference host.",
+)
 @click.option(
     "--version",
     "-v",
@@ -751,9 +755,9 @@ def device():
     "--docker-rank", "-dr", default="1", help="docker client rank index (from 1 to n).",
 )
 def login_as_model_device_agent(
-        userid, cloud, on_premise, master, version, local_server, runner_cmd, device_id, os_name, docker, docker_rank
+        userid, cloud, on_premise, master, infer_host, version, local_server, runner_cmd, device_id, os_name, docker, docker_rank
 ):
-    device_login_entry.login_as_model_device_agent(userid, cloud, on_premise, master, version, local_server, runner_cmd, device_id, os_name, docker, docker_rank)
+    device_login_entry.login_as_model_device_agent(userid, cloud, on_premise, master, infer_host, version, local_server, runner_cmd, device_id, os_name, docker, docker_rank)
 
 
 @device.command("logout", help="Logout from the ModelOps platform (model.fedml.ai)")

python/fedml/cli/model_deployment/device_client_constants.py

@@ -148,6 +148,28 @@ def get_local_ip():
         s.close()
         return ip
 
+    @staticmethod
+    def check_network_port_is_opened(port):
+        import socket
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        try:
+            s.connect(('localhost', int(port)))
+            s.settimeout(1)
+            s.shutdown(2)
+            return True
+        except:
+            return False
+
+    @staticmethod
+    def check_process_is_running(process_id):
+        for proc in psutil.process_iter():
+            try:
+                if process_id == proc.pid:
+                    return True
+            except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
+                pass
+        return False
+
     @staticmethod
     def unzip_file(zip_file, unzip_file_path):
         result = False
@@ -310,14 +332,20 @@ def exit_process(process):
             pass
 
     @staticmethod
-    def exec_console_with_script(script_path, should_capture_stdout=False, should_capture_stderr=False):
+    def exec_console_with_script(script_path, should_capture_stdout=False, should_capture_stderr=False, no_sys_out_err=False):
         stdout_flag = subprocess.PIPE if should_capture_stdout else sys.stdout
         stderr_flag = subprocess.PIPE if should_capture_stderr else sys.stderr
 
         if platform.system() == 'Windows':
-            script_process = subprocess.Popen(script_path, stdout=stdout_flag, stderr=stderr_flag)
+            if no_sys_out_err:
+                script_process = subprocess.Popen(script_path)
+            else:
+                script_process = subprocess.Popen(script_path, stdout=stdout_flag, stderr=stderr_flag)
         else:
-            script_process = subprocess.Popen(['bash', '-c', script_path], stdout=stdout_flag, stderr=stderr_flag)
+            if no_sys_out_err:
+                script_process = subprocess.Popen(['bash', '-c', script_path])
+            else:
+                script_process = subprocess.Popen(['bash', '-c', script_path], stdout=stdout_flag, stderr=stderr_flag)
 
         return script_process
 

python/fedml/cli/model_deployment/device_client_daemon.py

@@ -17,6 +17,7 @@
     parser.add_argument("--role", "-r", type=str, default="client")
     parser.add_argument("--device_id", "-id", type=str, default="0")
     parser.add_argument("--os_name", "-os", type=str, default="")
+    parser.add_argument("--infer_host", "-ih", type=str, default="127.0.0.1")
     args = parser.parse_args()
     args.user = args.user
 
@@ -40,7 +41,9 @@
                 "-id",
                 args.device_id,
                 "-os",
-                args.os_name
+                args.os_name,
+                "-ih",
+                args.infer_host
             ]
         )
         ret_code, exec_out, exec_err = ClientConstants.get_console_sys_out_pipe_err_results(login_pid)

python/fedml/cli/model_deployment/device_client_login.py

@@ -138,6 +138,7 @@ def __login_as_client(args, userid, version):
     ClientConstants.save_runner_infos(args.current_device_id + "." + args.os_name, edge_id, run_id=0)
 
     # Setup MQTT connection for communication with the FedML server.
+    runner.infer_host = args.infer_host
     runner.setup_agent_mqtt_connection(service_config)
 
     # Start mqtt looper
@@ -161,6 +162,7 @@ def logout():
     parser.add_argument("--role", "-r", type=str, default="md.on_premise_device")
     parser.add_argument("--device_id", "-id", type=str, default="0")
     parser.add_argument("--os_name", "-os", type=str, default="")
+    parser.add_argument("--infer_host", "-ih", type=str, default="127.0.0.1")
     args = parser.parse_args()
     args.user = args.user
     if args.type == 'login':

python/fedml/cli/model_deployment/device_client_runner.py

@@ -69,7 +69,7 @@ def __init__(self, args, edge_id=0, request_json=None, agent_config=None, run_id
 
         self.mlops_metrics = None
         self.client_active_list = dict()
-        # logging.info("Current directory of client agent: " + self.cur_dir)
+        self.infer_host = "127.0.0.1"
 
     def unzip_file(self, zip_file, unzip_file_path):
         result = False
@@ -178,7 +178,8 @@ def run(self):
             ClientConstants.INFERENCE_METRIC_PORT,
             use_gpu, memory_size,
             ClientConstants.INFERENCE_CONVERTOR_IMAGE,
-            ClientConstants.INFERENCE_SERVER_IMAGE)
+            ClientConstants.INFERENCE_SERVER_IMAGE,
+            self.infer_host)
         if inference_output_url == "":
             self.setup_client_mqtt_mgr()
             self.wait_client_mqtt_connected()
@@ -472,6 +473,7 @@ def callback_start_deployment(self, topic, payload):
         client_runner = FedMLClientRunner(
             self.args, edge_id=self.edge_id, request_json=request_json, agent_config=self.agent_config, run_id=run_id
         )
+        client_runner.infer_host = self.infer_host
         self.process = Process(target=client_runner.run)
         #client_runner.run()
         self.process.start()

python/fedml/cli/model_deployment/device_login_entry.py

@@ -22,7 +22,7 @@
 
 
 def login_as_model_device_agent(
-    userid, cloud, on_premise, master, version, local_server, runner_cmd, device_id, os_name, docker, docker_rank
+    userid, cloud, on_premise, master, infer_host, version, local_server, runner_cmd, device_id, os_name, docker, docker_rank
 ):
     account_id = userid[0]
     platform_url = "open.fedml.ai"
@@ -91,7 +91,9 @@ def login_as_model_device_agent(
                 "-id",
                 device_id,
                 "-os",
-                os_name
+                os_name,
+                "-ih",
+                infer_host
             ]
         ).pid
         sys_utils.save_login_process(ClientConstants.LOCAL_HOME_RUNNER_DIR_NAME,

python/fedml/cli/model_deployment/device_model_cards.py

@@ -174,6 +174,7 @@ def push_model(self, model_name, user_id, no_uploading_modelops=False):
             return "", ""
 
         model_storage_url = self.push_model_to_s3(model_name, model_zip_path, user_id)
+        print("Model storage url: {}".format(model_storage_url))
         if not no_uploading_modelops:
             if model_storage_url != "":
                 upload_result = self.upload_model_api(model_name, model_storage_url, user_id)