fix styles for 'develop ng' branch

intel · May 27, 2020 · 53ef6d2 · 53ef6d2
1 parent 7236094
commit 53ef6d2
Show file tree

Hide file tree

Showing 33 changed files with 417 additions and 453 deletions.
diff --git a/Makefile b/Makefile
@@ -27,7 +27,7 @@ all: venv
 # we need to update pip and setuptools because venv versions aren't latest
 # need to prepend $(ACTIVATE) everywhere because all make calls are in subshells
 # otherwise we won't be installing anything in the venv itself
-$(ACTIVATE): tests/requirements.txt
+$(ACTIVATE):
 	@echo "Updating virtualenv dependencies in: $(VIRTUALENV_DIR)..."
 	@test -d $(VIRTUALENV_DIR) || $(VIRTUALENV_EXE) $(VIRTUALENV_DIR)
 	@. $(ACTIVATE) && python -m pip install -r requirements-test.txt

diff --git a/benchmarks/common/base_benchmark_util.py b/benchmarks/common/base_benchmark_util.py
@@ -168,12 +168,11 @@ def _define_args(self):
                  "This option can be used by models (like SSD_Resnet34) "
                  "to do fine-tuning training or achieve convergence.",
             dest="backbone_model", default=None, type=check_valid_folder)
-        
+
         self._common_arg_parser.add_argument(
             "-g", "--in-graph", help="Full path to the input graph ",
             dest="input_graph", default=None, type=check_valid_filename)
 
-
         self._common_arg_parser.add_argument(
             "-k", "--benchmark-only",
             help="For benchmark measurement only. If neither --benchmark-only "

diff --git a/benchmarks/common/base_model_init.py b/benchmarks/common/base_model_init.py
@@ -58,31 +58,39 @@ def __init__(self, args, custom_args=[], platform_util=None):
             raise ValueError("Did not find any platform info.")
 
         # use case: bare-metal with openmpi, horovod and multi-node
-        if os.environ["MPI_HOSTNAMES"] != "None" and (not "DOCKER" in os.environ or os.environ["DOCKER"] == "False"):
-          if os.environ["MPI_NUM_PROCESSES"] != "None":
-            try:
-              # slots per host calculation using MPI_NUM_PROCESSES and number of hosts
-              host_names = os.environ["MPI_HOSTNAMES"]
-              number_of_hosts = len(host_names.split(','))
-              slots_per_host = int(int(os.environ["MPI_NUM_PROCESSES"]) / number_of_hosts)
-              host_names = ",".join([ host + ":" + str(slots_per_host) for host in host_names.split(',') ])
-              # see the [examples](https://horovod.readthedocs.io/en/latest/mpirun.html) for the mca flags
-              self.python_exe = "mpirun " + " -x LD_LIBRARY_PATH " + " -x PYTHONPATH " + " --allow-run-as-root -n " + os.environ["MPI_NUM_PROCESSES"] + " -H " + host_names + " -mca pml ob1 -mca btl ^openib -mca btl_tcp_if_exclude lo,docker0 --bind-to none --map-by slot " + self.python_exe
-            except Exception as exception:
-              raise ValueError("Caught exception calculating slots per host {}".format(str(exception)))
-          else:
-            raise ValueError("MPI_NUM_PROCESSES is required for MPI_HOSTNAMES and will be split evenly across the hosts.")
+        if os.environ["MPI_HOSTNAMES"] != "None" and ("DOCKER" not in os.environ or os.environ["DOCKER"] == "False"):
+            if os.environ["MPI_NUM_PROCESSES"] != "None":
+                try:
+                    # slots per host calculation using MPI_NUM_PROCESSES and number of hosts
+                    host_names = os.environ["MPI_HOSTNAMES"]
+                    number_of_hosts = len(host_names.split(','))
+                    slots_per_host = int(int(os.environ["MPI_NUM_PROCESSES"]) / number_of_hosts)
+                    host_names = ",".join([host + ":" + str(slots_per_host) for host in host_names.split(',')])
+                    # see the [examples](https://horovod.readthedocs.io/en/latest/mpirun.html) for the mca flags
+                    self.python_exe = "mpirun " + " -x LD_LIBRARY_PATH " + " -x PYTHONPATH " \
+                        + " --allow-run-as-root -n " + os.environ["MPI_NUM_PROCESSES"] + " -H " + host_names \
+                        + " -mca pml ob1 -mca btl ^openib -mca btl_tcp_if_exclude " \
+                          "lo,docker0 --bind-to none --map-by slot " \
+                        + self.python_exe
+                except Exception as exception:
+                    raise ValueError("Caught exception calculating slots per host {}".format(str(exception)))
+            else:
+                raise ValueError("MPI_NUM_PROCESSES is required for MPI_HOSTNAMES and will be split evenly across the "
+                                 "hosts.")
         # use case: docker with openmpi, single-node, multi-instance
         elif os.environ["MPI_NUM_PROCESSES"] != "None":
             if os.environ["MPI_NUM_PROCESSES_PER_SOCKET"] == "1":
-              # Map by socket using OpenMPI by default (PPS=1).
-              self.python_exe = "mpirun --allow-run-as-root -n " + os.environ["MPI_NUM_PROCESSES"] + " --map-by socket " + self.python_exe
+                # Map by socket using OpenMPI by default (PPS=1).
+                self.python_exe = "mpirun --allow-run-as-root -n " + os.environ["MPI_NUM_PROCESSES"] \
+                    + " --map-by socket " + self.python_exe
             else:
-              # number of processes per socket (pps)
-              pps = int(os.environ["MPI_NUM_PROCESSES_PER_SOCKET"])
-              split_a_socket = str(platform_util.num_cores_per_socket // pps)
-              # Launch pps MPI processes over one socket
-              self.python_exe = "mpirun --allow-run-as-root -n " + os.environ["MPI_NUM_PROCESSES"] + " --map-by ppr:" + str(pps) + ":socket:pe=" + split_a_socket + " --cpus-per-proc " + split_a_socket + " " + self.python_exe
+                # number of processes per socket (pps)
+                pps = int(os.environ["MPI_NUM_PROCESSES_PER_SOCKET"])
+                split_a_socket = str(platform_util.num_cores_per_socket // pps)
+                # Launch pps MPI processes over one socket
+                self.python_exe = "mpirun --allow-run-as-root -n " + os.environ["MPI_NUM_PROCESSES"] \
+                    + " --map-by ppr:" + str(pps) + ":socket:pe=" + split_a_socket + " --cpus-per-proc " \
+                    + split_a_socket + " " + self.python_exe
 
     def run_command(self, cmd):
         """
@@ -182,15 +190,14 @@ def set_num_inter_intra_threads(self, num_inter_threads=None, num_intra_threads=
             if not self.args.num_inter_threads:
                 self.args.num_inter_threads = self.platform_util.num_cpu_sockets
                 if os.environ["MPI_NUM_PROCESSES"] != "None":
-                  self.args.num_inter_threads = 1
+                    self.args.num_inter_threads = 1
             if not self.args.num_intra_threads:
                 if self.args.num_cores == -1:
                     self.args.num_intra_threads = \
                         int(self.platform_util.num_cores_per_socket *
                             self.platform_util.num_cpu_sockets)
                     if os.environ["MPI_NUM_PROCESSES"] != "None":
-                      self.args.num_intra_threads = \
-                             self.platform_util.num_cores_per_socket - 2
+                        self.args.num_intra_threads = self.platform_util.num_cores_per_socket - 2
                 else:
                     self.args.num_intra_threads = self.args.num_cores
 

diff --git a/benchmarks/common/platform_util.py b/benchmarks/common/platform_util.py
@@ -22,7 +22,6 @@
 from __future__ import division
 from __future__ import print_function
 
-import os
 import re
 import platform as system_platform
 import subprocess

diff --git a/benchmarks/image_recognition/tensorflow/mobilenet_v1/inference/fp32/model_init.py b/benchmarks/image_recognition/tensorflow/mobilenet_v1/inference/fp32/model_init.py
@@ -74,7 +74,7 @@ def __init__(self, args, custom_args=[], platform_util=None):
                 self.command_prefix, script_args_list)
 
     def parse_args(self):
-        if self.custom_args == None:
+        if self.custom_args is None:
             return
 
         parser = argparse.ArgumentParser()
@@ -101,8 +101,7 @@ def parse_args(self):
             help="name of output layer",
             type=str, default="MobilenetV1/Predictions/Reshape_1")
 
-        self.args = parser.parse_args(self.custom_args,
-            namespace=self.args)
+        self.args = parser.parse_args(self.custom_args, namespace=self.args)
 
     def run(self):
         self.run_command(self.command_prefix)
diff --git a/benchmarks/image_recognition/tensorflow/resnet50v1_5/inference/bfloat16/__init__.py b/benchmarks/image_recognition/tensorflow/resnet50v1_5/inference/bfloat16/__init__.py
@@ -15,4 +15,3 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
diff --git a/...image_recognition/tensorflow/resnet50v1_5/training/common_resnet50/resnet50_model_init.py b/...image_recognition/tensorflow/resnet50v1_5/training/common_resnet50/resnet50_model_init.py
@@ -25,7 +25,6 @@
 
 import os
 from argparse import ArgumentParser
-import time
 
 
 class ResNet50ModelInitializer(BaseModelInitializer):
@@ -72,7 +71,7 @@ def __init__(self, args, custom_args=[], platform_util=None):
 
         # We need to change directory to model source to avoid python
         # module name conflicts.
-        #self.benchmark_command = "cd " + self.args.model_source_dir + \
+        # self.benchmark_command = "cd " + self.args.model_source_dir + \
         #    "/models && " + self.get_command_prefix(args.socket_id) + \
         #    self.python_exe + " " + benchmark_script
 

diff --git a/benchmarks/language_modeling/tensorflow/bert_large/__init__.py b/benchmarks/language_modeling/tensorflow/bert_large/__init__.py
@@ -17,4 +17,3 @@
 #
 # SPDX-License-Identifier: EPL-2.0
 #
-
diff --git a/benchmarks/language_modeling/tensorflow/bert_large/inference/__init__.py b/benchmarks/language_modeling/tensorflow/bert_large/inference/__init__.py
@@ -17,4 +17,3 @@
 #
 # SPDX-License-Identifier: EPL-2.0
 #
-
diff --git a/benchmarks/language_modeling/tensorflow/bert_large/inference/bfloat16/__init__.py b/benchmarks/language_modeling/tensorflow/bert_large/inference/bfloat16/__init__.py
@@ -17,4 +17,3 @@
 #
 # SPDX-License-Identifier: EPL-2.0
 #
-
diff --git a/benchmarks/language_modeling/tensorflow/bert_large/inference/bfloat16/model_init.py b/benchmarks/language_modeling/tensorflow/bert_large/inference/bfloat16/model_init.py
@@ -26,7 +26,6 @@
 
 import os
 from argparse import ArgumentParser
-import time
 
 
 class ModelInitializer(BaseModelInitializer):
@@ -143,4 +142,3 @@ def run(self):
             else:
                 print("Warning: The {} preditions file was not found. Unable to "
                       "run the evaluation script.".format(predictions_json))
-
diff --git a/benchmarks/language_modeling/tensorflow/bert_large/inference/fp32/__init__.py b/benchmarks/language_modeling/tensorflow/bert_large/inference/fp32/__init__.py
@@ -17,4 +17,3 @@
 #
 # SPDX-License-Identifier: EPL-2.0
 #
-
diff --git a/benchmarks/language_modeling/tensorflow/bert_large/inference/fp32/model_init.py b/benchmarks/language_modeling/tensorflow/bert_large/inference/fp32/model_init.py
@@ -26,7 +26,6 @@
 
 import os
 from argparse import ArgumentParser
-import time
 
 
 class ModelInitializer(BaseModelInitializer):
@@ -142,4 +141,3 @@ def run(self):
             else:
                 print("Warning: The {} preditions file was not found. Unable to "
                       "run the evaluation script.".format(predictions_json))
-