Run flake8 in Travis and make code PEP8 compliant. (ray-project#387)

Author: robertnishihara

.travis.yml

@@ -35,6 +35,10 @@ matrix:
         - cd doc
         - pip install -r requirements-doc.txt
         - sphinx-build -W -b html -d _build/doctrees source _build/html
+        - cd ..
+        # Run Python linting.
+        - flake8 --ignore=E111,E114
+                 --exclude=python/ray/core/src/common/flatbuffers_ep-prefix/,python/ray/core/generated/,src/numbuf/thirdparty/,src/common/format/,examples/,doc/source/conf.py
     - os: linux
       dist: trusty
       env: VALGRIND=1 PYTHON=2.7

.travis/install-dependencies.sh

@@ -64,6 +64,9 @@ elif [[ "$LINT" == "1" ]]; then
   # Install miniconda.
   wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
   bash miniconda.sh -b -p $HOME/miniconda
+  export PATH="$HOME/miniconda/bin:$PATH"
+  # Install Python linting tools.
+  pip install flake8
 else
   echo "Unrecognized environment."
   exit 1

python/ray/__init__.py

@@ -2,19 +2,29 @@
 from __future__ import division
 from __future__ import print_function
 
+from ray.worker import (register_class, error_info, init, connect, disconnect,
+                        get, put, wait, remote, log_event, log_span,
+                        flush_log)
+from ray.actor import actor
+from ray.actor import get_gpu_ids
+from ray.worker import EnvironmentVariable, env
+from ray.worker import SCRIPT_MODE, WORKER_MODE, PYTHON_MODE, SILENT_MODE
+
 # Ray version string
 __version__ = "0.01"
 
+__all__ = ["register_class", "error_info", "init", "connect", "disconnect",
+           "get", "put", "wait", "remote", "log_event", "log_span",
+           "flush_log", "actor", "get_gpu_ids", "EnvironmentVariable", "env",
+           "SCRIPT_MODE", "WORKER_MODE", "PYTHON_MODE", "SILENT_MODE",
+           "__version__"]
+
 import ctypes
 # Windows only
 if hasattr(ctypes, "windll"):
-  # Makes sure that all child processes die when we die
-  # Also makes sure that fatal crashes result in process termination rather than an error dialog (the latter is annoying since we have a lot of processes)
-  # This is done by associating all child processes with a "job" object that imposes this behavior.
-  (lambda kernel32: (lambda job: (lambda n: kernel32.SetInformationJobObject(job, 9, "\0" * 17 + chr(0x8 | 0x4 | 0x20) + "\0" * (n - 18), n))(0x90 if ctypes.sizeof(ctypes.c_void_p) > ctypes.sizeof(ctypes.c_int) else 0x70) and kernel32.AssignProcessToJobObject(job, ctypes.c_void_p(kernel32.GetCurrentProcess())))(ctypes.c_void_p(kernel32.CreateJobObjectW(None, None))) if kernel32 is not None else None)(ctypes.windll.kernel32)
-
-from ray.worker import register_class, error_info, init, connect, disconnect, get, put, wait, remote, log_event, log_span, flush_log
-from ray.actor import actor
-from ray.actor import get_gpu_ids
-from ray.worker import EnvironmentVariable, env
-from ray.worker import SCRIPT_MODE, WORKER_MODE, PYTHON_MODE, SILENT_MODE
+  # Makes sure that all child processes die when we die. Also makes sure that
+  # fatal crashes result in process termination rather than an error dialog
+  # (the latter is annoying since we have a lot of processes). This is done by
+  # associating all child processes with a "job" object that imposes this
+  # behavior.
+  (lambda kernel32: (lambda job: (lambda n: kernel32.SetInformationJobObject(job, 9, "\0" * 17 + chr(0x8 | 0x4 | 0x20) + "\0" * (n - 18), n))(0x90 if ctypes.sizeof(ctypes.c_void_p) > ctypes.sizeof(ctypes.c_int) else 0x70) and kernel32.AssignProcessToJobObject(job, ctypes.c_void_p(kernel32.GetCurrentProcess())))(ctypes.c_void_p(kernel32.CreateJobObjectW(None, None))) if kernel32 is not None else None)(ctypes.windll.kernel32)  # noqa: E501

python/ray/actor.py

@@ -18,6 +18,7 @@
 # the worker is currently allowed to use.
 gpu_ids = []
 
+
 def get_gpu_ids():
   """Get the IDs of the GPU that are available to the worker.
 
@@ -26,12 +27,15 @@ def get_gpu_ids():
   """
   return gpu_ids
 
+
 def random_string():
   return np.random.bytes(20)
 
+
 def random_actor_id():
   return ray.local_scheduler.ObjectID(random_string())
 
+
 def get_actor_method_function_id(attr):
   """Get the function ID corresponding to an actor method.
 
@@ -47,10 +51,14 @@ def get_actor_method_function_id(attr):
   assert len(function_id) == 20
   return ray.local_scheduler.ObjectID(function_id)
 
+
 def fetch_and_register_actor(key, worker):
   """Import an actor."""
-  driver_id, actor_id_str, actor_name, module, pickled_class, assigned_gpu_ids, actor_method_names = \
-    worker.redis_client.hmget(key, ["driver_id", "actor_id", "name", "module", "class", "gpu_ids", "actor_method_names"])
+  (driver_id, actor_id_str, actor_name,
+   module, pickled_class, assigned_gpu_ids,
+   actor_method_names) = worker.redis_client.hmget(
+       key, ["driver_id", "actor_id", "name", "module", "class", "gpu_ids",
+             "actor_method_names"])
   actor_id = ray.local_scheduler.ObjectID(actor_id_str)
   actor_name = actor_name.decode("ascii")
   module = module.decode("ascii")
@@ -64,12 +72,14 @@ def fetch_and_register_actor(key, worker):
   class TemporaryActor(object):
     pass
   worker.actors[actor_id_str] = TemporaryActor()
+
   def temporary_actor_method(*xs):
     raise Exception("The actor with name {} failed to be imported, and so "
                     "cannot execute this method".format(actor_name))
   for actor_method_name in actor_method_names:
     function_id = get_actor_method_function_id(actor_method_name).id()
-    worker.functions[driver_id][function_id] = (actor_method_name, temporary_actor_method)
+    worker.functions[driver_id][function_id] = (actor_method_name,
+                                                temporary_actor_method)
 
   try:
     unpickled_class = pickling.loads(pickled_class)
@@ -84,11 +94,15 @@ def temporary_actor_method(*xs):
     # TODO(pcm): Why is the below line necessary?
     unpickled_class.__module__ = module
     worker.actors[actor_id_str] = unpickled_class.__new__(unpickled_class)
-    for (k, v) in inspect.getmembers(unpickled_class, predicate=(lambda x: inspect.isfunction(x) or inspect.ismethod(x))):
+    for (k, v) in inspect.getmembers(
+        unpickled_class, predicate=(lambda x: (inspect.isfunction(x) or
+                                               inspect.ismethod(x)))):
       function_id = get_actor_method_function_id(k).id()
       worker.functions[driver_id][function_id] = (k, v)
-      # We do not set worker.function_properties[driver_id][function_id] because
-      # we currently do need the actor worker to submit new tasks for the actor.
+      # We do not set worker.function_properties[driver_id][function_id]
+      # because we currently do need the actor worker to submit new tasks for
+      # the actor.
+
 
 def select_local_scheduler(local_schedulers, num_gpus, worker):
   """Select a local scheduler to assign this actor to.
@@ -119,26 +133,33 @@ def select_local_scheduler(local_schedulers, num_gpus, worker):
     # Loop through all of the local schedulers.
     for local_scheduler in local_schedulers:
       # See if there are enough available GPUs on this local scheduler.
-      local_scheduler_total_gpus = int(float(local_scheduler[b"num_gpus"].decode("ascii")))
-      gpus_in_use = worker.redis_client.hget(local_scheduler[b"ray_client_id"], b"gpus_in_use")
+      local_scheduler_total_gpus = int(float(
+          local_scheduler[b"num_gpus"].decode("ascii")))
+      gpus_in_use = worker.redis_client.hget(local_scheduler[b"ray_client_id"],
+                                             b"gpus_in_use")
       gpus_in_use = 0 if gpus_in_use is None else int(gpus_in_use)
       if gpus_in_use + num_gpus <= local_scheduler_total_gpus:
         # Attempt to reserve some GPUs for this actor.
-        new_gpus_in_use = worker.redis_client.hincrby(local_scheduler[b"ray_client_id"], b"gpus_in_use", num_gpus)
+        new_gpus_in_use = worker.redis_client.hincrby(
+            local_scheduler[b"ray_client_id"], b"gpus_in_use", num_gpus)
         if new_gpus_in_use > local_scheduler_total_gpus:
           # If we failed to reserve the GPUs, undo the increment.
-          worker.redis_client.hincrby(local_scheduler[b"ray_client_id"], b"gpus_in_use", num_gpus)
+          worker.redis_client.hincrby(local_scheduler[b"ray_client_id"],
+                                      b"gpus_in_use", num_gpus)
         else:
           # We succeeded at reserving the GPUs, so we are done.
           local_scheduler_id = local_scheduler[b"ray_client_id"]
           gpu_ids = list(range(new_gpus_in_use - num_gpus, new_gpus_in_use))
           break
     if local_scheduler_id is None:
       raise Exception("Could not find a node with enough GPUs to create this "
-                      "actor. The local scheduler information is {}.".format(local_schedulers))
+                      "actor. The local scheduler information is {}."
+                      .format(local_schedulers))
   return local_scheduler_id, gpu_ids
 
-def export_actor(actor_id, Class, actor_method_names, num_cpus, num_gpus, worker):
+
+def export_actor(actor_id, Class, actor_method_names, num_cpus, num_gpus,
+                 worker):
   """Export an actor to redis.
 
   Args:
@@ -158,13 +179,16 @@ def export_actor(actor_id, Class, actor_method_names, num_cpus, num_gpus, worker
   driver_id = worker.task_driver_id.id()
   for actor_method_name in actor_method_names:
     function_id = get_actor_method_function_id(actor_method_name).id()
-    worker.function_properties[driver_id][function_id] = (1, num_cpus, num_gpus)
+    worker.function_properties[driver_id][function_id] = (1, num_cpus,
+                                                          num_gpus)
 
   # Select a local scheduler for the actor.
   local_schedulers = state.get_local_schedulers(worker)
-  local_scheduler_id, gpu_ids = select_local_scheduler(local_schedulers, num_gpus, worker)
+  local_scheduler_id, gpu_ids = select_local_scheduler(local_schedulers,
+                                                       num_gpus, worker)
 
-  worker.redis_client.publish("actor_notifications", actor_id.id() + local_scheduler_id)
+  worker.redis_client.publish("actor_notifications",
+                              actor_id.id() + local_scheduler_id)
 
   d = {"driver_id": driver_id,
        "actor_id": actor_id.id(),
@@ -176,6 +200,7 @@ def export_actor(actor_id, Class, actor_method_names, num_cpus, num_gpus, worker
   worker.redis_client.hmset(key, d)
   worker.redis_client.rpush("Exports", key)
 
+
 def actor(*args, **kwargs):
   def make_actor_decorator(num_cpus=1, num_gpus=0):
     def make_actor(Class):
@@ -189,7 +214,8 @@ def actor_method_call(actor_id, attr, *args, **kwargs):
           raise Exception("Actors currently do not support **kwargs.")
         function_id = get_actor_method_function_id(attr)
         # TODO(pcm): Extend args with keyword args.
-        object_ids = ray.worker.global_worker.submit_task(function_id, "", args,
+        object_ids = ray.worker.global_worker.submit_task(function_id, "",
+                                                          args,
                                                           actor_id=actor_id)
         if len(object_ids) == 1:
           return object_ids[0]
@@ -199,24 +225,34 @@ def actor_method_call(actor_id, attr, *args, **kwargs):
       class NewClass(object):
         def __init__(self, *args, **kwargs):
           self._ray_actor_id = random_actor_id()
-          self._ray_actor_methods = {k: v for (k, v) in inspect.getmembers(Class, predicate=(lambda x: inspect.isfunction(x) or inspect.ismethod(x)))}
-          export_actor(self._ray_actor_id, Class, self._ray_actor_methods.keys(), num_cpus, num_gpus, ray.worker.global_worker)
+          self._ray_actor_methods = {
+              k: v for (k, v) in inspect.getmembers(
+                  Class, predicate=(lambda x: (inspect.isfunction(x) or
+                                               inspect.ismethod(x))))}
+          export_actor(self._ray_actor_id, Class,
+                       self._ray_actor_methods.keys(), num_cpus, num_gpus,
+                       ray.worker.global_worker)
           # Call __init__ as a remote function.
           if "__init__" in self._ray_actor_methods.keys():
             actor_method_call(self._ray_actor_id, "__init__", *args, **kwargs)
           else:
             print("WARNING: this object has no __init__ method.")
+
         # Make tab completion work.
         def __dir__(self):
           return self._ray_actor_methods
+
         def __getattribute__(self, attr):
           # The following is needed so we can still access self.actor_methods.
           if attr in ["_ray_actor_id", "_ray_actor_methods"]:
             return super(NewClass, self).__getattribute__(attr)
           if attr in self._ray_actor_methods.keys():
-            return lambda *args, **kwargs: actor_method_call(self._ray_actor_id, attr, *args, **kwargs)
+            return lambda *args, **kwargs: actor_method_call(
+                self._ray_actor_id, attr, *args, **kwargs)
           # There is no method with this name, so raise an exception.
-          raise AttributeError("'{}' Actor object has no attribute '{}'".format(Class, attr))
+          raise AttributeError("'{}' Actor object has no attribute '{}'"
+                               .format(Class, attr))
+
         def __repr__(self):
           return "Actor(" + self._ray_actor_id.hex() + ")"
 
@@ -230,7 +266,9 @@ def __repr__(self):
     return make_actor_decorator(num_cpus=1, num_gpus=0)(Class)
 
   # In this case, the actor decorator is something like @ray.actor(num_gpus=1).
-  if len(args) == 0 and len(kwargs) > 0 and all([key in ["num_cpus", "num_gpus"] for key in kwargs.keys()]):
+  if len(args) == 0 and len(kwargs) > 0 and all([key
+                                                 in ["num_cpus", "num_gpus"]
+                                                 for key in kwargs.keys()]):
     num_cpus = kwargs["num_cpus"] if "num_cpus" in kwargs.keys() else 1
     num_gpus = kwargs["num_gpus"] if "num_gpus" in kwargs.keys() else 0
     return make_actor_decorator(num_cpus=num_cpus, num_gpus=num_gpus)
@@ -240,4 +278,5 @@ def __repr__(self):
                   "some of the arguments 'num_cpus' or 'num_gpus' as in "
                   "'ray.actor(num_gpus=1)'.")
 
+
 ray.worker.global_worker.fetch_and_register["Actor"] = fetch_and_register_actor