[DO NOT MERGE] Rewrite YAPF scripts

.style.yapf

@@ -14,9 +14,19 @@ allow_multiline_dictionary_keys=False
 # Allow lambdas to be formatted on more than one line.
 allow_multiline_lambdas=False
 
+# Allow splits before the dictionary value.
+allow_split_before_dict_value=True
+
+# Number of blank lines surrounding top-level function and class
+# definitions.
+blank_lines_around_top_level_definition=2
+
 # Insert a blank line before a class-level docstring.
 blank_line_before_class_docstring=False
 
+# Insert a blank line before a module docstring.
+blank_line_before_module_docstring=False
+
 # Insert a blank line before a 'def' or 'class' immediately nested
 # within another 'def' or 'class'. For example:
 #
@@ -42,10 +52,26 @@ blank_line_before_nested_class_or_def=False
 #        'key1': 'value1',
 #        'key2': 'value2',
 #    })
-coalesce_brackets=True
+coalesce_brackets=False
 
 # The column limit.
-column_limit=79
+column_limit=80
+
+# The style for continuation alignment. Possible values are:
+#
+# - SPACE: Use spaces for continuation alignment. This is default behavior.
+# - FIXED: Use fixed number (CONTINUATION_INDENT_WIDTH) of columns
+#   (ie: CONTINUATION_INDENT_WIDTH/INDENT_WIDTH tabs) for continuation
+#   alignment.
+# - LESS: Slightly left if cannot vertically align continuation lines with
+#   indent characters.
+# - VALIGN-RIGHT: Vertically align continuation lines with indent
+#   characters. Slightly right (one more indent character) if cannot
+#   vertically align continuation lines with indent characters.
+#
+# For options FIXED, and VALIGN-RIGHT are only available when USE_TABS is
+# enabled.
+continuation_align_style=SPACE
 
 # Indent width used for line continuations.
 continuation_indent_width=4
@@ -66,7 +92,7 @@ continuation_indent_width=4
 #       start_ts=now()-timedelta(days=3),
 #       end_ts=now(),
 #   )        # <--- this bracket is dedented and on a separate line
-dedent_closing_brackets=False
+dedent_closing_brackets=True
 
 # Place each dictionary entry onto its own line.
 each_dict_entry_on_separate_line=True
@@ -90,13 +116,13 @@ i18n_function_call=
 #       'key2': value1 +
 #               value2,
 #   }
-indent_dictionary_value=True
+indent_dictionary_value=False
 
 # The number of columns to use for indentation.
 indent_width=4
 
 # Join short lines into one line. E.g., single line 'if' statements.
-join_multiple_lines=True
+join_multiple_lines=False
 
 # Do not include spaces around selected binary operators. For example:
 #
@@ -106,7 +132,7 @@ join_multiple_lines=True
 #
 #   1 + 2*3 - 4/5
 #
-no_spaces_around_selected_binary_operators=set([])
+no_spaces_around_selected_binary_operators=set()
 
 # Use spaces around default or named assigns.
 spaces_around_default_or_named_assign=False
@@ -123,12 +149,16 @@ space_between_ending_comma_and_closing_bracket=True
 
 # Split before arguments if the argument list is terminated by a
 # comma.
-split_arguments_when_comma_terminated=False
+split_arguments_when_comma_terminated=True
 
 # Set to True to prefer splitting before '&', '|' or '^' rather than
 # after.
 split_before_bitwise_operator=True
 
+# Split before the closing bracket if a list or dict literal doesn't fit on
+# a single line.
+split_before_closing_bracket=True
+
 # Split before a dictionary or set generator (comp_for). For example, note
 # the split before the 'for':
 #
@@ -138,6 +168,10 @@ split_before_bitwise_operator=True
 #   }
 split_before_dict_set_generator=True
 
+# Split after the opening paren which surrounds an expression if it doesn't
+# fit on a single line.
+split_before_expression_after_opening_paren=False
+
 # If an argument / parameter list is going to be split, then split before
 # the first argument.
 split_before_first_argument=False
@@ -149,6 +183,22 @@ split_before_logical_operator=True
 # Split named assignments onto individual lines.
 split_before_named_assigns=True
 
+# Set to True to split list comprehensions and generators that have
+# non-trivial expressions and multiple clauses before each of these
+# clauses. For example:
+#
+#   result = [
+#       a_long_var + 100 for a_long_var in xrange(1000)
+#       if a_long_var % 10]
+#
+# would reformat to something like:
+#
+#   result = [
+#       a_long_var + 100
+#       for a_long_var in xrange(1000)
+#       if a_long_var % 10]
+split_complex_comprehension=True
+
 # The penalty for splitting right after the opening bracket.
 split_penalty_after_opening_bracket=30
 
@@ -162,6 +212,10 @@ split_penalty_before_if_expr=0
 # operators.
 split_penalty_bitwise_operator=300
 
+# The penalty for splitting a list comprehension or generator
+# expression.
+split_penalty_comprehension=80
+
 # The penalty for characters over the column limit.
 split_penalty_excess_character=4500
 
@@ -187,3 +241,4 @@ split_penalty_logical_operator=300
 
 # Use the Tab character for indentation.
 use_tabs=False
+

.travis/yapf.sh

@@ -1,27 +1,30 @@
 #!/usr/bin/env bash
 
 # Cause the script to exit if a single command fails
-set -e
-
-ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
-
-pushd $ROOT_DIR/../test
-  find . -name '*.py' -type f -exec yapf --style=pep8 -i -r {} \;
-popd
-
-pushd $ROOT_DIR/../python
-  find . -name '*.py' -type f -not -path './ray/dataframe/*' -not -path './ray/rllib/*' -not -path './ray/cloudpickle/*' -exec yapf --style=pep8 -i -r {} \;
-popd
-
-CHANGED_FILES=(`git diff --name-only`)
-if [ "$CHANGED_FILES" ]; then
-  echo 'Reformatted staged files. Please review and stage the changes.'
-  echo
-  echo 'Files updated:'
-  for file in ${CHANGED_FILES[@]}; do
-    echo "  $file"
-  done
-  exit 1
-else
-  exit 0
+set -eo pipefail
+
+ROOT="$(git rev-parse --show-toplevel)"
+builtin cd "$ROOT"
+
+yapf \
+    --style "$ROOT/.style.yapf" \
+    --in-place --recursive --parallel \
+    --exclude 'python/ray/dataframe/' \
+    --exclude 'python/ray/rllib/' \
+    --exclude 'python/ray/cloudpickle/' \
+    -- \
+    'test/' 'python/'
+
+CHANGED_FILES=($(git diff --name-only))
+
+if [[ "${#CHANGED_FILES[@]}" -gt 0 ]]; then
+    echo 'Reformatted staged files. Please review and stage the changes.'
+    echo 'Files updated:'
+
+    for file in "${CHANGED_FILES[@]}"; do
+        echo "$file"
+    done
+
+    exit 1
 fi
+

python/ray/autoscaler/updater.py

@@ -75,9 +75,9 @@ def run(self):
             self.provider.set_node_tags(self.node_id,
                                         {TAG_RAY_NODE_STATUS: "UpdateFailed"})
             if self.logfile is not None:
-                print("----- BEGIN REMOTE LOGS -----\n" + open(
-                    self.logfile.name).read() + "\n----- END REMOTE LOGS -----"
-                      )
+                print("----- BEGIN REMOTE LOGS -----\n" +
+                      open(self.logfile.name).read() +
+                      "\n----- END REMOTE LOGS -----")
             raise e
         self.provider.set_node_tags(
             self.node_id, {

python/ray/experimental/array/distributed/linalg.py

@@ -76,10 +76,9 @@ def tsqr(a):
                 lower = [a.shape[1], 0]
                 upper = [2 * a.shape[1], core.BLOCK_SIZE]
             ith_index //= 2
-            q_block_current = ra.dot.remote(q_block_current,
-                                            ra.subarray.remote(
-                                                q_tree[ith_index, j], lower,
-                                                upper))
+            q_block_current = ra.dot.remote(
+                q_block_current,
+                ra.subarray.remote(q_tree[ith_index, j], lower, upper))
         q_result.objectids[i] = q_block_current
     r = current_rs[0]
     return q_result, ray.get(r)
@@ -222,10 +221,10 @@ def qr(a):
         y_col_block = core.subblocks.remote(y_res, [], [i])
         q = core.subtract.remote(
             q,
-            core.dot.remote(y_col_block,
-                            core.dot.remote(
-                                Ts[i],
-                                core.dot.remote(
-                                    core.transpose.remote(y_col_block), q))))
+            core.dot.remote(
+                y_col_block,
+                core.dot.remote(
+                    Ts[i],
+                    core.dot.remote(core.transpose.remote(y_col_block), q))))
 
     return ray.get(q), r_res

python/ray/experimental/state.py

@@ -750,8 +750,8 @@ def micros_rel(ts):
                         "name":
                         "SubmitTask",
                         "args": {},
-                        "id": (parent_info["worker_id"] +
-                               str(micros(min(parent_times))))
+                        "id": (parent_info["worker_id"] + str(
+                            micros(min(parent_times))))
                     }
                     full_trace.append(parent)
 
@@ -825,8 +825,8 @@ def micros_rel(ts):
                         "name":
                         "SubmitTask",
                         "args": {},
-                        "id": (parent_info["worker_id"] +
-                               str(micros(min(parent_times))))
+                        "id": (parent_info["worker_id"] + str(
+                            micros(min(parent_times))))
                     }
                     full_trace.append(parent)
 

python/ray/experimental/ui.py

@@ -451,8 +451,8 @@ def task_completion_time_update(abs_earliest, abs_latest, abs_num_tasks,
         # Create the distribution to plot
         distr = []
         for task_id, data in tasks.items():
-            distr.append(
-                data["store_outputs_end"] - data["get_arguments_start"])
+            distr.append(data["store_outputs_end"] -
+                         data["get_arguments_start"])
 
         # Create a histogram from the distribution
         top, bin_edges = np.histogram(distr, bins="auto")
@@ -520,10 +520,10 @@ def compute_utilizations(abs_earliest,
         # Walk over each time bucket that this task intersects, adding the
         # amount of time that the task intersects within each bucket
         for bucket_idx in range(start_bucket, end_bucket + 1):
-            bucket_start_time = ((
-                earliest_time + bucket_idx) * bucket_time_length)
-            bucket_end_time = ((earliest_time +
-                                (bucket_idx + 1)) * bucket_time_length)
+            bucket_start_time = (
+                (earliest_time + bucket_idx) * bucket_time_length)
+            bucket_end_time = (
+                (earliest_time + (bucket_idx + 1)) * bucket_time_length)
 
             task_start_time_within_bucket = max(task_start_time,
                                                 bucket_start_time)

python/ray/log_monitor.py

@@ -39,8 +39,9 @@ def __init__(self, redis_ip_address, redis_port, node_ip_address):
     def update_log_filenames(self):
         """Get the most up-to-date list of log files to monitor from Redis."""
         num_current_log_files = len(self.log_files)
-        new_log_filenames = self.redis_client.lrange("LOG_FILENAMES:{}".format(
-            self.node_ip_address), num_current_log_files, -1)
+        new_log_filenames = self.redis_client.lrange(
+            "LOG_FILENAMES:{}".format(self.node_ip_address),
+            num_current_log_files, -1)
         for log_filename in new_log_filenames:
             print("Beginning to track file {}".format(log_filename))
             assert log_filename not in self.log_files

python/ray/monitor.py

@@ -189,10 +189,9 @@ def cleanup_object_table(self):
                 if manager in self.dead_plasma_managers:
                     # If the object was on a dead plasma manager, remove that
                     # location entry.
-                    ok = self.state._execute_command(object_id,
-                                                     "RAY.OBJECT_TABLE_REMOVE",
-                                                     object_id.id(),
-                                                     hex_to_binary(manager))
+                    ok = self.state._execute_command(
+                        object_id, "RAY.OBJECT_TABLE_REMOVE", object_id.id(),
+                        hex_to_binary(manager))
                     if ok != b"OK":
                         log.warn("Failed to remove object location for dead "
                                  "plasma manager.")
@@ -507,8 +506,8 @@ def run(self):
         log.debug("{} dead local schedulers, {} plasma managers total, {} "
                   "dead plasma managers".format(
                       len(self.dead_local_schedulers),
-                      (len(self.live_plasma_managers) +
-                       len(self.dead_plasma_managers)),
+                      (len(self.live_plasma_managers) + len(
+                          self.dead_plasma_managers)),
                       len(self.dead_plasma_managers)))
 
         # Handle messages from the subscription channels.

python/ray/tune/hyperband.py

@@ -391,9 +391,9 @@ def _calculate_total_work(self, n, r, s):
 
     def __repr__(self):
         status = ", ".join([
-            "Max Size (n)={}".format(self._n), "Milestone (r)={}".format(
-                self._cumul_r), "completed={:.1%}".format(
-                    self.completion_percentage())
+            "Max Size (n)={}".format(self._n),
+            "Milestone (r)={}".format(self._cumul_r),
+            "completed={:.1%}".format(self.completion_percentage())
         ])
         counts = collections.Counter([t.status for t in self._all_trials])
         trial_statuses = ", ".join(