Merge branch 'master' into softq

* master: (56 commits)
  [xray] Turn on flushing to the GCS for the lineage cache (ray-project#1907)
  Single Big Object Parallel Transfer. (ray-project#1827)
  Remove num_threads as a parameter. (ray-project#1891)
  Adds Valgrind tests for multi-threaded object manager. (ray-project#1890)
  Pin cython version in docker base dependencies file. (ray-project#1898)
  Update arrow to efficiently serialize more types of numpy arrays. (ray-project#1889)
  updates (ray-project#1896)
  [DataFrame] Inherit documentation from Pandas (ray-project#1727)
  Update arrow and parquet-cpp. (ray-project#1875)
  raylet command line resource configuration plumbing (ray-project#1882)
  use raylet for remote ray nodes (ray-project#1880)
  [rllib] Propagate dim option to deepmind wrappers (ray-project#1876)
  [RLLib] DDPG (ray-project#1685)
  Lint Python files with Yapf (ray-project#1872)
  [DataFrame] Fixed repr, info, and memory_usage (ray-project#1874)
  Fix getattr compat (ray-project#1871)
  check if arrow build dir exists (ray-project#1863)
  [DataFrame] Encapsulate index and lengths into separate class (ray-project#1849)
  [DataFrame] Implemented __getattr__ (ray-project#1753)
  Add better analytics to docs (ray-project#1854)
  ...

# Conflicts:
#	python/ray/rllib/__init__.py
#	python/setup.py

.style.yapf

@@ -42,7 +42,7 @@ blank_line_before_nested_class_or_def=False
 #        'key1': 'value1',
 #        'key2': 'value2',
 #    })
-coalesce_brackets=False
+coalesce_brackets=True
 
 # The column limit.
 column_limit=79
@@ -90,7 +90,7 @@ i18n_function_call=
 #       'key2': value1 +
 #               value2,
 #   }
-indent_dictionary_value=False
+indent_dictionary_value=True
 
 # The number of columns to use for indentation.
 indent_width=4
@@ -187,4 +187,3 @@ split_penalty_logical_operator=300
 
 # Use the Tab character for indentation.
 use_tabs=False
-

.travis.yml

@@ -38,10 +38,12 @@ matrix:
         - export PATH="$HOME/miniconda/bin:$PATH"
         - cd doc
         - pip install -q -r requirements-doc.txt
+        - pip install yapf
         - sphinx-build -W -b html -d _build/doctrees source _build/html
         - cd ..
         # Run Python linting.
         - flake8 --exclude=python/ray/core/src/common/flatbuffers_ep-prefix/,python/ray/core/generated/,src/common/format/,doc/source/conf.py,python/ray/cloudpickle/
+        - .travis/yapf.sh
 
     - os: linux
       dist: trusty
@@ -59,6 +61,7 @@ matrix:
         - bash ../../../src/common/test/run_valgrind.sh
         - bash ../../../src/plasma/test/run_valgrind.sh
         - bash ../../../src/local_scheduler/test/run_valgrind.sh
+        - bash ../../../src/ray/test/run_object_manager_valgrind.sh
         - cd ../../..
 
         - python ./python/ray/plasma/test/test.py valgrind
@@ -102,6 +105,12 @@ install:
 
   - cd python/ray/core
   - bash ../../../src/ray/test/run_gcs_tests.sh
+  # Raylet tests.
+  - bash ../../../src/ray/test/run_object_manager_tests.sh
+  - ./src/ray/raylet/task_test
+  - ./src/ray/raylet/worker_pool_test
+  - ./src/ray/raylet/lineage_cache_test
+
   - bash ../../../src/common/test/run_tests.sh
   - bash ../../../src/plasma/test/run_tests.sh
   - bash ../../../src/local_scheduler/test/run_tests.sh
@@ -116,6 +125,8 @@ script:
   - python python/ray/local_scheduler/test/test.py
   - python python/ray/global_scheduler/test/test.py
 
+  - python -m pytest test/xray_test.py
+
   - python test/runtest.py
   - python test/array_test.py
   - python test/actor_test.py
@@ -134,8 +145,10 @@ script:
   # ray dataframe tests
   - python -m pytest python/ray/dataframe/test/test_dataframe.py
   - python -m pytest python/ray/dataframe/test/test_series.py
+  - python -m pytest python/ray/dataframe/test/test_concat.py
 
   # ray tune tests
+  - python python/ray/tune/test/dependency_test.py
   - python -m pytest python/ray/tune/test/trial_runner_test.py
   - python -m pytest python/ray/tune/test/trial_scheduler_test.py
   - python -m pytest python/ray/tune/test/tune_server_test.py

.travis/yapf.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+# Cause the script to exit if a single command fails
+set -e
+
+ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
+
+pushd $ROOT_DIR/../test
+  find . -name '*.py' -type f -exec yapf --style=pep8 -i -r {} \;
+popd
+
+pushd $ROOT_DIR/../python
+  find . -name '*.py' -type f -not -path './ray/dataframe/*' -not -path './ray/rllib/*' -not -path './ray/cloudpickle/*' -exec yapf --style=pep8 -i -r {} \;
+popd
+
+CHANGED_FILES=(`git diff --name-only`)
+if [ "$CHANGED_FILES" ]; then
+  echo 'Reformatted staged files. Please review and stage the changes.'
+  echo
+  echo 'Files updated:'
+  for file in ${CHANGED_FILES[@]}; do
+    echo "  $file"
+  done
+  exit 1
+else
+  exit 0
+fi

cmake/Modules/BuildUtils.cmake

@@ -226,3 +226,13 @@ function(ADD_RAY_TEST REL_TEST_NAME)
   endif()
   set_tests_properties(${TEST_NAME} PROPERTIES LABELS "unittest")
 endfunction()
+
+# A wrapper for target_link_libraries() that is compatible with NO_TESTS.
+function(RAY_TEST_LINK_LIBRARIES REL_TEST_NAME)
+  if(NO_TESTS)
+    return()
+  endif()
+  get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE)
+
+  target_link_libraries(${TEST_NAME} ${ARGN})
+endfunction()

doc/source/_templates/layout.html

@@ -0,0 +1,14 @@
+{% extends "!layout.html" %}
+
+{%- block extrahead %}
+<script async src="https://www.googletagmanager.com/gtag/js?id=UA-110413294-1"></script>
+<script>
+  window.dataLayer = window.dataLayer || [];
+  function gtag(){dataLayer.push(arguments);}
+  gtag('js', new Date());
+
+  gtag('config', 'UA-110413294-1');
+</script>
+
+{% endblock %}
+

doc/source/conf.py

@@ -38,7 +38,8 @@
                 "ray.core.generated.TaskInfo",
                 "ray.core.generated.TaskReply",
                 "ray.core.generated.ResultTableReply",
-                "ray.core.generated.TaskExecutionDependencies"]
+                "ray.core.generated.TaskExecutionDependencies",
+                "ray.core.generated.ClientTableData"]
 for mod_name in MOCK_MODULES:
   sys.modules[mod_name] = mock.Mock()
 

doc/source/installation.rst

@@ -31,15 +31,15 @@ features but may be subject to more bugs. To install these wheels, run the follo
 ===================  ===================
 
 
-.. _`Linux Python 3.6`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.3.1-cp36-cp36m-manylinux1_x86_64.whl
-.. _`Linux Python 3.5`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.3.1-cp35-cp35m-manylinux1_x86_64.whl
-.. _`Linux Python 3.4`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.3.1-cp34-cp34m-manylinux1_x86_64.whl
-.. _`Linux Python 3.3`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.3.1-cp33-cp33m-manylinux1_x86_64.whl
-.. _`Linux Python 2.7`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.3.1-cp27-cp27mu-manylinux1_x86_64.whl
-.. _`MacOS Python 3.6`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.3.1-cp36-cp36m-macosx_10_6_intel.whl
-.. _`MacOS Python 3.5`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.3.1-cp35-cp35m-macosx_10_6_intel.whl
-.. _`MacOS Python 3.4`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.3.1-cp34-cp34m-macosx_10_6_intel.whl
-.. _`MacOS Python 2.7`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.3.1-cp27-cp27m-macosx_10_6_intel.whl
+.. _`Linux Python 3.6`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.4.0-cp36-cp36m-manylinux1_x86_64.whl
+.. _`Linux Python 3.5`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.4.0-cp35-cp35m-manylinux1_x86_64.whl
+.. _`Linux Python 3.4`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.4.0-cp34-cp34m-manylinux1_x86_64.whl
+.. _`Linux Python 3.3`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.4.0-cp33-cp33m-manylinux1_x86_64.whl
+.. _`Linux Python 2.7`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.4.0-cp27-cp27mu-manylinux1_x86_64.whl
+.. _`MacOS Python 3.6`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.4.0-cp36-cp36m-macosx_10_6_intel.whl
+.. _`MacOS Python 3.5`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.4.0-cp35-cp35m-macosx_10_6_intel.whl
+.. _`MacOS Python 3.4`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.4.0-cp34-cp34m-macosx_10_6_intel.whl
+.. _`MacOS Python 2.7`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.4.0-cp27-cp27m-macosx_10_6_intel.whl
 
 
 Building Ray from source

doc/source/tune-config.rst

@@ -69,6 +69,30 @@ By default, each random variable and grid search point is sampled once. To take
 
 For more information on variant generation, see `variant_generator.py <https://github.com/ray-project/ray/blob/master/python/ray/tune/variant_generator.py>`__.
 
+Handling Large Datasets
+-----------------------
+
+You often will want to compute a large object (e.g., training data, model weights) on the driver and use that object within each trial. Ray Tune provides a ``pin_in_object_store`` utility function that can be used to broadcast such large objects. Objects pinned in this way will never be evicted from the Ray object store while the driver process is running, and can be efficiently retrieved from any task via ``get_pinned_object``.
+
+.. code-block:: python
+
+    import ray
+    from ray.tune import register_trainable, run_experiments
+    from ray.tune.util import pin_in_object_store, get_pinned_object
+
+    import numpy as np
+
+    ray.init()
+
+    # X_id can be referenced in closures
+    X_id = pin_in_object_store(np.random.random(size=100000000))
+
+    def f(config, reporter):
+        X = get_pinned_object(X_id)
+        # use X
+
+    register_trainable("f", f)
+    run_experiments(...)
 
 Resource Allocation
 -------------------

doc/source/tune.rst

@@ -62,7 +62,7 @@ Features
 
 Ray Tune has the following features:
 
--  Scalable implementations of search algorithms such as `Population Based Training (PBT) <pbt.html>`__, `Median Stopping Rule <hyperband.html#median-stopping-rule>`__, and `HyperBand <hyperband.html>`__.
+-  Scalable implementations of search algorithms such as `Population Based Training (PBT) <pbt.html>`__, `Median Stopping Rule <hyperband.html#median-stopping-rule>`__, Model-Based Optimization (HyperOpt), and `HyperBand <hyperband.html>`__.
 
 -  Integration with visualization tools such as `TensorBoard <https://www.tensorflow.org/get_started/summaries_and_tensorboard>`__, `rllab's VisKit <https://media.readthedocs.org/pdf/rllab/latest/rllab.pdf>`__, and a `parallel coordinates visualization <https://en.wikipedia.org/wiki/Parallel_coordinates>`__.
 
@@ -94,12 +94,28 @@ You can find the code for Ray Tune `here on GitHub <https://github.com/ray-proje
 Trial Schedulers
 ----------------
 
-By default, Ray Tune schedules trials in serial order with the ``FIFOScheduler`` class. However, you can also specify a custom scheduling algorithm that can early stop trials, perturb parameters, or incorporate suggestions from an external service. Currently implemented trial schedulers include `Population Based Training (PBT) <pbt.html>`__, `Median Stopping Rule <hyperband.html#median-stopping-rule>`__, and `HyperBand <hyperband.html>`__.
+By default, Ray Tune schedules trials in serial order with the ``FIFOScheduler`` class. However, you can also specify a custom scheduling algorithm that can early stop trials, perturb parameters, or incorporate suggestions from an external service. Currently implemented trial schedulers include `Population Based Training (PBT) <pbt.html>`__, `Median Stopping Rule <hyperband.html#median-stopping-rule>`__, Model-Based Optimization (HyperOpt), and `HyperBand <hyperband.html>`__.
 
 .. code-block:: python
 
     run_experiments({...}, scheduler=AsyncHyperBandScheduler())
 
+
+HyperOpt Integration
+--------------------
+
+The``HyperOptScheduler`` is a Trial Scheduler that is backed by HyperOpt to perform sequential model-based hyperparameter optimization.
+In order to use this scheduler, you will need to install HyperOpt via the following command:
+
+.. code-block:: bash
+
+    $ pip install --upgrade git+git://github.com/hyperopt/hyperopt.git
+
+An example of this can be found in `hyperopt_example.py <https://github.com/ray-project/ray/blob/master/python/ray/tune/examples/hyperopt_example.py>`__.
+
+.. autoclass:: ray.tune.hpo_scheduler.HyperOptScheduler
+
+
 Visualizing Results
 -------------------
 

docker/base-deps/Dockerfile

@@ -25,6 +25,7 @@ RUN apt-get update \
         libgcc \
     && /opt/conda/bin/conda clean -y --all \
     && /opt/conda/bin/pip install \
-        flatbuffers
+        flatbuffers \
+        cython==0.27.3
 
 ENV PATH "/opt/conda/bin:$PATH"

docker/examples/Dockerfile

@@ -4,4 +4,5 @@ FROM ray-project/deploy
 RUN conda install -y -c conda-forge tensorflow
 RUN apt-get install -y zlib1g-dev
 RUN pip install gym[atari] opencv-python==3.2.0.8 smart_open
+RUN pip install --upgrade git+git://github.com/hyperopt/hyperopt.git
 # RUN conda install -y -q pytorch torchvision -c soumith

examples/custom_env/custom_env.py

@@ -27,11 +27,11 @@ def __init__(self, config):
             0.0, self.end_pos, shape=(1,), dtype=np.float32)
         self._spec = EnvSpec("SimpleCorridor-{}-v0".format(self.end_pos))
 
-    def _reset(self):
+    def reset(self):
         self.cur_pos = 0
         return [self.cur_pos]
 
-    def _step(self, action):
+    def step(self, action):
         assert action in [0, 1]
         if action == 0 and self.cur_pos > 0:
             self.cur_pos -= 1

python/build-wheel-macos.sh

@@ -47,6 +47,11 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do
   PYTHON_EXE=$MACPYTHON_PY_PREFIX/$PY_MM/bin/python$PY_MM
   PIP_CMD="$(dirname $PYTHON_EXE)/pip$PY_MM"
 
+  pushd /tmp
+    # Install latest version of pip to avoid brownouts
+    curl https://bootstrap.pypa.io/get-pip.py | $PYTHON_EXE
+  popd
+
   pushd python
     # Install setuptools_scm because otherwise when building the wheel for
     # Python 3.6, we see an error.

python/ray/__init__.py

@@ -12,8 +12,8 @@
 
 # Add the directory containing pyarrow to the Python path so that we find the
 # pyarrow version packaged with ray and not a pre-existing pyarrow.
-pyarrow_path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
-                            "pyarrow_files")
+pyarrow_path = os.path.join(
+    os.path.abspath(os.path.dirname(__file__)), "pyarrow_files")
 sys.path.insert(0, pyarrow_path)
 
 # See https://github.com/ray-project/ray/issues/131.
@@ -27,29 +27,29 @@
 try:
     import pyarrow  # noqa: F401
 except ImportError as e:
-    if ((hasattr(e, "msg") and isinstance(e.msg, str) and
-            ("libstdc++" in e.msg or "CXX" in e.msg))):
+    if ((hasattr(e, "msg") and isinstance(e.msg, str)
+         and ("libstdc++" in e.msg or "CXX" in e.msg))):
         # This code path should be taken with Python 3.
         e.msg += helpful_message
-    elif (hasattr(e, "message") and isinstance(e.message, str) and
-          ("libstdc++" in e.message or "CXX" in e.message)):
+    elif (hasattr(e, "message") and isinstance(e.message, str)
+          and ("libstdc++" in e.message or "CXX" in e.message)):
         # This code path should be taken with Python 2.
-        condition = (hasattr(e, "args") and isinstance(e.args, tuple) and
-                     len(e.args) == 1 and isinstance(e.args[0], str))
+        condition = (hasattr(e, "args") and isinstance(e.args, tuple)
+                     and len(e.args) == 1 and isinstance(e.args[0], str))
         if condition:
-            e.args = (e.args[0] + helpful_message,)
+            e.args = (e.args[0] + helpful_message, )
         else:
             if not hasattr(e, "args"):
                 e.args = ()
             elif not isinstance(e.args, tuple):
-                e.args = (e.args,)
-            e.args += (helpful_message,)
+                e.args = (e.args, )
+            e.args += (helpful_message, )
     raise
 
 from ray.local_scheduler import _config  # noqa: E402
-from ray.worker import (error_info, init, connect, disconnect,
-                        get, put, wait, remote, log_event, log_span,
-                        flush_log, get_gpu_ids, get_webui_url,
+from ray.worker import (error_info, init, connect, disconnect, get, put, wait,
+                        remote, log_event, log_span, flush_log, get_gpu_ids,
+                        get_webui_url,
                         register_custom_serializer)  # noqa: E402
 from ray.worker import (SCRIPT_MODE, WORKER_MODE, PYTHON_MODE,
                         SILENT_MODE)  # noqa: E402
@@ -63,11 +63,13 @@
 # Fix this.
 __version__ = "0.4.0"
 
-__all__ = ["error_info", "init", "connect", "disconnect", "get", "put", "wait",
-           "remote", "log_event", "log_span", "flush_log", "actor", "method",
-           "get_gpu_ids", "get_webui_url", "register_custom_serializer",
-           "SCRIPT_MODE", "WORKER_MODE", "PYTHON_MODE", "SILENT_MODE",
-           "global_state", "_config", "__version__"]
+__all__ = [
+    "error_info", "init", "connect", "disconnect", "get", "put", "wait",
+    "remote", "log_event", "log_span", "flush_log", "actor", "method",
+    "get_gpu_ids", "get_webui_url", "register_custom_serializer",
+    "SCRIPT_MODE", "WORKER_MODE", "PYTHON_MODE", "SILENT_MODE", "global_state",
+    "_config", "__version__"
+]
 
 import ctypes  # noqa: E402
 # Windows only