Merge branch 'master' of https://github.com/ray-project/ray into traj…

…ectory_view_api_enable_by_default_for_some_tf � Conflicts: � rllib/policy/dynamic_tf_policy.py � rllib/policy/eager_tf_policy.py � rllib/policy/policy.py � rllib/policy/tf_policy_template.py � rllib/utils/tf_ops.py
ray-project · Nov 4, 2020 · 14f0e22 · 14f0e22
2 parents 2826753 + b7531fb
commit 14f0e22
Show file tree

Hide file tree

Showing 101 changed files with 1,640 additions and 1,932 deletions.
diff --git a/.bazelrc b/.bazelrc
@@ -7,6 +7,7 @@ build --enable_platform_specific_config
 build --action_env=PATH
 # For --compilation_mode=dbg, consider enabling checks in the standard library as well (below).
 build --compilation_mode=opt
+#build --cxxopt="-std=c++11"
 # This workaround is needed to prevent Bazel from compiling the same file twice (once PIC and once not).
 build:linux --force_pic
 build:macos --force_pic
@@ -35,6 +36,7 @@ build:msvc     --per_file_copt="-\\.(asm|S)$@-WX"
 # Ignore warnings for protobuf generated files and external projects.
 build --per_file_copt="\\.pb\\.cc$@-w"
 build --per_file_copt="-\\.(asm|S)$,external/.*@-w"
+#build --per_file_copt="external/.*@-Wno-unused-result"
 # Ignore minor warnings for host tools, which we generally can't control
 build:clang-cl --host_copt="-Wno-inconsistent-missing-override"
 build:clang-cl --host_copt="-Wno-microsoft-unqualified-friend"
@@ -60,6 +62,10 @@ build:windows --color=yes
 build:clang-cl --per_file_copt="-\\.(asm|S)$@-fansi-escape-codes"
 build:clang-cl --per_file_copt="-\\.(asm|S)$@-fcolor-diagnostics"
 
+build:manylinux2010 --copt="-Wno-unused-result"
+build:manylinux2010 --linkopt="-lrt"
+
+
 # Debug build flags. Uncomment in '-c dbg' builds to enable checks in the C++ standard library:
 #build:linux --cxxopt="-D_GLIBCXX_DEBUG=1"
 #build:linux --cxxopt="-D_GLIBCXX_DEBUG_PEDANTIC=1"

diff --git a/build-docker.sh b/build-docker.sh
@@ -7,7 +7,7 @@ set -x
 
 GPU=""
 BASE_IMAGE="ubuntu:focal"
-WHEEL_URL="https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux1_x86_64.whl"
+WHEEL_URL="https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl"
 PYTHON_VERSION=""
 
 while [[ $# -gt 0 ]]
@@ -90,4 +90,4 @@ if [ $BUILD_EXAMPLES ]; then
     fi
 fi
 
-rm -rf "$WHEEL_DIR"
+rm -rf "$WHEEL_DIR"
diff --git a/ci/travis/build-docker-images.py b/ci/travis/build-docker-images.py
@@ -105,7 +105,6 @@ def _build_cpu_gpu_images(image_name) -> List[str]:
                 start = datetime.datetime.now()
                 current_iter = start
                 for line in output:
-                    # print(line)
                     if datetime.datetime.now(
                     ) - current_iter >= datetime.timedelta(minutes=5):
                         current_iter = datetime.datetime.now()

diff --git a/ci/travis/ci.sh b/ci/travis/ci.sh
@@ -289,8 +289,8 @@ build_wheels() {
 
       # This command should be kept in sync with ray/python/README-building-wheels.md,
       # except the "${MOUNT_BAZEL_CACHE[@]}" part.
-      suppress_output docker run --rm -w /ray -v "${PWD}":/ray "${MOUNT_BAZEL_CACHE[@]}" \
-        rayproject/arrow_linux_x86_64_base:python-3.8.0 /ray/python/build-wheel-manylinux1.sh
+      docker run --rm -w /ray -v "${PWD}":/ray "${MOUNT_BAZEL_CACHE[@]}" \
+      quay.io/pypa/manylinux2014_x86_64 /ray/python/build-wheel-manylinux2014.sh
       ;;
     darwin*)
       # This command should be kept in sync with ray/python/README-building-wheels.md.

diff --git a/ci/travis/install-bazel.sh b/ci/travis/install-bazel.sh
@@ -44,6 +44,7 @@ esac
   fi
 )
 
+export PATH=/opt/python/cp36-cp36m/bin:$PATH
 python="$(command -v python3 || command -v python || echo python)"
 version="$("${python}" -s -c "import runpy, sys; runpy.run_path(sys.argv.pop(), run_name='__api__')" bazel_version "${ROOT_DIR}/../../python/setup.py")"
 if [ "${OSTYPE}" = "msys" ]; then

diff --git a/dashboard/client/build/.fakebuild b/dashboard/client/build/.fakebuild
diff --git a/dashboard/client/build/static/.fakestatic b/dashboard/client/build/static/.fakestatic
diff --git a/dashboard/client/src/pages/dashboard/memory/Memory.tsx b/dashboard/client/src/pages/dashboard/memory/Memory.tsx
@@ -1,6 +1,7 @@
 import {
   Box,
   Button,
+  CircularProgress,
   createStyles,
   FormControl,
   InputLabel,
@@ -18,6 +19,7 @@ import { useDispatch, useSelector } from "react-redux";
 import {
   getMemoryTable,
   MemoryGroupByKey,
+  MemoryTable,
   MemoryTableResponse,
   setMemoryTableCollection,
 } from "../../../api";
@@ -69,6 +71,14 @@ const useMemoryInfoStyles = makeStyles((theme: Theme) =>
 );
 
 const memoryTableSelector = (state: StoreState) => state.dashboard.memoryTable;
+const isEmpty = (memoryTable: MemoryTable) =>
+  Object.keys(memoryTable.group).length === 0;
+const loadTimerComplete = (mountedAt: Date) => {
+  const secondsBetween = Math.abs(
+    (new Date().getTime() - mountedAt.getTime()) / 1000,
+  );
+  return secondsBetween > 10;
+};
 
 const fetchMemoryTable = (
   groupByKey: MemoryGroupByKey,
@@ -83,7 +93,7 @@ const fetchMemoryTable = (
 const MemoryInfo: React.FC<{}> = () => {
   const memoryTable = useSelector(memoryTableSelector);
   const dispatch = useDispatch();
-
+  const mountedAt = new Date();
   const [paused, setPaused] = useState(false);
   const pauseButtonIcon = paused ? <PlayArrowIcon /> : <PauseIcon />;
 
@@ -97,6 +107,7 @@ const MemoryInfo: React.FC<{}> = () => {
       setMemoryTableCollection(false);
     };
   }, []);
+
   // Set up polling memory data
   const fetchData = useCallback(
     fetchMemoryTable(groupBy, (resp) =>
@@ -117,17 +128,19 @@ const MemoryInfo: React.FC<{}> = () => {
       }
     };
     return cleanup;
-  }, [paused, fetchData]);
+  }, [paused, fetchData, dispatch]);
 
-  if (!memoryTable) {
+  if (!memoryTable || (isEmpty(memoryTable) && !loadTimerComplete(mountedAt))) {
     return (
-      <Typography color="textSecondary">Loading memory information</Typography>
+      <Typography align="center" color="textSecondary" variant="h3">
+        <CircularProgress /> Loading
+      </Typography>
     );
   }
-  if (Object.keys(memoryTable.group).length === 0) {
+  if (isEmpty(memoryTable) && loadTimerComplete(mountedAt)) {
     return (
-      <Typography color="textSecondary">
-        Finished loading, but have found no memory data yet.
+      <Typography align="center" color="textSecondary" variant="h3">
+        Finished loading, but nothing found.
       </Typography>
     );
   }
@@ -170,6 +183,10 @@ const MemoryInfo: React.FC<{}> = () => {
         color="primary"
         className={classes.pauseButton}
         onClick={() => {
+          if (intervalId.current) {
+            clearInterval(intervalId.current);
+            intervalId.current = null;
+          }
           setMemoryTableCollection(!paused);
           setPaused(!paused);
         }}

diff --git a/doc/dev/RELEASE_PROCESS.rst b/doc/dev/RELEASE_PROCESS.rst
@@ -129,9 +129,9 @@ This document describes the process for creating new releases.
        export RAY_VERSION=...  # e.g., 0.7.0
 
        # Linux Wheels
-       pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp36-cp36m-manylinux1_x86_64.whl
-       pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp37-cp37m-manylinux1_x86_64.whl
-       pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp38-cp38-manylinux1_x86_64.whl
+       pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp36-cp36m-manylinux2014_x86_64.whl
+       pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp37-cp37m-manylinux2014_x86_64.whl
+       pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp38-cp38-manylinux2014_x86_64.whl
 
        # Mac Wheels
        pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp36-cp36m-macosx_10_13_intel.whl

diff --git a/doc/dev/download_wheels.sh b/doc/dev/download_wheels.sh
@@ -11,9 +11,9 @@ if [[ -z "$RAY_VERSION" ]]; then
 fi
 
 # Make sure Linux wheels are downloadable without errors.
-wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp36-cp36m-manylinux1_x86_64.whl"
-wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp37-cp37m-manylinux1_x86_64.whl"
-wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp38-cp38-manylinux1_x86_64.whl"
+wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp36-cp36m-manylinux2014_x86_64.whl"
+wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp37-cp37m-manylinux2014_x86_64.whl"
+wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp38-cp38-manylinux2014_x86_64.whl"
 # Make sure macos wheels are downloadable without errors.
 wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp36-cp36m-macosx_10_13_intel.whl"
 wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp37-cp37m-macosx_10_13_intel.whl"

diff --git a/doc/examples/newsreader/ray-project/cluster.yaml b/doc/examples/newsreader/ray-project/cluster.yaml
@@ -24,7 +24,7 @@ setup_commands:
   - echo 'export PATH="$HOME/anaconda3/bin:$PATH"' >> ~/.bashrc
   - sudo apt -y update
   - sudo apt -y install npm
-  - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux1_x86_64.whl
+  - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl
 
 # How Ray will authenticate with newly launched nodes.
 auth:

diff --git a/doc/source/advanced.rst b/doc/source/advanced.rst
@@ -77,7 +77,7 @@ When you have multiple tasks that need to wait on some condition, you can use a
     signal = SignalActor.remote()
     tasks = [wait_and_go.remote(signal) for _ in range(4)]
     print("ready...")
-    # Tasks will all be waiting for the singals.
+    # Tasks will all be waiting for the signals.
     print("set..")
     ray.get(signal.send.remote())
 

diff --git a/doc/source/development.rst b/doc/source/development.rst
@@ -23,7 +23,7 @@ RLlib, Tune, Autoscaler, and most Python files do not require you to build and c
 
 .. code-block:: shell
 
-    pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp38-cp38-manylinux1_x86_64.whl
+    pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp38-cp38-manylinux2014_x86_64.whl
 
 2. Fork and clone the project to your machine. Connect your repository to the upstream (main project) ray repository.
 

diff --git a/doc/source/installation.rst b/doc/source/installation.rst
@@ -51,9 +51,9 @@ instead of the ones above:
 `Linux Python 3.6`_  `MacOS Python 3.6`_  `Windows Python 3.6`_
 ===================  ===================  ======================
 
-.. _`Linux Python 3.8`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp38-cp38-manylinux1_x86_64.whl
-.. _`Linux Python 3.7`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux1_x86_64.whl
-.. _`Linux Python 3.6`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux1_x86_64.whl
+.. _`Linux Python 3.8`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp38-cp38-manylinux2014_x86_64.whl
+.. _`Linux Python 3.7`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl
+.. _`Linux Python 3.6`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl
 
 .. _`MacOS Python 3.8`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp38-cp38-macosx_10_13_x86_64.whl
 .. _`MacOS Python 3.7`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-macosx_10_13_intel.whl

diff --git a/doc/source/tune/api_docs/internals.rst b/doc/source/tune/api_docs/internals.rst
@@ -116,7 +116,7 @@ Trial
 Callbacks
 ---------
 
-.. autoclass:: ray.tune.trial_runner.Callback
+.. autoclass:: ray.tune.callback.Callback
    :members:
 
 

diff --git a/doc/source/tune/api_docs/schedulers.rst b/doc/source/tune/api_docs/schedulers.rst
@@ -16,7 +16,7 @@ All Trial Schedulers take in a ``metric``, which is a value returned in the resu
 Summary
 -------
 
-Tune includes distributed implementations of early stopping algorithms such as `Median Stopping Rule <https://research.google.com/pubs/pub46180.html>`__, `HyperBand <https://arxiv.org/abs/1603.06560>`__, and `ASHA <https://openreview.net/forum?id=S1Y7OOlRZ>`__. Tune also includes a distributed implementation of `Population Based Training (PBT) <https://deepmind.com/blog/population-based-training-neural-networks>`__ and `Population Based Bandits (PB2) <https://arxiv.org/abs/2002.02518>`__.
+Tune includes distributed implementations of early stopping algorithms such as `Median Stopping Rule <https://research.google.com/pubs/pub46180.html>`__, `HyperBand <https://arxiv.org/abs/1603.06560>`__, and `ASHA <https://openreview.net/forum?id=S1Y7OOlRZ>`__. Tune also includes a distributed implementation of `Population Based Training (PBT) <https://deepmind.com/blog/population-based-training-neural-networks>`__.
 
 .. tip:: The easiest scheduler to start with is the ``ASHAScheduler`` which will aggressively terminate low-performing trials.
 
@@ -48,11 +48,7 @@ When using schedulers, you may face compatibility issues, as shown in the below
    * - :ref:`Population Based Training <tune-scheduler-pbt>`
      - Yes
      - Not Compatible
-     - :doc:`Link </tune/examples/pbt_function>`
-   * - :ref:`Population Based Bandits <tune-scheduler-pb2>`
-     - Yes
-     - Not Compatible
-     - :doc:`Basic Example </tune/examples/pb2_example>`, :doc:`PPO example </tune/examples/pb2_ppo_example>`
+     - :doc:`Link </tune/examples/pbt_example>`
 
 .. _tune-scheduler-hyperband:
 
@@ -176,38 +172,6 @@ replay utility in practice.
 
 .. autoclass:: ray.tune.schedulers.PopulationBasedTrainingReplay
 
-
-.. _tune-scheduler-pb2:
-
-Population Based Bandits (PB2) (tune.schedulers.PB2)
--------------------------------------------------------------------
-
-Tune includes a distributed implementation of `Population Based Bandits (PB2) <https://arxiv.org/abs/2002.02518>`__. This can be enabled by setting the ``scheduler`` parameter of ``tune.run``, e.g.
-
-.. code-block:: python
-
-    pb2_scheduler = PB2(
-            time_attr='time_total_s',
-            metric='mean_accuracy',
-            mode='max',
-            perturbation_interval=600.0,
-            hyperparam_bounds={
-                "lr": [1e-3, 1e-5],
-                "alpha": [0.0, 1.0],
-            ...
-            })
-    tune.run( ... , scheduler=pb2_scheduler)
-
-This code builds upon PBT, with the main difference being that instead of using random perturbations, PB2 selects new hyperparameter configurations using a Gaussian Process model. 
-
-When the PB2 scheduler is enabled, each trial variant is treated as a member of the population. Periodically, top-performing trials are checkpointed (this requires your Trainable to support :ref:`save and restore <tune-checkpoint>`). Low-performing trials clone the checkpoints of top performers and perturb the configurations in the hope of discovering an even better variation.
-
-The primary motivation for PB2 is the ability to find promising hyperparamters with only a small population size. With that in mind, you can run this :doc:`PB2 PPO example </tune/examples/pb2_ppo_example>` to compare PB2 vs. PBT, with a population size of ``4`` (as in the paper). The example uses the ``BipedalWalker`` environment so does not require any additional licenses.
-
-
-.. autoclass:: ray.tune.schedulers.PB2
-
-
 .. _tune-scheduler-bohb:
 
 BOHB (tune.schedulers.HyperBandForBOHB)

diff --git a/doc/source/tune/examples/index.rst b/doc/source/tune/examples/index.rst
@@ -18,7 +18,6 @@ General Examples
 - :doc:`/tune/examples/pbt_example`: Example of using a Trainable class with PopulationBasedTraining scheduler.
 - :doc:`/tune/examples/pbt_function`: Example of using the function API with a PopulationBasedTraining scheduler.
 - :doc:`/tune/examples/pbt_ppo_example`: Example of optimizing a distributed RLlib algorithm (PPO) with the PopulationBasedTraining scheduler.
-- :doc:`/tune/examples/pb2_ppo_example`: Example of optimizing a distributed RLlib algorithm (PPO) with the PB2 scheduler. Uses a small population size of 4, so can train on a laptop.
 - :doc:`/tune/examples/logging_example`: Example of custom loggers and custom trial directory naming.
 
 Search Algorithm Examples

diff --git a/doc/source/tune/examples/pb2_example.rst b/doc/source/tune/examples/pb2_example.rst
diff --git a/doc/source/tune/examples/pb2_ppo_example.rst b/doc/source/tune/examples/pb2_ppo_example.rst
diff --git a/python/README-building-wheels.md b/python/README-building-wheels.md
@@ -1,4 +1,4 @@
-# Building manylinux1 wheels
+# Building manylinux2014 wheels
 
 **WARNING:** To cause everything to be rebuilt, this script will delete ALL changes to the
 repository, including both changes to tracked files, and ANY untracked files.
@@ -9,7 +9,7 @@ produce .whl files owned by root.
 Inside the root directory (i.e., one level above this python directory), run
 
 ```
-docker run -e TRAVIS_COMMIT=<commit_number_to_use> --rm -w /ray -v `pwd`:/ray -ti rayproject/arrow_linux_x86_64_base:python-3.8.0 /ray/python/build-wheel-manylinux1.sh
+docker run -e TRAVIS_COMMIT=<commit_number_to_use> --rm -w /ray -v `pwd`:/ray -ti quay.io/pypa/manylinux2014_x86_64  /ray/python/build-wheel-manylinux2014.sh
 ```
 
 The wheel files will be placed in the .whl directory.

diff --git a/python/build-wheel-manylinux1.sh → python/build-wheel-manylinux2014.sh b/python/build-wheel-manylinux1.sh → python/build-wheel-manylinux2014.sh
@@ -22,18 +22,25 @@ NUMPY_VERSIONS=("1.14.5"
                 "1.14.5"
                 "1.14.5")
 
-sudo apt-get install unzip
+yum -y install unzip zip sudo
+yum -y install java-1.8.0-openjdk java-1.8.0-openjdk-devel xz
+
 /ray/ci/travis/install-bazel.sh
+# Put bazel into the PATH if building Bazel from source
+# export PATH=/root/bazel-3.2.0/output:$PATH:/root/bin
+
+# If converting down to manylinux2010, the following configuration should
+# be set for bazel
+#echo "build --config=manylinux2010" >> /root/.bazelrc
+echo "build --incompatible_linkopts_to_linklibs" >> /root/.bazelrc
 
-# Put bazel into the PATH
-export PATH=$PATH:/root/bin
 
 # Install and use the latest version of Node.js in order to build the dashboard.
 set +x
 curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
 source "$HOME"/.nvm/nvm.sh
-nvm install $NODE_VERSION
-nvm use node
+nvm install "$NODE_VERSION"
+nvm use "$NODE_VERSION"
 
 # Build the dashboard so its static assets can be included in the wheel.
 # TODO(mfitton): switch this back when deleting old dashboard code.
@@ -66,7 +73,8 @@ for ((i=0; i<${#PYTHONS[@]}; ++i)); do
       exit 1
     fi
 
-    PATH=/opt/python/${PYTHON}/bin:$PATH /opt/python/"${PYTHON}"/bin/python setup.py bdist_wheel
+    PATH=/opt/python/${PYTHON}/bin:/root/bazel-3.2.0/output:$PATH \
+    /opt/python/"${PYTHON}"/bin/python setup.py bdist_wheel
     # In the future, run auditwheel here.
     mv dist/*.whl ../.whl/
   popd
@@ -76,6 +84,6 @@ done
 # hack, we should use auditwheel instead.
 for path in .whl/*.whl; do
   if [ -f "${path}" ]; then
-    mv "${path}" "${path//linux/manylinux1}"
+    mv "${path}" "${path//linux/manylinux2014}"
   fi
 done
diff --git a/python/ray/_raylet.pyx b/python/ray/_raylet.pyx
@@ -492,6 +492,10 @@ cdef execute_task(
                 core_worker.store_task_outputs(
                     worker, outputs, c_return_ids, returns)
         except Exception as error:
+            # If the debugger is enabled, drop into the remote pdb here.
+            if "RAY_PDB" in os.environ:
+                ray.util.pdb.post_mortem()
+
             if (<int>task_type == <int>TASK_TYPE_ACTOR_CREATION_TASK):
                 worker.mark_actor_init_failed(error)