Clone catapult and generate html files during installation. (ray-proj…

…ect#956)

* Clone catapult and generate static html during setup.

* Include UI files in installation.

* Fix directory to clone catapult to and fix linting.

* Use absolute path.

* Make sure we find a sufficiently new version of python2 when building wheels.

* Copy the trace_viewer_full.html file to the local directory if it is not present.

* Make sure wheels fail to build if UI is not included.

python/build-wheel-macos.sh

@@ -58,7 +58,7 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do
     $PIP_CMD install wheel
     # Add the correct Python to the path and build the wheel. This is only
     # needed so that the installation finds the cython executable.
-    PATH=$MACPYTHON_PY_PREFIX/$PY_MM/bin:$PATH $PYTHON_EXE setup.py bdist_wheel
+    INCLUDE_UI=1 PATH=$MACPYTHON_PY_PREFIX/$PY_MM/bin:$PATH $PYTHON_EXE setup.py bdist_wheel
     mv dist/*.whl ../.whl/
   popd
 done

python/build-wheel-manylinux1.sh

@@ -6,6 +6,12 @@ echo 10
 EOF
 chmod +x /usr/bin/nproc
 
+# Remove this old Python 2.4.3 executable, and make the "python2" command find
+# a newer version of Python. We need this for autogenerating some files for the
+# UI.
+rm -f /usr/bin/python2
+ln -s /opt/python/cp27-cp27m/bin/python2 /usr/bin/python2
+
 mkdir .whl
 for PYTHON in cp27-cp27mu cp33-cp33m cp34-cp34m cp35-cp35m cp36-cp36m; do
   # The -f flag is passed twice to also run git clean in the arrow subdirectory.
@@ -16,7 +22,7 @@ for PYTHON in cp27-cp27mu cp33-cp33m cp34-cp34m cp35-cp35m cp36-cp36m; do
     # Fix the numpy version because this will be the oldest numpy version we can
     # support.
     /opt/python/${PYTHON}/bin/pip install numpy==1.10.4 cython
-    PATH=/opt/python/${PYTHON}/bin:$PATH /opt/python/${PYTHON}/bin/python setup.py bdist_wheel
+    INCLUDE_UI=1 PATH=/opt/python/${PYTHON}/bin:$PATH /opt/python/${PYTHON}/bin/python setup.py bdist_wheel
     # In the future, run auditwheel here.
     mv dist/*.whl ../.whl/
   popd

python/ray/experimental/ui.py

@@ -4,7 +4,6 @@
 import pprint
 import ray
 import shutil
-import subprocess
 import tempfile
 import time
 
@@ -289,65 +288,6 @@ def _get_temp_file_path(**kwargs):
     return os.path.relpath(temp_file_path)
 
 
-# Helper function that ensures that catapult is cloned to the correct location
-# and that the HTML files required for task trace embedding are in the same
-# directory as the web UI.
-def _setup_trace_dependencies():
-    catapult_home = "/tmp/ray/catapult"
-    catapult_commit = "33a9271eb3cf5caf925293ec6a4b47c94f1ac968"
-    try:
-        # Check if we're inside a git repo
-        cmd = ["git",
-               "-C",
-               catapult_home,
-               "rev-parse",
-               "--is-inside-work-tree"]
-        subprocess.check_call(cmd)
-
-    except subprocess.CalledProcessError:
-        # Error on non-zero exit code (e.g. - ".git not found")
-        if not os.path.exists(os.path.join(catapult_home)):
-            print(
-                "Cloning catapult to {} (this may take a while...)".format(
-                    catapult_home))
-            cmd = ["git",
-                   "clone",
-                   "https://github.com/catapult-project/catapult.git",
-                   catapult_home]
-            subprocess.check_call(cmd)
-
-        # Checks out the commit associated with allowing different arrow
-        # colors. This can and should be removed after catapult's next
-        # release.
-        print("Checking out commit {}.".format(catapult_commit))
-        cmd = ["git", "-C", catapult_home, "checkout", catapult_commit]
-        subprocess.check_call(cmd)
-
-    # Path to the embedded trace viewer HTML file.
-    embedded_trace_path = os.path.join(catapult_home,
-                                       "tracing",
-                                       "bin",
-                                       "index.html")
-    # Checks that the trace viewer renderer file exists, generates it if it
-    # doesn't.
-    if not os.path.exists("trace_viewer_full.html"):
-        vulcanize_bin = os.path.join(catapult_home,
-                                     "tracing",
-                                     "bin",
-                                     "vulcanize_trace_viewer")
-        # TODO(rkn): The vulcanize_trace_viewer script currently requires
-        # Python 2. Remove this dependency.
-        cmd = ["python2",
-               vulcanize_bin,
-               "--config",
-               "chrome",
-               "--output",
-               "trace_viewer_full.html"]
-        subprocess.check_call(cmd)
-
-    return catapult_home, embedded_trace_path
-
-
 def task_timeline():
     path_input = widgets.Button(description="View task timeline")
 
@@ -381,6 +321,14 @@ def task_timeline():
     display(widgets.HBox([label_options, breakdown_opt]))
     display(path_input)
 
+    # Check that the trace viewer renderer file is present, and copy it to the
+    # current working directory if it is not present.
+    if not os.path.exists("trace_viewer_full.html"):
+        shutil.copy(
+            os.path.join(os.path.dirname(os.path.abspath(__file__)),
+                         "../core/src/catapult_files/trace_viewer_full.html"),
+            "trace_viewer_full.html")
+
     def handle_submit(sender):
         json_tmp = tempfile.mktemp() + ".json"
 
@@ -425,9 +373,9 @@ def handle_submit(sender):
 
         print("Opening html file in browser...")
 
-        # Check that the catapult repo is cloned to the correct location
-        print(_setup_trace_dependencies())
-        catapult_home, trace_viewer_path = _setup_trace_dependencies()
+        trace_viewer_path = os.path.join(
+            os.path.dirname(os.path.abspath(__file__)),
+            "../core/src/catapult_files/index.html")
 
         html_file_path = _get_temp_file_path(suffix=".html")
         json_file_path = _get_temp_file_path(suffix=".json")

python/setup.py

@@ -23,6 +23,8 @@
     "ray/core/src/local_scheduler/local_scheduler",
     "ray/core/src/local_scheduler/liblocal_scheduler_library.so",
     "ray/core/src/global_scheduler/global_scheduler",
+    "ray/core/src/catapult_files/index.html",
+    "ray/core/src/catapult_files/trace_viewer_full.html",
     "ray/WebUI.ipynb"
 ]
 

src/thirdparty/build_thirdparty.sh

@@ -42,6 +42,9 @@ else
   FLATBUFFERS_HOME=""
 fi
 
+# Clone catapult and build the static HTML needed for the UI.
+bash "$TP_DIR/build_ui.sh"
+
 echo "building arrow"
 cd $TP_DIR/arrow/cpp
 mkdir -p $TP_DIR/arrow/cpp/build

src/thirdparty/build_ui.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+set -x
+
+# Cause the script to exit if a single command fails.
+set -e
+
+TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
+
+CATAPULT_COMMIT=33a9271eb3cf5caf925293ec6a4b47c94f1ac968
+CATAPULT_HOME=$TP_DIR/catapult
+VULCANIZE_BIN=$CATAPULT_HOME/tracing/bin/vulcanize_trace_viewer
+
+CATAPULT_FILES=$TP_DIR/../../python/ray/core/src/catapult_files
+
+# This is where we will copy the files that need to be packaged with the wheels.
+mkdir -p $CATAPULT_FILES
+
+if [ ! type python2 > /dev/null ]; then
+  echo "cannot properly set up UI without a python2 executable"
+  if [ "$INCLUDE_UI" == "1" ]; then
+    # Since the UI is explicitly supposed to be included, fail here.
+    exit 1
+  else
+    # Let installation continue without building the UI.
+    exit 0
+  fi
+fi
+
+# Download catapult and use it to autogenerate some static html if it isn't
+# already present.
+if [ ! -d $CATAPULT_HOME ]; then
+  echo "setting up catapult"
+  # Clone the catapult repository.
+  git clone https://github.com/catapult-project/catapult.git $CATAPULT_HOME
+  # Check out the appropriate commit from catapult.
+  pushd $CATAPULT_HOME
+    git checkout $CATAPULT_COMMIT
+  popd
+
+  python2 $VULCANIZE_BIN --config chrome --output $CATAPULT_FILES/trace_viewer_full.html
+  cp $CATAPULT_HOME/tracing/bin/index.html $CATAPULT_FILES/index.html
+fi