[Dashboard] Add initial version of new dashboard (ray-project#5730)

BUILD.bazel

@@ -796,9 +796,6 @@ filegroup(
         "python/ray/core/generated/ray/__init__.py",
         "python/ray/core/generated/ray/protocol/__init__.py",
         "python/ray/dashboard/dashboard.py",
-        "python/ray/dashboard/index.html",
-        "python/ray/dashboard/res/main.css",
-        "python/ray/dashboard/res/main.js",
         "python/ray/experimental/*.py",
         "python/ray/internal/*.py",
         "python/ray/projects/*.py",

ci/travis/install-dependencies.sh

@@ -69,3 +69,9 @@ else
   echo "Unrecognized environment."
   exit 1
 fi
+
+if [[ "$PYTHON" == "3.5" ]] || [[ "$MAC_WHEELS" == "1" ]]; then
+  # Install the latest version of Node.js in order to build the dashboard.
+  source $HOME/.nvm/nvm.sh
+  nvm install node
+fi

ci/travis/install-ray.sh

@@ -17,6 +17,12 @@ elif [[ "$PYTHON" == "3.5" ]]; then
   export PATH="$HOME/miniconda/bin:$PATH"
 
   pushd "$ROOT_DIR/../../python"
+    pushd ray/dashboard/client
+      source $HOME/.nvm/nvm.sh
+      nvm use node
+      npm ci
+      npm run build
+    popd
     python setup.py install --user
   popd
 

ci/travis/test-wheels.sh

@@ -22,6 +22,7 @@ else
 fi
 
 TEST_SCRIPT="tests/test_microbenchmarks.py"
+UI_TEST_SCRIPT="tests/test_webui.py"
 
 if [[ "$platform" == "linux" ]]; then
   # First test Python 2.7.
@@ -62,6 +63,10 @@ if [[ "$platform" == "linux" ]]; then
   INSTALLED_RAY_DIRECTORY=$(dirname "$($PYTHON_EXE -u -c "import ray; print(ray.__file__)" | tail -n1)")
   $PYTHON_EXE -m pytest -v "$INSTALLED_RAY_DIRECTORY/$TEST_SCRIPT"
 
+  # Run the UI test to make sure that the packaged UI works.
+  $PIP_CMD install -q aiohttp psutil requests setproctitle
+  $PYTHON_EXE -m pytest -v "$INSTALLED_RAY_DIRECTORY/$UI_TEST_SCRIPT"
+
   # Check that the other wheels are present.
   NUMBER_OF_WHEELS=$(ls -1q "$ROOT_DIR"/../../.whl/*.whl | wc -l)
   if [[ "$NUMBER_OF_WHEELS" != "4" ]]; then
@@ -98,6 +103,12 @@ elif [[ "$platform" == "macosx" ]]; then
     # Run a simple test script to make sure that the wheel works.
     INSTALLED_RAY_DIRECTORY=$(dirname "$($PYTHON_EXE -u -c "import ray; print(ray.__file__)" | tail -n1)")
     $PYTHON_EXE -m pytest -v "$INSTALLED_RAY_DIRECTORY/$TEST_SCRIPT"
+
+    if (( $(echo "$PY_MM >= 3.0" | bc) )); then
+      # Run the UI test to make sure that the packaged UI works.
+      $PIP_CMD install -q aiohttp psutil requests setproctitle
+      $PYTHON_EXE -m pytest -v "$INSTALLED_RAY_DIRECTORY/$UI_TEST_SCRIPT"
+    fi
   done
 else
   echo "Unrecognized environment."

doc/source/installation.rst

@@ -48,7 +48,8 @@ Building Ray from Source
 
 Installing from ``pip`` should be sufficient for most Ray users.
 
-However, should you need to build from source, follow instructions below for both Linux and MacOS.
+However, should you need to build from source, follow instructions below for
+both Linux and MacOS.
 
 Dependencies
 ~~~~~~~~~~~~
@@ -80,7 +81,6 @@ For MacOS, run the following commands:
 
   pip install cython==0.29.0
 
-
 If you are using Anaconda, you may also need to run the following.
 
 .. code-block:: bash
@@ -100,9 +100,35 @@ Ray can be built from the repository as follows.
   # Install Bazel.
   ray/ci/travis/install-bazel.sh
 
+  # Optionally build the dashboard (requires Node.js, see below for more information).
+  pushd ray/python/ray/dashboard/client
+  npm ci
+  npm run build
+  popd
+
+  # Install Ray.
   cd ray/python
   pip install -e . --verbose  # Add --user if you see a permission denied error.
 
+
+[Optional] Dashboard support
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you would like to use the dashboard, you will additionally need to install
+`Node.js`_ and build the dashboard before installing Ray. The relevant build
+steps are included in the installation instructions above.
+
+.. _`Node.js`: https://nodejs.org/
+
+The dashboard requires Python 3, and can be enabled by setting
+``include_webui=True`` during initialization, i.e.
+
+.. code-block:: python
+
+  import ray
+  ray.init(include_webui=True)
+
+
 Docker Source Images
 --------------------
 

python/build-wheel-macos.sh

@@ -38,6 +38,10 @@ NUMPY_VERSIONS=("1.14.5"
 mkdir -p $DOWNLOAD_DIR
 mkdir -p .whl
 
+# Use the latest version of Node.js in order to build the dashboard.
+source $HOME/.nvm/nvm.sh
+nvm use node
+
 for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do
   PY_VERSION=${PY_VERSIONS[i]}
   PY_INST=${PY_INSTS[i]}
@@ -58,10 +62,16 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do
   PIP_CMD="$(dirname $PYTHON_EXE)/pip$PY_MM"
 
   pushd /tmp
-    # Install latest version of pip to avoid brownouts
+    # Install latest version of pip to avoid brownouts.
     curl https://bootstrap.pypa.io/get-pip.py | $PYTHON_EXE
   popd
 
+  # Build the dashboard so its static assets can be included in the wheel.
+  pushd python/ray/dashboard/client
+    npm ci
+    npm run build
+  popd
+
   pushd python
     # Setuptools on CentOS is too old to install arrow 0.9.0, therefore we upgrade.
     $PIP_CMD install --upgrade setuptools

python/build-wheel-manylinux1.sh

@@ -35,14 +35,28 @@ export PATH=$PATH:/root/bin
 rm -f /usr/bin/python2
 ln -s /opt/python/cp27-cp27m/bin/python2 /usr/bin/python2
 
+# Install and use the latest version of Node.js in order to build the dashboard.
+curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
+source $HOME/.nvm/nvm.sh
+nvm install node
+nvm use node
+
 mkdir .whl
 for ((i=0; i<${#PYTHONS[@]}; ++i)); do
   PYTHON=${PYTHONS[i]}
   NUMPY_VERSION=${NUMPY_VERSIONS[i]}
+
   # The -f flag is passed twice to also run git clean in the arrow subdirectory.
   # The -d flag removes directories. The -x flag ignores the .gitignore file,
   # and the -e flag ensures that we don't remove the .whl directory.
   git clean -f -f -x -d -e .whl
+
+  # Build the dashboard so its static assets can be included in the wheel.
+  pushd python/ray/dashboard/client
+    npm ci
+    npm run build
+  popd
+
   pushd python
     # Fix the numpy version because this will be the oldest numpy version we can
     # support.

python/ray/autoscaler/aws/development-example.yaml

@@ -88,14 +88,18 @@ setup_commands:
     # Install basics.
     - sudo apt-get update
     - sudo apt-get install -y build-essential curl unzip
+    # Install Node.js in order to build the dashboard.
+    - curl -sL https://deb.nodesource.com/setup_12.x | sudo -E bash
+    - sudo apt-get install -y nodejs
     # Install Anaconda.
     - wget https://repo.continuum.io/archive/Anaconda3-5.0.1-Linux-x86_64.sh || true
     - bash Anaconda3-5.0.1-Linux-x86_64.sh -b -p $HOME/anaconda3 || true
     - echo 'export PATH="$HOME/anaconda3/bin:$PATH"' >> ~/.bashrc
     # Build Ray.
     - git clone https://github.com/ray-project/ray || true
     - ray/ci/travis/install-bazel.sh
-    - pip install boto3==1.4.8 cython==0.29.0
+    - cd ray/python/ray/dashboard/client; npm ci; npm run build
+    - pip install boto3==1.4.8 cython==0.29.0 aiohttp psutil setproctitle
     - cd ray/python; pip install -e . --verbose
 
 # Custom commands that will be run on the head node after common setup.
@@ -107,7 +111,7 @@ worker_setup_commands: []
 # Command to start ray on the head node. You don't need to change this.
 head_start_ray_commands:
     - ray stop
-    - ulimit -n 65536; ray start --head --num-redis-shards=10 --redis-port=6379 --autoscaling-config=~/ray_bootstrap_config.yaml
+    - ulimit -n 65536; ray start --head --include-webui --num-redis-shards=10 --redis-port=6379 --autoscaling-config=~/ray_bootstrap_config.yaml
 
 # Command to start ray on worker nodes. You don't need to change this.
 worker_start_ray_commands:

python/ray/dashboard/client/.gitignore

@@ -0,0 +1,23 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+
+# testing
+/coverage
+
+# production
+/build
+
+# misc
+.DS_Store
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local
+
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*

python/ray/dashboard/client/README.rst

@@ -0,0 +1,65 @@
+This project was bootstrapped with `Create React App
+<https://github.com/facebook/create-react-app>`__.
+
+Available Scripts
+-----------------
+
+In the project directory, you can run:
+
+``npm start``
+~~~~~~~~~~~~~
+
+Runs the app in the development mode. Open `http://localhost:3000
+<http://localhost:3000>`__ to view it in the browser.
+
+The page will reload if you make edits. You will also see any lint errors in the
+console.
+
+``npm test``
+~~~~~~~~~~~~
+
+Launches the test runner in the interactive watch mode. See the section about
+`running tests
+<https://facebook.github.io/create-react-app/docs/running-tests>`__ for more
+information.
+
+``npm run build``
+~~~~~~~~~~~~~~~~~
+
+Builds the app for production to the ``build`` folder. It correctly bundles
+React in production mode and optimizes the build for the best performance.
+
+The build is minified and the filenames include the hashes. Your app is ready to
+be deployed!
+
+See the section about `deployment
+<https://facebook.github.io/create-react-app/docs/deployment>`__ for more
+information.
+
+``npm run eject``
+~~~~~~~~~~~~~~~~~
+
+Note: this is a one-way operation. Once you ``eject``, you can’t go back!
+
+If you aren’t satisfied with the build tool and configuration choices, you can
+``eject`` at any time. This command will remove the single build dependency from
+your project.
+
+Instead, it will copy all the configuration files and the transitive
+dependencies (Webpack, Babel, ESLint, etc) right into your project so you have
+full control over them. All of the commands except ``eject`` will still work,
+but they will point to the copied scripts so you can tweak them. At this point
+you’re on your own.
+
+You don’t have to ever use ``eject``. The curated feature set is suitable for
+small and middle deployments, and you shouldn’t feel obligated to use this
+feature. However we understand that this tool wouldn’t be useful if you couldn’t
+customize it when you are ready for it.
+
+Learn More
+----------
+
+You can learn more in the `Create React App documentation
+<https://facebook.github.io/create-react-app/docs/getting-started>`__.
+
+To learn React, check out the `React documentation <https://reactjs.org/>`__.