[Dashboard] Turn on new dashboard by default pt 2

.gitignore

@@ -1,4 +1,5 @@
 # The build output should clearly not be checked in
+*test-output.xml
 /bazel-*
 /python/ray/core
 /python/ray/pickle5_files/
@@ -11,7 +12,7 @@
 /thirdparty/pkg/
 /build/java
 .jar
-
+/dashboard/client/build
 # Files generated by flatc should be ignored
 /src/ray/gcs/format/*_generated.h
 /src/ray/object_manager/format/*_generated.h

.travis.yml

@@ -448,9 +448,6 @@ script:
   # ray serve tests
   - if [ $RAY_CI_SERVE_AFFECTED == "1" ]; then ./ci/keep_alive bazel test --config=ci $(./scripts/bazel_export_options) --test_tag_filters=-jenkins_only python/ray/serve/...; fi
 
-  # ray dashboard tests
-  - if [ "$RAY_CI_DASHBOARD_AFFECTED" == "1" ]; then ./ci/keep_alive bazel test python/ray/dashboard/...; fi
-
   # ray new dashboard tests
   - if [ "$RAY_CI_DASHBOARD_AFFECTED" == "1" ]; then ./ci/keep_alive bazel test python/ray/new_dashboard/...; fi
 

BUILD.bazel

@@ -1786,8 +1786,7 @@ filegroup(
         "python/ray/core/generated/__init__.py",
         "python/ray/core/generated/ray/__init__.py",
         "python/ray/core/generated/ray/protocol/__init__.py",
-        "python/ray/dashboard/*.py",
-        "python/ray/dashboard/metrics_exporter/*.py",
+        "python/ray/new_dashboard/**/*.py",
         "python/ray/experimental/*.py",
         "python/ray/util/*.py",
         "python/ray/internal/*.py",

ci/travis/ci.sh

@@ -159,7 +159,6 @@ test_python() {
       -python/ray/tests:test_resource_demand_scheduler
       -python/ray/tests:test_stress  # timeout
       -python/ray/tests:test_stress_sharded  # timeout
-      -python/ray/tests:test_webui
     )
   fi
   if [ 0 -lt "${#args[@]}" ]; then  # Any targets to test?
@@ -208,7 +207,7 @@ build_dashboard_front_end() {
     { echo "WARNING: Skipping dashboard due to NPM incompatibilities with Windows"; } 2> /dev/null
   else
     (
-      cd ray/dashboard/client
+      cd ray/new_dashboard/client
       set +x  # suppress set -x since it'll get very noisy here
       . "${HOME}/.nvm/nvm.sh"
       nvm use --silent node
@@ -334,7 +333,7 @@ lint_bazel() {
 
 lint_web() {
   (
-    cd "${WORKSPACE_DIR}"/python/ray/dashboard/client
+    cd "${WORKSPACE_DIR}"/python/ray/new_dashboard/client
     set +x # suppress set -x since it'll get very noisy here
     . "${HOME}/.nvm/nvm.sh"
     install_npm_project

ci/travis/format.sh

@@ -39,12 +39,6 @@ check_command_exist yapf
 check_command_exist flake8
 check_command_exist mypy
 
-ver=$(yapf --version)
-if ! echo "$ver" | grep -q 0.23.0; then
-    echo "Wrong YAPF version installed: 0.23.0 is required, not $ver. $YAPF_DOWNLOAD_COMMAND_MSG"
-    exit 1
-fi
-
 # this stops git rev-parse from failing if we run this from the .git directory
 builtin cd "$(dirname "${BASH_SOURCE:-$0}")"
 

ci/travis/test-wheels.sh

@@ -22,7 +22,8 @@ if [ -z "${BUILD_DIR}" ]; then
 fi
 TEST_DIR="${BUILD_DIR}/python/ray/tests"
 TEST_SCRIPTS=("$TEST_DIR/test_microbenchmarks.py" "$TEST_DIR/test_basic.py")
-UI_TEST_SCRIPT="${BUILD_DIR}/python/ray/tests/test_webui.py"
+DASHBOARD_TEST_SCRIPT="${BUILD_DIR}/python/ray/tests/test_dashboard.py"
+
 
 function retry {
   local n=1
@@ -77,9 +78,7 @@ if [[ "$platform" == "linux" ]]; then
     for SCRIPT in "${TEST_SCRIPTS[@]}"; do
         retry "$PYTHON_EXE" "$SCRIPT"
     done
-
-    # Run the UI test to make sure that the packaged UI works.
-    retry "$PYTHON_EXE" "$UI_TEST_SCRIPT"
+    retry "$PYTHON_EXE" "$DASHBOARD_TEST_SCRIPT"
   done
 
   # Check that the other wheels are present.
@@ -118,12 +117,6 @@ elif [[ "$platform" == "macosx" ]]; then
     for SCRIPT in "${TEST_SCRIPTS[@]}"; do
       retry "$PYTHON_EXE" "$SCRIPT"
     done
-
-    if (( $(echo "$PY_MM >= 3.0" | bc) )); then
-      # Run the UI test to make sure that the packaged UI works.
-      retry "$PYTHON_EXE" "$UI_TEST_SCRIPT"
-    fi
-
   done
 elif [ "${platform}" = windows ]; then
   echo "WARNING: Wheel testing not yet implemented for Windows."

dashboard/agent.py

@@ -3,6 +3,7 @@
 import logging
 import logging.handlers
 import os
+import platform
 import sys
 import socket
 import json
@@ -38,6 +39,7 @@
 class DashboardAgent(object):
     def __init__(self,
                  redis_address,
+                 dashboard_agent_port,
                  redis_password=None,
                  temp_dir=None,
                  log_dir=None,
@@ -51,6 +53,7 @@ def __init__(self,
         self.redis_password = redis_password
         self.temp_dir = temp_dir
         self.log_dir = log_dir
+        self.dashboard_agent_port = dashboard_agent_port
         self.metrics_export_port = metrics_export_port
         self.node_manager_port = node_manager_port
         self.object_store_name = object_store_name
@@ -59,7 +62,8 @@ def __init__(self,
         assert self.node_id, "Empty node id (RAY_NODE_ID)."
         self.ip = ray._private.services.get_node_ip_address()
         self.server = aiogrpc.server(options=(("grpc.so_reuseport", 0), ))
-        self.grpc_port = self.server.add_insecure_port("[::]:0")
+        self.grpc_port = self.server.add_insecure_port(
+            f"[::]:{self.dashboard_agent_port}")
         logger.info("Dashboard agent grpc address: %s:%s", self.ip,
                     self.grpc_port)
         self.aioredis_client = None
@@ -186,6 +190,11 @@ async def _check_parent():
         required=True,
         type=int,
         help="The port to expose metrics through Prometheus.")
+    parser.add_argument(
+        "--dashboard-agent-port",
+        required=True,
+        type=int,
+        help="The port on which the dashboard agent will receive GRPCs.")
     parser.add_argument(
         "--node-manager-port",
         required=True,
@@ -247,35 +256,23 @@ async def _check_parent():
         format(dashboard_consts.LOGGING_ROTATE_BACKUP_COUNT))
     parser.add_argument(
         "--log-dir",
-        required=False,
+        required=True,
         type=str,
         default=None,
         help="Specify the path of log directory.")
     parser.add_argument(
         "--temp-dir",
-        required=False,
+        required=True,
         type=str,
         default=None,
         help="Specify the path of the temporary directory use by Ray process.")
 
     args = parser.parse_args()
     try:
-        if args.temp_dir:
-            temp_dir = "/" + args.temp_dir.strip("/")
-        else:
-            temp_dir = "/tmp/ray"
-        os.makedirs(temp_dir, exist_ok=True)
-
-        if args.log_dir:
-            log_dir = args.log_dir
-        else:
-            log_dir = os.path.join(temp_dir, "session_latest/logs")
-        os.makedirs(log_dir, exist_ok=True)
-
         if args.logging_filename:
             logging_handlers = [
                 logging.handlers.RotatingFileHandler(
-                    os.path.join(log_dir, args.logging_filename),
+                    os.path.join(args.log_dir, args.logging_filename),
                     maxBytes=args.logging_rotate_bytes,
                     backupCount=args.logging_rotate_backup_count)
             ]
@@ -288,9 +285,10 @@ async def _check_parent():
 
         agent = DashboardAgent(
             args.redis_address,
+            args.dashboard_agent_port,
             redis_password=args.redis_password,
-            temp_dir=temp_dir,
-            log_dir=log_dir,
+            temp_dir=args.temp_dir,
+            log_dir=args.log_dir,
             metrics_export_port=args.metrics_export_port,
             node_manager_port=args.node_manager_port,
             object_store_name=args.object_store_name,
@@ -304,7 +302,7 @@ async def _check_parent():
             args.redis_address, password=args.redis_password)
         traceback_str = ray.utils.format_error_message(traceback.format_exc())
         message = ("The agent on node {} failed with the following "
-                   "error:\n{}".format(os.uname()[1], traceback_str))
+                   "error:\n{}".format(platform.uname()[1], traceback_str))
         ray.utils.push_error_to_driver_through_redis(
             redis_client, ray_constants.DASHBOARD_AGENT_DIED_ERROR, message)
         raise e

dashboard/client/src/api.ts

@@ -60,7 +60,7 @@ type ProcessStats = {
     iowait: number;
   };
   cpuPercent: number;
-}
+};
 
 export type Worker = {
   pid: number;

dashboard/client/src/pages/dashboard/logical-view/Actor.tsx

@@ -195,9 +195,7 @@ const Actor: React.FC<ActorProps> = ({ actor }) => {
           </span>
         )}
       </Typography>
-      <ActorDetailsPane
-        actor={actor}
-      />
+      <ActorDetailsPane actor={actor} />
       {isFullActorInfo(actor) && (
         <React.Fragment>
           {actorCustomDisplay.length > 0 && (