New web UI. (ray-project#176)

* remove node.js webui

* temp commit

* flesh out web ui

* add documentation

* add ray timeline

* Small changes to documentation and formatting.

.travis/check-git-clang-format-output.sh

@@ -8,7 +8,7 @@ else
   base_commit="$TRAVIS_BRANCH"
   echo "Running clang-format against branch $base_commit, with hash $(git rev-parse $base_commit)"
 fi
-output="$(.travis/git-clang-format --binary clang-format-3.8 --commit $base_commit --diff --exclude '(.*thirdparty/|.*redismodule.h)')"
+output="$(.travis/git-clang-format --binary clang-format-3.8 --commit $base_commit --diff --exclude '(.*thirdparty/|.*redismodule.h|.*webui*)')"
 if [ "$output" == "no modified files to format" ] || [ "$output" == "clang-format did not modify any files" ] ; then
   echo "clang-format passed."
   exit 0

webui/.eslintrc.json

@@ -0,0 +1,16 @@
+{
+  "extends": ["eslint:recommended", "google"],
+  "env": {
+    "browser": true
+  },
+  "plugins": [
+    "html"
+  ],
+  "rules": {
+    "no-var": "off",
+    "new-cap": ["error", { "capIsNewExceptions": ["Polymer"] }]
+  },
+  "globals": {
+    "Polymer": true
+  }
+}

webui/.gitattributes

@@ -0,0 +1 @@
+* text=auto

webui/.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,33 @@
+<!-- Instructions: https://github.com/PolymerElements/polymer-starter-kit/CONTRIBUTING.md#filing-issues -->
+### Description
+<!-- Example: The `paper-foo` element causes the page to turn pink when clicked. -->
+
+### Expected outcome
+
+<!-- Example: The page stays the same color. -->
+
+### Actual outcome
+
+<!-- Example: The page turns pink. -->
+
+### Live Demo
+<!-- Example: https://jsbin.com/cagaye/edit?html,output -->
+
+### Steps to reproduce
+
+<!-- Example
+1. Put a `paper-foo` element in the page.
+2. Open the page in a web browser.
+3. Click the `paper-foo` element.
+-->
+
+### Browsers Affected
+<!-- Check all that apply -->
+- [ ] Chrome
+- [ ] Firefox
+- [ ] Safari 9
+- [ ] Safari 8
+- [ ] Safari 7
+- [ ] Edge
+- [ ] IE 11
+- [ ] IE 10

webui/.gitignore

@@ -0,0 +1,3 @@
+bower_components/
+build/
+node_modules/

webui/.travis.yml

@@ -0,0 +1,19 @@
+language: node_js
+sudo: required
+dist: trusty
+addons:
+  firefox: latest
+  apt:
+    sources:
+      - google-chrome
+    packages:
+      - google-chrome-stable
+node_js:
+  - '6'
+  - '5'
+  - '4'
+before_script:
+  - npm install -g bower polymer-cli
+  - bower install
+script:
+  - xvfb-run npm test

webui/README.md

@@ -0,0 +1,86 @@
+# The Ray Web UI
+
+This is Ray's Web UI. It consists of two components:
+
+* The **frontend** is a [Polymer](https://www.polymer-project.org/1.0/) app that
+  uses [google-charts](https://elements.polymer-project.org/elements/google-chart)
+  for visualization.
+* The **backend** is a Python 3 websocket server (see `backend/ray_ui.py`) that
+  connects to Redis and potentially Ray.
+
+### Prerequisites
+
+The Ray Web UI requires Python 3.
+
+Install [polymer-cli](https://github.com/Polymer/polymer-cli):
+
+    pip install aioredis websockets
+    npm install -g polymer-cli
+
+### Setup
+
+The following must be done once.
+
+    cd webui
+    bower install
+
+### Start the backend
+
+First start Ray and note down the address of the Redis server. Then run
+
+    cd webui/backend
+    python ray_ui.py --redis-address 127.0.0.1:6379 --port 8888
+
+where you substitute your Redis address appropriately.
+
+### Start the frontend development server
+
+To start the front end, run the following.
+
+    cd webui
+    polymer serve --open
+
+The web UI can then be accessed at `http://localhost:8080`.
+
+### Build
+
+This command performs HTML, CSS, and JS minification on the application
+dependencies, and generates a service-worker.js file with code to pre-cache the
+dependencies based on the entrypoint and fragments specified in `polymer.json`.
+The minified files are output to the `build/unbundled` folder, and are suitable
+for serving from a HTTP/2+Push compatible server.
+
+In addition the command also creates a fallback `build/bundled` folder,
+generated using fragment bundling, suitable for serving from non
+H2/push-compatible servers or to clients that do not support H2/Push.
+
+    polymer build
+
+### Preview the build
+
+This command serves the minified version of the app at `http://localhost:8080`
+in an unbundled state, as it would be served by a push-compatible server:
+
+    polymer serve build/unbundled
+
+This command serves the minified version of the app at `http://localhost:8080`
+generated using fragment bundling:
+
+    polymer serve build/bundled
+
+### Run tests
+
+This command will run
+[Web Component Tester](https://github.com/Polymer/web-component-tester) against
+the browsers currently installed on your machine.
+
+    polymer test
+
+### Adding a new view
+
+You can extend the app by adding more views that will be demand-loaded e.g.
+based on the route, or to progressively render non-critical sections of the
+application.  Each new demand-loaded fragment should be added to the list of
+`fragments` in the included `polymer.json` file.  This will ensure those
+components and their dependencies are added to the list of pre-cached components
+(and will have bundles created in the fallback `bundled` build).

webui/backend/ray_ui.py

@@ -0,0 +1,115 @@
+import aioredis
+import argparse
+import asyncio
+import binascii
+from collections import defaultdict
+import json
+import websockets
+
+parser = argparse.ArgumentParser(description="parse information for the web ui")
+parser.add_argument("--port", type=int, help="port to use for the web ui")
+parser.add_argument("--redis-address", required=True, type=str, help="the address to use for redis")
+
+loop = asyncio.get_event_loop()
+
+IDENTIFIER_LENGTH = 20
+
+def hex_identifier(identifier):
+  return binascii.hexlify(identifier).decode()
+
+def identifier(hex_identifier):
+  return binascii.unhexlify(hex_identifier)
+
+def key_to_hex_identifier(key):
+  return hex_identifier(key[(key.index(b":") + 1):(key.index(b":") + IDENTIFIER_LENGTH + 1)])
+
+def key_to_hex_identifiers(key):
+  # Extract worker_id and task_id from key of the form prefix:worker_id:task_id.
+  offset = key.index(b":") + 1
+  worker_id = hex_identifier(key[offset:(offset + IDENTIFIER_LENGTH)])
+  offset += IDENTIFIER_LENGTH + 1
+  task_id = hex_identifier(key[offset:(offset + IDENTIFIER_LENGTH)])
+  return worker_id, task_id
+
+
+async def hello(websocket, path):
+  conn = await aioredis.create_connection((redis_host, redis_port), loop=loop)
+
+  while True:
+    command = json.loads(await websocket.recv())
+    print("received command {}".format(command))
+
+    if command["command"] == "get-workers":
+      result = []
+      workers = await conn.execute("keys", "WorkerInfo:*")
+      for key in workers:
+        content = await conn.execute("hgetall", key)
+        worker_id = key_to_hex_identifier(key)
+        result.append({"worker": worker_id, "export_counter": int(content[1])})
+      await websocket.send(json.dumps(result))
+    elif command["command"] == "get-clients":
+      result = []
+      clients = await conn.execute("keys", "CL:*")
+      for key in clients:
+        content = await conn.execute("hgetall", key)
+        result.append({"client": hex_identifier(content[1]),
+                       "node_ip_address": content[3].decode(),
+                       "client_type": content[5].decode()})
+      await websocket.send(json.dumps(result))
+    elif command["command"] == "get-objects":
+      result = []
+      objects = await conn.execute("keys", "OI:*")
+      for key in objects:
+        content = await conn.execute("hgetall", key)
+        result.append({"object_id": hex_identifier(content[1]),
+                       "hash": hex_identifier(content[3]),
+                       "data_size": content[5].decode()})
+      await websocket.send(json.dumps(result))
+    elif command["command"] == "get-object-info":
+      # TODO(pcm): Get the object here (have to connect to ray) and ship content
+      # and type back to webclient. One challenge here is that the naive
+      # implementation will block the web ui backend, which is not ok if it is
+      # serving multiple users.
+      await websocket.send(json.dumps({"object_id": "none"}))
+    elif command["command"] == "get-tasks":
+      result = []
+      tasks = await conn.execute("keys", "TT:*")
+      for key in tasks:
+        content = await conn.execute("hgetall", key)
+        result.append({"task_id": key_to_hex_identifier(key),
+                       "state": int(content[1]),
+                       "node_id": hex_identifier(content[3])})
+      await websocket.send(json.dumps(result))
+    elif command["command"] == "get-timeline":
+      tasks = defaultdict(list)
+      for key in await conn.execute("keys", "event_log:*"):
+        worker_id, task_id = key_to_hex_identifiers(key)
+        content = await conn.execute("lrange", key, "0", "-1")
+        data = json.loads(content[0].decode())
+        begin_and_end_time = [timestamp for (timestamp, task, kind, info) in data if task == "ray:task"]
+        tasks[worker_id].append({"task_id": task_id,
+                                 "start_task": min(begin_and_end_time),
+                                 "end_task": max(begin_and_end_time)})
+      await websocket.send(json.dumps(tasks))
+    elif command["command"] == "get-events":
+      result = []
+      for key in await conn.execute("keys", "event_log:*"):
+        worker_id, task_id = key_to_hex_identifiers(key)
+        answer = await conn.execute("lrange", key, "0", "-1")
+        assert len(answer) == 1
+        events = json.loads(answer[0].decode())
+        result.extend([{"worker_id": worker_id,
+                        "task_id": task_id,
+                        "time": event[0],
+                        "type": event[1]} for event in events])
+      await websocket.send(json.dumps(result))
+
+if __name__ == "__main__":
+  args = parser.parse_args()
+  redis_address = args.redis_address.split(":")
+  redis_host, redis_port = redis_address[0], int(redis_address[1])
+
+  start_server = websockets.serve(hello, "localhost", args.port)
+
+  loop.run_until_complete(start_server)
+  loop.run_forever()

webui/bower.json

@@ -0,0 +1,26 @@
+{
+  "name": "polymer-starter-kit",
+  "authors": [
+    "The Polymer Authors"
+  ],
+  "license": "https://polymer.github.io/LICENSE.txt",
+  "dependencies": {
+    "app-layout": "PolymerElements/app-layout#^0.10.0",
+    "app-route": "PolymerElements/app-route#^0.9.0",
+    "iron-flex-layout": "PolymerElements/iron-flex-layout#^1.0.0",
+    "iron-icon": "PolymerElements/iron-icon#^1.0.0",
+    "iron-iconset-svg": "PolymerElements/iron-iconset-svg#^1.0.0",
+    "iron-localstorage": "PolymerElements/iron-localstorage#^1.0.0",
+    "iron-media-query": "PolymerElements/iron-media-query#^1.0.0",
+    "iron-pages": "PolymerElements/iron-pages#^1.0.0",
+    "iron-selector": "PolymerElements/iron-selector#^1.0.0",
+    "paper-icon-button": "PolymerElements/paper-icon-button#~1.1.0",
+    "polymer": "Polymer/polymer#^1.6.0",
+    "vaadin-grid": "^1.2.1",
+    "google-chart": "^1.1.1"
+  },
+  "devDependencies": {
+    "web-component-tester": "^4.0.0"
+  },
+  "private": true
+}