[xray] Integrate worker.py with raylet. (ray-project#1810)

* Integrate worker with raylet.

* Begin allowing worker to attach to cluster.

* Fix linting and documentation.

* Fix linting.

* Comment tests back in.

* Fix type of worker command.

* Remove xray python files and tests.

* Fix from rebase.

* Add test.

* Copy over raylet executable.

* Small cleanup.

.travis.yml

@@ -104,7 +104,6 @@ install:
   - bash ../../../src/ray/test/run_gcs_tests.sh
   # Raylet tests.
   - bash ../../../src/ray/test/run_object_manager_tests.sh
-  - bash ../../../src/ray/test/run_task_test.sh
   - ./src/ray/raylet/task_test
   - ./src/ray/raylet/worker_pool_test
   - ./src/ray/raylet/lineage_cache_test
@@ -123,6 +122,8 @@ script:
   - python python/ray/local_scheduler/test/test.py
   - python python/ray/global_scheduler/test/test.py
 
+  - python -m pytest test/xray_test.py
+
   - python test/runtest.py
   - python test/array_test.py
   - python test/actor_test.py

doc/source/conf.py

@@ -38,7 +38,8 @@
                 "ray.core.generated.TaskInfo",
                 "ray.core.generated.TaskReply",
                 "ray.core.generated.ResultTableReply",
-                "ray.core.generated.TaskExecutionDependencies"]
+                "ray.core.generated.TaskExecutionDependencies",
+                "ray.core.generated.ClientTableData"]
 for mod_name in MOCK_MODULES:
   sys.modules[mod_name] = mock.Mock()
 

python/ray/scripts/scripts.py

@@ -86,11 +86,13 @@ def cli():
               help="enable support for huge pages in the object store")
 @click.option("--autoscaling-config", required=False, type=str,
               help="the file that contains the autoscaling config")
+@click.option("--use-raylet", is_flag=True, default=False,
+              help="use the raylet code path, this is not supported yet")
 def start(node_ip_address, redis_address, redis_port, num_redis_shards,
           redis_max_clients, redis_shard_ports, object_manager_port,
           object_store_memory, num_workers, num_cpus, num_gpus, resources,
           head, no_ui, block, plasma_directory, huge_pages,
-          autoscaling_config):
+          autoscaling_config, use_raylet):
     # Convert hostnames to numerical IP address.
     if node_ip_address is not None:
         node_ip_address = services.address_to_ip(node_ip_address)
@@ -161,7 +163,8 @@ def start(node_ip_address, redis_address, redis_port, num_redis_shards,
             include_webui=(not no_ui),
             plasma_directory=plasma_directory,
             huge_pages=huge_pages,
-            autoscaling_config=autoscaling_config)
+            autoscaling_config=autoscaling_config,
+            use_raylet=use_raylet)
         print(address_info)
         print("\nStarted Ray on this node. You can add additional nodes to "
               "the cluster by calling\n\n"
@@ -227,7 +230,8 @@ def start(node_ip_address, redis_address, redis_port, num_redis_shards,
             redirect_output=True,
             resources=resources,
             plasma_directory=plasma_directory,
-            huge_pages=huge_pages)
+            huge_pages=huge_pages,
+            use_raylet=use_raylet)
         print(address_info)
         print("\nStarted Ray on this node. If you wish to terminate the "
               "processes that have been started, run\n\n"
@@ -242,7 +246,7 @@ def start(node_ip_address, redis_address, redis_port, num_redis_shards,
 @click.command()
 def stop():
     subprocess.call(["killall global_scheduler plasma_store plasma_manager "
-                     "local_scheduler"], shell=True)
+                     "local_scheduler raylet"], shell=True)
 
     # Find the PID of the monitor process and kill it.
     subprocess.call(["kill $(ps aux | grep monitor.py | grep -v grep | "