[asv] Pushing to s3 (#2246)

Author: richardliaw

python/README-benchmarks.rst

@@ -7,4 +7,23 @@ You can run the benchmark suite by doing the following:
 2. Run ``asv dev`` in this directory.
 
 To run ASV inside docker, you can use the following command:
-`docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA bash -c '/ray/test/jenkins_tests/run_asv.sh'`
+``docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA bash -c '/ray/test/jenkins_tests/run_asv.sh'``
+
+Visualizing Benchmarks
+======================
+
+To visualize benchmarks, you must copy the S3 bucket down to `$RAY_DIR/python`. Assuming asv is installed,
+
+.. code-block::
+
+  cd $RAY_DIR/python
+  aws s3 sync s3://$BUCKET/ASV/ .
+
+Then, you can run:
+
+.. code-block::
+
+  asv publish --no-pull
+  asv preview
+
+This creates the directory and then launches a server.

python/asv.conf.json

@@ -11,7 +11,7 @@
 
     // The URL or local path of the source code repository for the
     // project being benchmarked
-    "repo": ".",
+    "repo": "../",
 
     // List of branches to benchmark. If not provided, defaults to "master"
     // (for git) or "default" (for mercurial).
@@ -101,7 +101,7 @@
 
     // The directory (relative to the current directory) that raw benchmark
     // results are stored in.  If not provided, defaults to "results".
-    // "results_dir": "results",
+    "results_dir": "ASV_RESULTS",
 
     // The directory (relative to the current directory) that the html tree
     // should be written to.  If not provided, defaults to "html".

test/jenkins_tests/run_asv.sh

@@ -6,10 +6,22 @@ set -e
 # Show explicitly which commands are currently running.
 set -x
 
+BUCKET_NAME=ray-integration-testing/ASV
+COMMIT=$(cat /ray/git-rev)
+ASV_RESULTS_DIR=/ray/python/ASV_RESULTS
+pip install awscli
+
+# Install Ray fork of ASV
 git clone https://github.com/ray-project/asv.git /tmp/asv/ || true
 cd /tmp/asv/
 pip install -e .
 
 cd /ray/python/
-asv machine --yes
-asv run --show-stderr --python=same --force-record-commit=$(cat ../git-rev)
+asv machine --machine jenkins
+mkdir $ASV_RESULTS_DIR || true
+aws s3 cp s3://$BUCKET_NAME/ASV_RESULTS/benchmarks.json $ASV_RESULTS_DIR/benchmarks.json || true
+
+asv run --show-stderr --python=same --force-record-commit=$COMMIT
+
+aws s3 cp $ASV_RESULTS_DIR/benchmarks.json s3://$BUCKET_NAME/ASV_RESULTS/benchmarks_$COMMIT.json
+aws s3 sync $ASV_RESULTS_DIR/ s3://$BUCKET_NAME/ASV_RESULTS/