Add basic functionality for Cython functions and actors (ray-project#…

…1193)

* Add basic functionality for Cython functions and actors

* Fix up per @pcmoritz comments

* Fixes per @richardliaw comments

* Fixes per @robertnishihara comments

* Forgot double quotes when updating masked_log

* Remove import typing for Python 2 compatibility

.gitignore

@@ -48,6 +48,9 @@
 *.dylib
 *.dll
 
+# Cython-generated files
+*.c
+
 # Incremental linking files
 *.ilk
 
@@ -95,6 +98,7 @@ scripts/nodes.txt
 
 # CMake
 cmake-build-debug/
+build
 
 # Python setup files
 *.egg-info

.travis.yml

@@ -91,6 +91,7 @@ install:
   - ./.travis/install-dependencies.sh
   - export PATH="$HOME/miniconda/bin:$PATH"
   - ./.travis/install-ray.sh
+  - ./.travis/install-cython-examples.sh
 
   - cd python/ray/core
   - bash ../../../src/common/test/run_tests.sh
@@ -120,6 +121,7 @@ script:
   - python test/monitor_test.py
   - python test/trial_runner_test.py
   - python test/trial_scheduler_test.py
+  - python test/cython_test.py
 
   - python -m pytest python/ray/rllib/test/test_catalog.py
 

.travis/install-cython-examples.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+# Cause the script to exit if a single command fails
+set -e
+
+ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
+
+echo "PYTHON is $PYTHON"
+
+cython_examples="$ROOT_DIR/../examples/cython"
+
+if [[ "$PYTHON" == "2.7" ]]; then
+
+   pushd $cython_examples
+   pip install scipy
+   python setup.py install --user
+   popd
+
+elif [[ "$PYTHON" == "3.5" ]]; then
+   export PATH="$HOME/miniconda/bin:$PATH"
+
+   pushd $cython_examples
+   pip install scipy
+   python setup.py install --user
+   popd
+
+elif [[ "$LINT" == "1" ]]; then
+   export PATH="$HOME/miniconda/bin:$PATH"
+
+   pushd $cython_examples
+   python setup.py install --user
+   popd
+
+else
+   echo "Unrecognized Python version."
+   exit 1
+fi

doc/source/example-cython.rst

@@ -0,0 +1,36 @@
+Cython
+======
+
+Getting Started
+---------------
+
+This document provides examples of using Cython-generated code in ``ray``. To get started, run the following from directory ``$RAY_HOME/examples/cython``:
+
+.. code-block:: bash
+
+   pip install scipy # For BLAS example
+   python setup.py develop
+   python cython_main.py --help
+
+You can import the ``cython_examples`` module from a Python script or interpreter.
+
+Notes
+-----
+* You **must** include the following two lines at the top of any ``*.pyx`` file:
+
+.. code-block:: python
+
+   #!python
+   # cython: embedsignature=True, binding=True
+
+* You cannot decorate Cython functions within a ``*.pyx`` file (there are ways around this, but creates a leaky abstraction between Cython and Python that would be very challenging to support generally). Instead, prefer the following in your Python code:
+
+.. code-block:: python
+
+   some_cython_func = ray.remote(some_cython_module.some_cython_func)
+
+* You cannot transfer memory buffers to a remote function (see ``example8``, which currently fails); your remote function must return a value
+* Have a look at ``cython_main.py``, ``cython_simple.pyx``, and ``setup.py`` for examples of how to call, define, and build Cython code, respectively. The Cython `documentation <http://cython.readthedocs.io/>`_ is also very helpful.
+* Several limitations come from Cython's own `unsupported <https://github.com/cython/cython/wiki/Unsupported>`_ Python features.
+* We currently do not support compiling and distributing Cython code to ``ray`` clusters. In other words, Cython developers are responsible for compiling and distributing any Cython code to their cluster (much as would be the case for users who need Python packages like ``scipy``).
+* For most simple use cases, developers need not worry about Python 2 or 3, but users who do need to care can have a look at the ``language_level`` Cython compiler directive (see `here <http://cython.readthedocs.io/en/latest/src/reference/compilation.html>`_).

doc/source/index.rst

@@ -57,6 +57,7 @@ Example Program
    example-a3c.rst
    example-lbfgs.rst
    example-evolution-strategies.rst
+   example-cython.rst
    using-ray-with-tensorflow.rst
 
 .. toctree::

examples/cython/cython_examples/__init__.py

@@ -0,0 +1,31 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from .cython_simple import simple_func, fib, fib_int, \
+        fib_cpdef, fib_cdef, simple_class
+from .masked_log import masked_log
+
+from .cython_blas import \
+        compute_self_corr_for_voxel_sel, \
+        compute_kernel_matrix, \
+        compute_single_self_corr_syrk, \
+        compute_single_self_corr_gemm, \
+        compute_corr_vectors, \
+        compute_single_matrix_multiplication
+
+__all__ = [
+        "simple_func",
+        "fib",
+        "fib_int",
+        "fib_cpdef",
+        "fib_cdef",
+        "simple_class",
+        "masked_log",
+        "compute_self_corr_for_voxel_sel",
+        "compute_kernel_matrix",
+        "compute_single_self_corr_syrk",
+        "compute_single_self_corr_gemm",
+        "compute_corr_vectors",
+        "compute_single_matrix_multiplication"
+        ]