SINGA-303 Create conda package

Update Dockerfile for jenkins to install conda tools
Reset the jenkin's jobs to
1. compile and do unittest of SINGA GPU version
2. create conda packages for GPU SINGA

Update the conda meta.yml to replace libprotobuf with protobuf, as protobuf includes the libs.

Add conda-forge into the channels before conda build for osx

Enable rpath for macosx, otherwise the lib files will link to itself but with a fixed path (from otool -L)

Replace nightly with main for the label of the conda package to main, otherwise we need 'conda install -c nusdbsystem/label/nightly singa'

Author: nudles

CMakeLists.txt

@@ -34,6 +34,7 @@ IF(UNIX)
 ENDIF()
 IF(APPLE)
     SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -stdlib=libc++ -g -O2 -fPIC -Wall -pthread")
+    SET(CMAKE_MACOSX_RPATH 1)
 ENDIF()
 
 IF(WIN32)

tool/conda/meta.yaml

@@ -17,13 +17,12 @@ requirements:
     - numpy >=1.10
     - swig >=3.0
     - openblas >=0.2.10
-    - libprotobuf 3.0.0
+    - protobuf 3.0.0
     - glog 0.3.4
 
   run:
     - python 2.7*
     - numpy >=1.10
-    - libprotobuf 3.0.0
     - protobuf 3.0.0
     - glog 0.3.4
     - openblas >=0.2.10

tool/jenkins/README.md

@@ -17,9 +17,18 @@ Create a multi-configuration project and configure project as follows:
 
 ### Description
 This job automatically pulls latest commits from Apache incubator-singa github repository, then for different environments
-* compile and test SINGA
-* create PySINGA wheel files
-* create Debian packages
+
+* compile and test SINGA on GPUs
+* create Debian GPU packages
+* create anaconda GPU packages
+
+The working nodes (or Docker containers) are configured in Jenkins-Manage Jenkins-Mange Nodes.
+Each node should configure the following environment variable
+1. CUDA_VERSION, e.g., 7.5
+2. CUDNN_VERSION e.g, 5
+3. ANACONDA_UPLOAD_TOKEN
+4. SINGA_NAME=singa-cuda${CUDA_VERSION}-cudnn${CUDNN_VERSION}
+5. OS_VERSION, e.g., ubuntu14.04
 
 ### General
   * Discard old builds - Max # of builds to keep - 50
@@ -33,31 +42,32 @@ This job automatically pulls latest commits from Apache incubator-singa github r
   * Poll SCM - Schedule - ``H/30 * * * *`` (pull every 30 minutes)
 
 ### Configuration Matrix
-  * User-defined Axis - name ``lang`` values ``CPP CUDA``
+  * User-defined Axis - name ``lang`` values ``CUDA`` (Add CPP if you want to test CPU code)
   * Slave - name ``env`` Node/label: tick available nodes
 
 ### Build
-  * Execute shell - command - ``bash -ex tool/jenkins/jenkins_wheel.sh $lang``
+  * compile and do unit test on GPU
+    Execute shell - command - ``bash -ex tool/jenkins/test.sh $lang``
+    `$lang` is set in **Configuration Matrix* section
 
-### Post-build Actions
-  * Publish JUnit test result report - Test report XMLs - ``**/gtest.xml, **/unittest.xml``
-  * (optional) Archive the artifacts - ``build/python/dist/**.whl, build/debian/**.deb``
-  * Send build artifacts (wheel) over SSH
-    * jenkins_wheel.sh packages the .whl file into $BUILD_ID.tar.gz. Inside the tar file,
-      the folder layout is `build_id/commit_hash/os_lang/*.whl`, where `os_lang` is the combination of os version, device programming language (cuda/cpp/opencl) and cudnn version.
-    * In `Manage Jenkins`-`Configure System`, configure the SSH for connecting to the remote public server and set the target folder location
-    * Source files - `build/python/dist/*.tar.gz`
-    * Remove prefix - `build/python/dist`
-    * Remote directory - `wheel/linux`
-    * Exec a command on the remote server to decompress the package and add a symlink to the latest build. E.g., on a Solaris server the command is
+  * create Debian package
+    Execute shell - command - ``bash -ex tool/debian/build.sh --python --$lang``
 
-            cd <target_folder>/wheel/linux && gunzip $BUILD_ID.tar.gz && tar xf $BUILD_ID.tar && chmod -R 755 $BUILD_ID && /bin/rm -f $BUILD_ID.tar && /bin/rm -f latest && ln -s $BUILD_ID/* latest
+  * create conda package
+    Execute shell - command -
 
-    * The file links on the remote public server would be like
+        git push --mirror <https://username:auth_token@github.com/mirror_repo>
+        /root/miniconda/conda tool/conda/
+        export CONDA_BLD_PATH=/root/conda-bld-$BUILD_NUMBER
+        mkdir $CONDA_BLD_PATH
+        /root/miniconda/anaconda -t $ANACONDA_UPLOAD_TOKEN upload -u $USER -l main $CONDA_BLD_PATH/$OS/singa-*.tar.bz2 --force
 
-            wheel/Linux/32/84d56b7/ubuntu16.04-cpp/singa-1.0.1-py2-none-any.whl
-            wheel/Linux/32/84d56b7/ubuntu16.04-cuda8.0-cudnn5/singa-1.0.1-py2-none-any.whl
+    It first pushes to a mirror site to invoke travis-ci for CPU package creation;
+    Then it creates the conda package for GPU and uploads the package.
 
+### Post-build Actions
+  * Publish JUnit test result report - Test report XMLs - ``**/gtest.xml, **/unittest.xml``
+  * (optional) Archive the artifacts - ``build/python/dist/**.whl, build/debian/**.deb``
   * Send build artifacts (Debian package) over SSH for wheel
     * ../debian/build.sh packages the .deb file into $BUILD_ID.tar.gz. Inside the tar file,
       the folder layout is `build_id/commit_hash/os_lang/*.deb`, where `os_lang` is the combination of os version, device programming language (cuda/cpp/opencl) and cudnn version.
@@ -71,29 +81,12 @@ This job automatically pulls latest commits from Apache incubator-singa github r
 
     * The file links on the remote public server would be like
 
-            debian/32/84d56b7/ubuntu16.04-cpp/singa-1.0.1-py2-none-any.whl
-            debian/32/84d56b7/ubuntu16.04-cuda8.0-cudnn5/singa-1.0.1-py2-none-any.whl
-
-### Notes for Mac OS X
-The Job configuration is the same as stated above except,
-* Machines (or docker containers) with Mac OS X should be configured as jenkin nodes; the following environmental variables should be exported in the .bashrc or in the node configuration
-
-    # 10.11 or 10.12
-    export OS_VERSION=macosx10.11
-    # if using Homebrew
-    export PATH=/usr/local/bin:$PATH
-    # for numpy.i
-    export CPLUS_INCLUDE_PATH=`python -c "import numpy; print numpy.get_include()"`:$CPLUS_INCLUDE_PATH
-
-* Set User-defined Axis to None and tick only Mac OS X nodes
+            debian/32/84d56b7/ubuntu16.04-cpp/singa-1.0.1.deb
+            debian/32/84d56b7/ubuntu16.04-cuda8.0-cudnn5/singa-1.0.1.deb
 
-* Send build artifacts (wheel) over SSH
-  * The remote directly -- `wheel/macosx`
-  * The shell command is
 
-            cd <target_folder>/wheel/macosx && gunzip $BUILD_ID.tar.gz && tar xf $BUILD_ID.tar && chmod -R 755 $BUILD_ID && /bin/rm -f $BUILD_ID.tar && /bin/rm -f latest && ln -s $BUILD_ID/* latest
 ### Docker Images
-We provide in `docker` a number of singa docker images for Jenkins to use as slaves.
+We provide in a number of singa docker [images](../docker) for Jenkins to use as slaves.
 To run the docker images,
 
     nvidia-docker run --name <jenkins-slaveXX> -d <Image ID>

tool/jenkins/docker/ubuntu14.04/devel/Dockerfile

@@ -16,15 +16,15 @@
 #
 # Base unbuntu 14.04 image from nvidia/cuda
 # Change tags to build with different cuda/cudnn versions:
-#   FROM nvidia/cuda:8.0-cudnn5-devel
-#   FROM nvidia/cuda:7.5-cudnn5-devel
-#   FROM nvidia/cuda:7.5-cudnn4-devel
-#   FROM nvidia/cuda:7.0-cudnn4-devel
-FROM nvidia/cuda:7.5-cudnn5-devel
+#   FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu14.04
+#   FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04
+#   FROM nvidia/cuda:7.5-cudnn4-devel-ubuntu14.04
+#   FROM nvidia/cuda:7.0-cudnn4-devel-ubuntu14.04
+FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04
 
 # install dependencies
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends git build-essential autoconf libtool cmake libpcre3-dev python-dev python-pip wget openssh-server \
+    && apt-get install -y --no-install-recommends git build-essential autoconf libtool cmake libprotobuf-dev libopenblas-dev libpcre3-dev python-dev python-pip protobuf-compiler wget openssh-server \
     && apt-get clean && apt-get autoremove && apt-get autoclean \
     && rm -rf /var/lib/apt/lists/* \
     && pip install -U pip wheel numpy setuptools unittest-xml-reporting protobuf
@@ -35,14 +35,22 @@ RUN wget http://prdownloads.sourceforge.net/swig/swig-3.0.10.tar.gz && \
     tar zxf swig-3.0.10.tar.gz && cd swig-3.0.10 && \
     ./configure && make && make install
 
+# install conda, conda-build and anaconda-client
+RUN wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh;
+RUN bash miniconda.sh -b -p $HOME/miniconda
+RUN $HOME/miniconda/bin/conda config --set always_yes yes --set changeps1 no
+RUN $HOME/miniconda/bin/conda update -q conda
+RUN $HOME/miniconda/bin/conda install conda-build
+RUN $HOME/miniconda/bin/conda install anaconda-client
+ENV PATH $HOME/miniconda/bin:${PATH}
+
 # set environment
 ENV CPLUS_INCLUDE_PATH /usr/local/lib/python2.7/dist-packages/numpy/core/include:${CPLUS_INCLUDE_PATH}
-ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH}
-ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH}
-ENV OS_VERSION ubuntu14.04
+# ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH}
+# ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH}
 
 # download singa source
-RUN git clone https://github.com/apache/incubator-singa.git
+# RUN git clone https://github.com/apache/incubator-singa.git
 
 # config ssh service
 RUN mkdir /var/run/sshd

tool/jenkins/docker/ubuntu16.04/devel/Dockerfile

@@ -36,7 +36,6 @@ RUN apt-get update \
 ENV CPLUS_INCLUDE_PATH /usr/local/lib/python2.7/dist-packages/numpy/core/include:${CPLUS_INCLUDE_PATH}
 ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH}
 ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH}
-ENV OS_VERSION ubuntu16.04
 
 # download singa source
 RUN git clone https://github.com/apache/incubator-singa.git

tool/jenkins/jenkins_wheel.sh tool/jenkins/jenkins_test.shtool/jenkins/jenkins_wheel.sh renamed to tool/jenkins/jenkins_test.sh

@@ -18,28 +18,24 @@
 # * limitations under the License.
 # */
 
-# This script is used by Jenkins to compile and distribute SINGA wheel file.
+# This script is used by Jenkins to compile and test SINGA
 
-echo Compile, test and distribute PySINGA...
+echo Compile and test SINGA...
 echo parameters: $1
 echo workspace: `pwd`
 echo OS version: `cat /etc/issue`
 echo kernal version: `uname -a`
 echo CUDA version: $CUDA_VERSION
 echo CUDNN version: $CUDNN_VERSION
-echo OS name: $OS_VERSION
 COMMIT=`git rev-parse --short HEAD`
 echo COMMIT HASH: $COMMIT
 # set parameters
 CUDA="OFF"
 CUDNN="OFF"
-FOLDER=$BUILD_NUMBER/$COMMIT/$OS_VERSION-cpp
 if [ $1 = "CUDA" ]; then
   CUDA="ON"
   CUDNN="ON"
-  FOLDER=$BUILD_NUMBER/$COMMIT/$OS_VERSION-cuda$CUDA_VERSION-cudnn$CUDNN_VERSION
 fi
-echo wheel file folder: build/python/dist/whl/$FOLDER
 
 # setup env
 rm -rf build
@@ -51,20 +47,11 @@ fi
 
 # compile singa c++
 cd build
-cmake -DUSE_CUDNN=$CUDNN -DUSE_CUDA=$CUDA -DUSE_MODULES=ON $EXTRA_ARGS ../
+cmake -DUSE_CUDA=$CUDA -DENABLE_TEST=ON $EXTRA_ARGS ../
 make
 # unit test cpp code
 ./bin/test_singa --gtest_output=xml:./gtest.xml
-# compile pysinga
-cd python
-python setup.py bdist_wheel
-# mv whl file to a folder whose name identifies the OS, CUDA, CUDNN etc.
-cd dist
-mkdir -p $FOLDER
-mv *.whl $FOLDER/
-tar czf $BUILD_NUMBER.tar.gz $FOLDER/*
-
 # unit test python code
-cd ../../../test/python
+cd ../test/python
 PYTHONPATH=../../build/python/ python run.py
 echo Job finished...

tool/travis/conda.sh

@@ -27,12 +27,12 @@ export PATH="$HOME/miniconda/bin:$PATH"
 conda config --set anaconda_upload no
 
 # save the package at given folder, then we can upload using singa-*.tar.bz2
-suffix=`TZ=Asia/Singapore date +%Y-%m-%d-%H-%M-%S`
+suffix=$TRAVIS_JOB_NUMBER  #`TZ=Asia/Singapore date +%Y-%m-%d-%H-%M-%S`
 export CONDA_BLD_PATH=~/conda-bld-$suffix
 mkdir $CONDA_BLD_PATH
 
 conda build tool/conda/
 
 # turn off debug to hide the token in travis log
 set +x
-anaconda -t $ANACONDA_UPLOAD_TOKEN upload -u $USER -l nightly $CONDA_BLD_PATH/$OS/singa-*.tar.bz2 --force
+anaconda -t $ANACONDA_UPLOAD_TOKEN upload -u $USER -l main $CONDA_BLD_PATH/$OS/singa-*.tar.bz2 --force

tool/travis/depends.sh

@@ -43,4 +43,5 @@ else
   conda update -q conda
   conda install conda-build
   conda install anaconda-client
+  conda config --add channels conda-forge
 fi