[pip] script for making python wheel for mxnet, CPU-only version (#4814)

* scripts for making wheel with libmxnet.so with statically linked dependencies

* address comments #4814

.gitignore

@@ -133,3 +133,12 @@ lib
 #Notebook Automated Test
 !tests/nightly/test_config.txt
 !tests/nightly/TestNotebook
+
+# pip building tools
+tools/pip_package/build
+tools/pip_package/dist
+tools/pip_package/mxnet.egg-info
+tools/pip_package/mxnet
+
+# temporary path for building dependencies when building wheel
+deps/

make/pip_linux_cpu.mk

@@ -0,0 +1,153 @@
+#-------------------------------------------------------------------------------
+#  Template configuration for compiling mxnet for making python wheel
+#-------------------------------------------------------------------------------
+
+#---------------------
+# choice of compiler
+#--------------------
+
+export CC = gcc
+export CXX = g++
+export NVCC = nvcc
+
+# whether compile with options for MXNet developer
+DEV = 0
+
+# whether compile with debug
+DEBUG = 0
+
+# whether compiler with profiler
+USE_PROFILER =
+
+# the additional link flags you want to add
+ADD_LDFLAGS += -lopencv_core -lopencv_imgproc -lopencv_highgui
+
+# the additional compile flags you want to add
+ADD_CFLAGS += -Ldeps/lib -Ideps/include
+
+#---------------------------------------------
+# matrix computation libraries for CPU/GPU
+#---------------------------------------------
+
+# choose the version of blas you want to use
+# can be: mkl, blas, atlas, openblas
+# in default use atlas for linux while apple for osx
+USE_BLAS=openblas
+
+# whether use opencv during compilation
+# you can disable it, however, you will not able to use
+# imbin iterator
+USE_OPENCV = 1
+
+# whether use CUDA during compile
+USE_CUDA = 0
+
+# add the path to CUDA library to link and compile flag
+# if you have already add them to environment variable, leave it as NONE
+# USE_CUDA_PATH = /usr/local/cuda
+USE_CUDA_PATH = NONE
+
+# whether use CuDNN R3 library
+USE_CUDNN = 0
+
+# CUDA architecture setting: going with all of them.
+# For CUDA < 6.0, comment the *_50 lines for compatibility.
+CUDA_ARCH := -gencode arch=compute_30,code=sm_30 \
+		-gencode arch=compute_35,code=sm_35 \
+		-gencode arch=compute_50,code=sm_50 \
+		-gencode arch=compute_50,code=compute_50
+
+# whether use cuda runtime compiling for writing kernels in native language (i.e. Python)
+USE_NVRTC = 0
+
+# use openmp for parallelization
+USE_OPENMP = 1
+
+
+# MKL ML Library for Intel CPU/Xeon Phi
+# Please refer to MKL_README.md for details
+
+# MKL ML Library folder, need to be root for /usr/local
+# Change to User Home directory for standard user
+# For USE_BLAS!=mkl only
+MKLML_ROOT=/usr/local
+
+# whether use MKL2017 library
+USE_MKL2017 = 0
+
+# whether use MKL2017 experimental feature for high performance
+# Prerequisite USE_MKL2017=1
+USE_MKL2017_EXPERIMENTAL = 0
+
+# whether use NNPACK library
+USE_NNPACK = 0
+
+# add path to intel library, you may need it for MKL, if you did not add the path
+# to environment variable
+USE_INTEL_PATH = NONE
+
+# If use MKL, choose static link automatically to allow python wrapper
+ifeq ($(USE_BLAS), mkl)
+USE_STATIC_MKL = 1
+else
+USE_STATIC_MKL = NONE
+endif
+
+#----------------------------
+# Settings for power and arm arch
+#----------------------------
+ARCH := $(shell uname -a)
+ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64))
+	USE_SSE=0
+else
+	USE_SSE=1
+endif
+
+#----------------------------
+# distributed computing
+#----------------------------
+
+# whether or not to enable multi-machine supporting
+USE_DIST_KVSTORE = 0
+
+# whether or not allow to read and write HDFS directly. If yes, then hadoop is
+# required
+USE_HDFS = 0
+
+# path to libjvm.so. required if USE_HDFS=1
+LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server
+
+# whether or not allow to read and write AWS S3 directly. If yes, then
+# libcurl4-openssl-dev is required, it can be installed on Ubuntu by
+# sudo apt-get install -y libcurl4-openssl-dev
+USE_S3 = 0
+
+#----------------------------
+# additional operators
+#----------------------------
+
+# path to folders containing projects specific operators that you don't want to put in src/operators
+EXTRA_OPERATORS =
+
+
+#----------------------------
+# plugins
+#----------------------------
+
+# whether to use caffe integration. This requires installing caffe.
+# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH
+# CAFFE_PATH = $(HOME)/caffe
+# MXNET_PLUGINS += plugin/caffe/caffe.mk
+
+# whether to use torch integration. This requires installing torch.
+# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH
+# TORCH_PATH = $(HOME)/torch
+# MXNET_PLUGINS += plugin/torch/torch.mk
+
+# WARPCTC_PATH = $(HOME)/warp-ctc
+# MXNET_PLUGINS += plugin/warpctc/warpctc.mk
+
+# whether to use sframe integration. This requires build sframe
+# git@github.com:dato-code/SFrame.git
+# SFRAME_PATH = $(HOME)/SFrame
+# MXNET_PLUGINS += plugin/sframe/plugin.mk

tools/pip_package/MANIFEST.in

@@ -0,0 +1,3 @@
+include README
+recursive-include * *.py
+recursive-include * *.so

tools/pip_package/make_pip_package.sh

@@ -1 +1,161 @@
+#!/usr/bin/env bash
+
+# Assuming the script is run at mxnet/tools/pip_package
+# This script builds from scratch the dependencies of mxnet into static
+# librareis and statically links them to produce a (mostly) standalone
+# libmxnet.so, then packages it into the python wheel.
+# It assumes the build environment to be a sandbox that doesn't have the .so
+# objects for the dependencies, i.e. zlib, openblas, libjpeg, libpng, libtiff
+# and opencv.
+
+# Install necessary build tools
+if [ -n "$(command -v apt-get)" ]; then
+    sudo apt-get update;
+    sudo apt-get install -y build-essential git python-pip zip pkg-config cmake
+elif [ -n "$(command -v yum)" ]; then
+    sudo yum install -y cmake
+    sudo yum groupinstall -y "Development Tools"
+    sudo yum install -y python27 python27-setuptools python27-tools python-pip
+else
+    echo "Need a package manager to install build tools, e.g. apt/yum"
+    exit 1
+fi
+sudo pip install -U pip setuptools wheel
+
+# Set up path as temporary working directory
+DEPS_PATH=$PWD/../../deps
+mkdir $DEPS_PATH
+
+# Dependencies can be updated here. Be sure to verify the download link before
+# changing. The dependencies are:
+ZLIB_VERSION=1.2.6
+OPENBLAS_VERSION=0.2.19
+JPEG_VERSION=8.4.0
+PNG_VERSION=1.5.10
+TIFF_VERSION=3.8.2
+OPENCV_VERSION=2.4.13
+
+# Setup path to dependencies
+export PKG_CONFIG_PATH=$DEPS_PATH/lib/pkgconfig:$DEPS_PATH/lib64/pkgconfig:$PKG_CONFIG_PATH
+export CPATH=$DEPS_PATH/include:$CPATH
+
+# Position Independent code must be turned on for statically linking .a
+export CC="gcc -fPIC"
+export CXX="g++ -fPIC"
+
+# Download and build zlib
+curl -L https://github.com/LuaDist/zlib/archive/$ZLIB_VERSION.zip -o $DEPS_PATH/zlib.zip
+unzip $DEPS_PATH/zlib.zip -d $DEPS_PATH
+mkdir $DEPS_PATH/zlib-$ZLIB_VERSION/build
+cd $DEPS_PATH/zlib-$ZLIB_VERSION/build
+cmake -D CMAKE_BUILD_TYPE=RELEASE \
+      -D CMAKE_INSTALL_PREFIX=$DEPS_PATH \
+      -D BUILD_SHARED_LIBS=OFF ..
+make -j$(nproc)
+make install
+cd -
+
+# download and build openblas
+curl -L https://github.com/xianyi/OpenBLAS/archive/v$OPENBLAS_VERSION.zip -o $DEPS_PATH/openblas.zip
+unzip $DEPS_PATH/openblas.zip -d $DEPS_PATH
+cd $DEPS_PATH/OpenBLAS-$OPENBLAS_VERSION
+make FC=gfortran -j $(($(nproc) + 1))
+make PREFIX=$DEPS_PATH install
+cd -
+ln -s $DEPS_PATH/lib/libopenblas_haswellp-r0.2.19.a $DEPS_PATH/lib/libcblas.a
+
+# download and build libjpeg
+curl -L https://github.com/LuaDist/libjpeg/archive/$JPEG_VERSION.zip -o $DEPS_PATH/libjpeg.zip
+unzip $DEPS_PATH/libjpeg.zip -d $DEPS_PATH
+cd $DEPS_PATH/libjpeg-$JPEG_VERSION
+./configure --disable-shared --prefix=$DEPS_PATH
+make -j$(nproc)
+make test
+make install
+cd -
+
+# download and build libpng
+curl -L https://github.com/LuaDist/libpng/archive/$PNG_VERSION.zip -o $DEPS_PATH/libpng.zip
+unzip $DEPS_PATH/libpng.zip -d $DEPS_PATH
+mkdir $DEPS_PATH/libpng-$PNG_VERSION/build
+cd $DEPS_PATH/libpng-$PNG_VERSION/build
+cmake -D CMAKE_BUILD_TYPE=RELEASE \
+      -D CMAKE_INSTALL_PREFIX=$DEPS_PATH \
+      -D PNG_CONFIGURE_LIBPNG=-fPIC \
+      -D BUILD_SHARED_LIBS=OFF ..
+make -j$(nproc)
+make install
+cd -
+
+# download and build libtiff
+curl -L https://github.com/LuaDist/libtiff/archive/$TIFF_VERSION.zip -o $DEPS_PATH/libtiff.zip
+unzip $DEPS_PATH/libtiff.zip -d $DEPS_PATH
+cd $DEPS_PATH/libtiff-$TIFF_VERSION
+./configure --disable-shared --prefix=$DEPS_PATH
+make -j$(nproc)
+make install
+cd -
+
+# download and build opencv since we need the static library
+curl -L https://github.com/Itseez/opencv/archive/$OPENCV_VERSION.zip -o $DEPS_PATH/opencv.zip
+unzip $DEPS_PATH/opencv.zip -d $DEPS_PATH
+mkdir $DEPS_PATH/opencv-$OPENCV_VERSION/build
+cd $DEPS_PATH/opencv-$OPENCV_VERSION/build
+cmake -D WITH_1394=OFF \
+      -D WITH_AVFOUNDATION=OFF \
+      -D WITH_CUDA=OFF \
+      -D WITH_VTK=OFF \
+      -D WITH_CUFFT=OFF \
+      -D WITH_CUBLAS=OFF \
+      -D WITH_NVCUVID=OFF \
+      -D WITH_EIGEN=ON \
+      -D WITH_VFW=OFF \
+      -D WITH_FFMPEG=OFF \
+      -D WITH_GSTREAMER=OFF \
+      -D WITH_GTK=OFF \
+      -D WITH_JASPER=OFF \
+      -D WITH_JPEG=ON \
+      -D WITH_PNG=ON \
+      -D WITH_QUICKTIME=OFF \
+      -D WITH_TBB=ON \
+      -D WITH_TIFF=OFF \
+      -D WITH_V4L=OFF \
+      -D WITH_LIBV4L=OFF \
+      -D WITH_DSHOW=OFF \
+      -D WITH_MSMF=OFF \
+      -D WITH_OPENCL=OFF \
+      -D WITH_OPENCLAMDFFT=OFF \
+      -D WITH_OPENCLAMDBLAS=OFF \
+      -D BUILD_SHARED_LIBS=OFF \
+      -D BUILD_opencv_apps=OFF \
+      -D BUILD_opencv_gpu=OFF \
+      -D BUILD_opencv_video=OFF \
+      -D BUILD_opencv_contrib=OFF \
+      -D BUILD_opencv_nonfree=OFF \
+      -D BUILD_opencv_flann=OFF \
+      -D BUILD_opencv_features2d=OFF \
+      -D BUILD_opencv_calib3d=OFF \
+      -D BUILD_opencv_objdetect=OFF \
+      -D BUILD_opencv_ml=OFF \
+      -D BUILD_opencv_photo=OFF \
+      -D BUILD_DOCS=OFF \
+      -D BUILD_PACKAGE=OFF \
+      -D CMAKE_BUILD_TYPE=RELEASE \
+      -D CMAKE_INSTALL_PREFIX=$DEPS_PATH ..
+make -j $(nproc)
+make install # user will always have access to home, so no sudo needed
+cd -
+
+# Although .so building is explicitly turned off for most libraries, sometimes
+# they still get created. So, remove them just to make sure they don't
+# interfere, or otherwise we might get libmxnet.so that is not self-contained.
+rm $DEPS_PATH/{lib,lib64}/*.{so,so.0}
+
+# Go to the parent path and build mxnet
+cd ../../
+cp make/pip_$(uname | tr '[:upper:]' '[:lower:]')_cpu.mk config.mk
+make -j $(nproc)
+
+# Generate wheel. The output is in the mxnet/tools/pip_package/dist path.
+cd tools/pip_package
 python setup.py bdist_wheel

tools/pip_package/setup.py

@@ -2,11 +2,9 @@
 """Setup mxnet package."""
 from __future__ import absolute_import
 import os
-import sys
 import shutil
 
 from setuptools import setup, find_packages
-from setuptools.extension import Extension
 from setuptools.dist import Distribution
 
 # We can not import `mxnet.info.py` in setup.py directly since mxnet/__init__.py
@@ -20,12 +18,15 @@
 __version__ = libinfo['__version__']
 
 class BinaryDistribution(Distribution):
-    def is_pure(self):
-        return False
     def has_ext_modules(self):
         return True
 
-shutil.rmtree(os.path.join(CURRENT_DIR, 'mxnet'))#, ignore_errors=True)
+
+DEPENDENCIES = [
+    'numpy',
+]
+
+shutil.rmtree(os.path.join(CURRENT_DIR, 'mxnet'), ignore_errors=True)
 shutil.copytree(os.path.join(CURRENT_DIR, '../../python/mxnet'),
                 os.path.join(CURRENT_DIR, 'mxnet'))
 shutil.copy(LIB_PATH[0], os.path.join(CURRENT_DIR, 'mxnet'))
@@ -37,5 +38,6 @@ def has_ext_modules(self):
       packages=find_packages(),
       package_data={'mxnet': [os.path.join('mxnet', os.path.basename(LIB_PATH[0]))]},
       include_package_data=True,
+      install_requires=DEPENDENCIES,
       distclass=BinaryDistribution,
       url='https://github.com/dmlc/mxnet')