Skip to content

Commit 7165476

Browse files
szhapiiswrong
szha
authored and
piiswrong
committed
[pip] script for making python wheel for mxnet, CPU-only version (apache#4814)
* scripts for making wheel with libmxnet.so with statically linked dependencies * address comments apache#4814
1 parent 7c197d6 commit 7165476

File tree

5 files changed

+332
-5
lines changed

5 files changed

+332
-5
lines changed

‎.gitignore

+9
Original file line numberDiff line numberDiff line change
@@ -133,3 +133,12 @@ lib
133133
#Notebook Automated Test
134134
!tests/nightly/test_config.txt
135135
!tests/nightly/TestNotebook
136+
137+
# pip building tools
138+
tools/pip_package/build
139+
tools/pip_package/dist
140+
tools/pip_package/mxnet.egg-info
141+
tools/pip_package/mxnet
142+
143+
# temporary path for building dependencies when building wheel
144+
deps/

‎make/pip_linux_cpu.mk

+153
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
#-------------------------------------------------------------------------------
2+
# Template configuration for compiling mxnet for making python wheel
3+
#-------------------------------------------------------------------------------
4+
5+
#---------------------
6+
# choice of compiler
7+
#--------------------
8+
9+
export CC = gcc
10+
export CXX = g++
11+
export NVCC = nvcc
12+
13+
# whether compile with options for MXNet developer
14+
DEV = 0
15+
16+
# whether compile with debug
17+
DEBUG = 0
18+
19+
# whether compiler with profiler
20+
USE_PROFILER =
21+
22+
# the additional link flags you want to add
23+
ADD_LDFLAGS += -lopencv_core -lopencv_imgproc -lopencv_highgui
24+
25+
# the additional compile flags you want to add
26+
ADD_CFLAGS += -Ldeps/lib -Ideps/include
27+
28+
#---------------------------------------------
29+
# matrix computation libraries for CPU/GPU
30+
#---------------------------------------------
31+
32+
# choose the version of blas you want to use
33+
# can be: mkl, blas, atlas, openblas
34+
# in default use atlas for linux while apple for osx
35+
USE_BLAS=openblas
36+
37+
# whether use opencv during compilation
38+
# you can disable it, however, you will not able to use
39+
# imbin iterator
40+
USE_OPENCV = 1
41+
42+
# whether use CUDA during compile
43+
USE_CUDA = 0
44+
45+
# add the path to CUDA library to link and compile flag
46+
# if you have already add them to environment variable, leave it as NONE
47+
# USE_CUDA_PATH = /usr/local/cuda
48+
USE_CUDA_PATH = NONE
49+
50+
# whether use CuDNN R3 library
51+
USE_CUDNN = 0
52+
53+
# CUDA architecture setting: going with all of them.
54+
# For CUDA < 6.0, comment the *_50 lines for compatibility.
55+
CUDA_ARCH := -gencode arch=compute_30,code=sm_30 \
56+
-gencode arch=compute_35,code=sm_35 \
57+
-gencode arch=compute_50,code=sm_50 \
58+
-gencode arch=compute_50,code=compute_50
59+
60+
# whether use cuda runtime compiling for writing kernels in native language (i.e. Python)
61+
USE_NVRTC = 0
62+
63+
# use openmp for parallelization
64+
USE_OPENMP = 1
65+
66+
67+
# MKL ML Library for Intel CPU/Xeon Phi
68+
# Please refer to MKL_README.md for details
69+
70+
# MKL ML Library folder, need to be root for /usr/local
71+
# Change to User Home directory for standard user
72+
# For USE_BLAS!=mkl only
73+
MKLML_ROOT=/usr/local
74+
75+
# whether use MKL2017 library
76+
USE_MKL2017 = 0
77+
78+
# whether use MKL2017 experimental feature for high performance
79+
# Prerequisite USE_MKL2017=1
80+
USE_MKL2017_EXPERIMENTAL = 0
81+
82+
# whether use NNPACK library
83+
USE_NNPACK = 0
84+
85+
# add path to intel library, you may need it for MKL, if you did not add the path
86+
# to environment variable
87+
USE_INTEL_PATH = NONE
88+
89+
# If use MKL, choose static link automatically to allow python wrapper
90+
ifeq ($(USE_BLAS), mkl)
91+
USE_STATIC_MKL = 1
92+
else
93+
USE_STATIC_MKL = NONE
94+
endif
95+
96+
#----------------------------
97+
# Settings for power and arm arch
98+
#----------------------------
99+
ARCH := $(shell uname -a)
100+
ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64))
101+
USE_SSE=0
102+
else
103+
USE_SSE=1
104+
endif
105+
106+
#----------------------------
107+
# distributed computing
108+
#----------------------------
109+
110+
# whether or not to enable multi-machine supporting
111+
USE_DIST_KVSTORE = 0
112+
113+
# whether or not allow to read and write HDFS directly. If yes, then hadoop is
114+
# required
115+
USE_HDFS = 0
116+
117+
# path to libjvm.so. required if USE_HDFS=1
118+
LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server
119+
120+
# whether or not allow to read and write AWS S3 directly. If yes, then
121+
# libcurl4-openssl-dev is required, it can be installed on Ubuntu by
122+
# sudo apt-get install -y libcurl4-openssl-dev
123+
USE_S3 = 0
124+
125+
#----------------------------
126+
# additional operators
127+
#----------------------------
128+
129+
# path to folders containing projects specific operators that you don't want to put in src/operators
130+
EXTRA_OPERATORS =
131+
132+
133+
#----------------------------
134+
# plugins
135+
#----------------------------
136+
137+
# whether to use caffe integration. This requires installing caffe.
138+
# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH
139+
# CAFFE_PATH = $(HOME)/caffe
140+
# MXNET_PLUGINS += plugin/caffe/caffe.mk
141+
142+
# whether to use torch integration. This requires installing torch.
143+
# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH
144+
# TORCH_PATH = $(HOME)/torch
145+
# MXNET_PLUGINS += plugin/torch/torch.mk
146+
147+
# WARPCTC_PATH = $(HOME)/warp-ctc
148+
# MXNET_PLUGINS += plugin/warpctc/warpctc.mk
149+
150+
# whether to use sframe integration. This requires build sframe
151+
# git@github.com:dato-code/SFrame.git
152+
# SFRAME_PATH = $(HOME)/SFrame
153+
# MXNET_PLUGINS += plugin/sframe/plugin.mk

‎tools/pip_package/MANIFEST.in

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
include README
2+
recursive-include * *.py
3+
recursive-include * *.so

‎tools/pip_package/make_pip_package.sh

+160
Original file line numberDiff line numberDiff line change
@@ -1 +1,161 @@
1+
#!/usr/bin/env bash
2+
3+
# Assuming the script is run at mxnet/tools/pip_package
4+
# This script builds from scratch the dependencies of mxnet into static
5+
# librareis and statically links them to produce a (mostly) standalone
6+
# libmxnet.so, then packages it into the python wheel.
7+
# It assumes the build environment to be a sandbox that doesn't have the .so
8+
# objects for the dependencies, i.e. zlib, openblas, libjpeg, libpng, libtiff
9+
# and opencv.
10+
11+
# Install necessary build tools
12+
if [ -n "$(command -v apt-get)" ]; then
13+
sudo apt-get update;
14+
sudo apt-get install -y build-essential git python-pip zip pkg-config cmake
15+
elif [ -n "$(command -v yum)" ]; then
16+
sudo yum install -y cmake
17+
sudo yum groupinstall -y "Development Tools"
18+
sudo yum install -y python27 python27-setuptools python27-tools python-pip
19+
else
20+
echo "Need a package manager to install build tools, e.g. apt/yum"
21+
exit 1
22+
fi
23+
sudo pip install -U pip setuptools wheel
24+
25+
# Set up path as temporary working directory
26+
DEPS_PATH=$PWD/../../deps
27+
mkdir $DEPS_PATH
28+
29+
# Dependencies can be updated here. Be sure to verify the download link before
30+
# changing. The dependencies are:
31+
ZLIB_VERSION=1.2.6
32+
OPENBLAS_VERSION=0.2.19
33+
JPEG_VERSION=8.4.0
34+
PNG_VERSION=1.5.10
35+
TIFF_VERSION=3.8.2
36+
OPENCV_VERSION=2.4.13
37+
38+
# Setup path to dependencies
39+
export PKG_CONFIG_PATH=$DEPS_PATH/lib/pkgconfig:$DEPS_PATH/lib64/pkgconfig:$PKG_CONFIG_PATH
40+
export CPATH=$DEPS_PATH/include:$CPATH
41+
42+
# Position Independent code must be turned on for statically linking .a
43+
export CC="gcc -fPIC"
44+
export CXX="g++ -fPIC"
45+
46+
# Download and build zlib
47+
curl -L https://github.com/LuaDist/zlib/archive/$ZLIB_VERSION.zip -o $DEPS_PATH/zlib.zip
48+
unzip $DEPS_PATH/zlib.zip -d $DEPS_PATH
49+
mkdir $DEPS_PATH/zlib-$ZLIB_VERSION/build
50+
cd $DEPS_PATH/zlib-$ZLIB_VERSION/build
51+
cmake -D CMAKE_BUILD_TYPE=RELEASE \
52+
-D CMAKE_INSTALL_PREFIX=$DEPS_PATH \
53+
-D BUILD_SHARED_LIBS=OFF ..
54+
make -j$(nproc)
55+
make install
56+
cd -
57+
58+
# download and build openblas
59+
curl -L https://github.com/xianyi/OpenBLAS/archive/v$OPENBLAS_VERSION.zip -o $DEPS_PATH/openblas.zip
60+
unzip $DEPS_PATH/openblas.zip -d $DEPS_PATH
61+
cd $DEPS_PATH/OpenBLAS-$OPENBLAS_VERSION
62+
make FC=gfortran -j $(($(nproc) + 1))
63+
make PREFIX=$DEPS_PATH install
64+
cd -
65+
ln -s $DEPS_PATH/lib/libopenblas_haswellp-r0.2.19.a $DEPS_PATH/lib/libcblas.a
66+
67+
# download and build libjpeg
68+
curl -L https://github.com/LuaDist/libjpeg/archive/$JPEG_VERSION.zip -o $DEPS_PATH/libjpeg.zip
69+
unzip $DEPS_PATH/libjpeg.zip -d $DEPS_PATH
70+
cd $DEPS_PATH/libjpeg-$JPEG_VERSION
71+
./configure --disable-shared --prefix=$DEPS_PATH
72+
make -j$(nproc)
73+
make test
74+
make install
75+
cd -
76+
77+
# download and build libpng
78+
curl -L https://github.com/LuaDist/libpng/archive/$PNG_VERSION.zip -o $DEPS_PATH/libpng.zip
79+
unzip $DEPS_PATH/libpng.zip -d $DEPS_PATH
80+
mkdir $DEPS_PATH/libpng-$PNG_VERSION/build
81+
cd $DEPS_PATH/libpng-$PNG_VERSION/build
82+
cmake -D CMAKE_BUILD_TYPE=RELEASE \
83+
-D CMAKE_INSTALL_PREFIX=$DEPS_PATH \
84+
-D PNG_CONFIGURE_LIBPNG=-fPIC \
85+
-D BUILD_SHARED_LIBS=OFF ..
86+
make -j$(nproc)
87+
make install
88+
cd -
89+
90+
# download and build libtiff
91+
curl -L https://github.com/LuaDist/libtiff/archive/$TIFF_VERSION.zip -o $DEPS_PATH/libtiff.zip
92+
unzip $DEPS_PATH/libtiff.zip -d $DEPS_PATH
93+
cd $DEPS_PATH/libtiff-$TIFF_VERSION
94+
./configure --disable-shared --prefix=$DEPS_PATH
95+
make -j$(nproc)
96+
make install
97+
cd -
98+
99+
# download and build opencv since we need the static library
100+
curl -L https://github.com/Itseez/opencv/archive/$OPENCV_VERSION.zip -o $DEPS_PATH/opencv.zip
101+
unzip $DEPS_PATH/opencv.zip -d $DEPS_PATH
102+
mkdir $DEPS_PATH/opencv-$OPENCV_VERSION/build
103+
cd $DEPS_PATH/opencv-$OPENCV_VERSION/build
104+
cmake -D WITH_1394=OFF \
105+
-D WITH_AVFOUNDATION=OFF \
106+
-D WITH_CUDA=OFF \
107+
-D WITH_VTK=OFF \
108+
-D WITH_CUFFT=OFF \
109+
-D WITH_CUBLAS=OFF \
110+
-D WITH_NVCUVID=OFF \
111+
-D WITH_EIGEN=ON \
112+
-D WITH_VFW=OFF \
113+
-D WITH_FFMPEG=OFF \
114+
-D WITH_GSTREAMER=OFF \
115+
-D WITH_GTK=OFF \
116+
-D WITH_JASPER=OFF \
117+
-D WITH_JPEG=ON \
118+
-D WITH_PNG=ON \
119+
-D WITH_QUICKTIME=OFF \
120+
-D WITH_TBB=ON \
121+
-D WITH_TIFF=OFF \
122+
-D WITH_V4L=OFF \
123+
-D WITH_LIBV4L=OFF \
124+
-D WITH_DSHOW=OFF \
125+
-D WITH_MSMF=OFF \
126+
-D WITH_OPENCL=OFF \
127+
-D WITH_OPENCLAMDFFT=OFF \
128+
-D WITH_OPENCLAMDBLAS=OFF \
129+
-D BUILD_SHARED_LIBS=OFF \
130+
-D BUILD_opencv_apps=OFF \
131+
-D BUILD_opencv_gpu=OFF \
132+
-D BUILD_opencv_video=OFF \
133+
-D BUILD_opencv_contrib=OFF \
134+
-D BUILD_opencv_nonfree=OFF \
135+
-D BUILD_opencv_flann=OFF \
136+
-D BUILD_opencv_features2d=OFF \
137+
-D BUILD_opencv_calib3d=OFF \
138+
-D BUILD_opencv_objdetect=OFF \
139+
-D BUILD_opencv_ml=OFF \
140+
-D BUILD_opencv_photo=OFF \
141+
-D BUILD_DOCS=OFF \
142+
-D BUILD_PACKAGE=OFF \
143+
-D CMAKE_BUILD_TYPE=RELEASE \
144+
-D CMAKE_INSTALL_PREFIX=$DEPS_PATH ..
145+
make -j $(nproc)
146+
make install # user will always have access to home, so no sudo needed
147+
cd -
148+
149+
# Although .so building is explicitly turned off for most libraries, sometimes
150+
# they still get created. So, remove them just to make sure they don't
151+
# interfere, or otherwise we might get libmxnet.so that is not self-contained.
152+
rm $DEPS_PATH/{lib,lib64}/*.{so,so.0}
153+
154+
# Go to the parent path and build mxnet
155+
cd ../../
156+
cp make/pip_$(uname | tr '[:upper:]' '[:lower:]')_cpu.mk config.mk
157+
make -j $(nproc)
158+
159+
# Generate wheel. The output is in the mxnet/tools/pip_package/dist path.
160+
cd tools/pip_package
1161
python setup.py bdist_wheel

‎tools/pip_package/setup.py

+7-5
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,9 @@
22
"""Setup mxnet package."""
33
from __future__ import absolute_import
44
import os
5-
import sys
65
import shutil
76

87
from setuptools import setup, find_packages
9-
from setuptools.extension import Extension
108
from setuptools.dist import Distribution
119

1210
# We can not import `mxnet.info.py` in setup.py directly since mxnet/__init__.py
@@ -20,12 +18,15 @@
2018
__version__ = libinfo['__version__']
2119

2220
class BinaryDistribution(Distribution):
23-
def is_pure(self):
24-
return False
2521
def has_ext_modules(self):
2622
return True
2723

28-
shutil.rmtree(os.path.join(CURRENT_DIR, 'mxnet'))#, ignore_errors=True)
24+
25+
DEPENDENCIES = [
26+
'numpy',
27+
]
28+
29+
shutil.rmtree(os.path.join(CURRENT_DIR, 'mxnet'), ignore_errors=True)
2930
shutil.copytree(os.path.join(CURRENT_DIR, '../../python/mxnet'),
3031
os.path.join(CURRENT_DIR, 'mxnet'))
3132
shutil.copy(LIB_PATH[0], os.path.join(CURRENT_DIR, 'mxnet'))
@@ -37,5 +38,6 @@ def has_ext_modules(self):
3738
packages=find_packages(),
3839
package_data={'mxnet': [os.path.join('mxnet', os.path.basename(LIB_PATH[0]))]},
3940
include_package_data=True,
41+
install_requires=DEPENDENCIES,
4042
distclass=BinaryDistribution,
4143
url='https://github.com/dmlc/mxnet')

0 commit comments

Comments
 (0)