-
Notifications
You must be signed in to change notification settings - Fork 751
/
Copy pathcppbuild.sh
executable file
·192 lines (171 loc) · 7.13 KB
/
cppbuild.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
#!/bin/bash
# This file is meant to be included by the parent cppbuild.sh script
if [[ -z "$PLATFORM" ]]; then
pushd ..
bash cppbuild.sh "$@" pytorch
popd
exit
fi
export BUILD_TEST=0
export CUDACXX="/usr/local/cuda/bin/nvcc"
export CUDA_HOME="/usr/local/cuda"
export CUDNN_HOME="/usr/local/cuda"
export MAX_JOBS=$MAKEJ
export USE_CUDA=0
export USE_NUMPY=0
export USE_OPENMP=1
export USE_SYSTEM_NCCL=1
if [[ "$EXTENSION" == *gpu ]]; then
export USE_CUDA=1
export USE_FAST_NVCC=0
export CUDA_SEPARABLE_COMPILATION=OFF
export TORCH_CUDA_ARCH_LIST="3.5+PTX"
fi
export PYTHON_BIN_PATH=$(which python3)
if [[ $PLATFORM == windows* ]]; then
export PYTHON_BIN_PATH=$(which python.exe)
fi
PYTORCH_VERSION=1.13.1
mkdir -p "$PLATFORM$EXTENSION"
cd "$PLATFORM$EXTENSION"
INSTALL_PATH=`pwd`
if [[ ! -d pytorch ]]; then
git clone https://github.com/pytorch/pytorch
fi
cd pytorch
git reset --hard
git checkout v$PYTORCH_VERSION
git submodule update --init --recursive
git submodule foreach --recursive 'git reset --hard'
# https://github.com/pytorch/pytorch/pull/66219
#patch -Np1 < ../../../pytorch.patch
CPYTHON_PATH="$INSTALL_PATH/../../../cpython/cppbuild/$PLATFORM/"
OPENBLAS_PATH="$INSTALL_PATH/../../../openblas/cppbuild/$PLATFORM/"
NUMPY_PATH="$INSTALL_PATH/../../../numpy/cppbuild/$PLATFORM/"
if [[ -n "${BUILD_PATH:-}" ]]; then
PREVIFS="$IFS"
IFS="$BUILD_PATH_SEPARATOR"
for P in $BUILD_PATH; do
if [[ $(find "$P" -name Python.h) ]]; then
CPYTHON_PATH="$P"
elif [[ -f "$P/include/openblas_config.h" ]]; then
OPENBLAS_PATH="$P"
elif [[ -f "$P/python/numpy/core/include/numpy/numpyconfig.h" ]]; then
NUMPY_PATH="$P"
fi
done
IFS="$PREVIFS"
fi
CPYTHON_PATH="${CPYTHON_PATH//\\//}"
OPENBLAS_PATH="${OPENBLAS_PATH//\\//}"
NUMPY_PATH="${NUMPY_PATH//\\//}"
if [[ -f "$CPYTHON_PATH/include/python3.11/Python.h" ]]; then
# setup.py won't pick up the right libgfortran.so without this
export LD_LIBRARY_PATH="$OPENBLAS_PATH/lib/:$CPYTHON_PATH/lib/:$NUMPY_PATH/lib/"
export PYTHON_BIN_PATH="$CPYTHON_PATH/bin/python3.11"
export PYTHON_INCLUDE_PATH="$CPYTHON_PATH/include/python3.11/"
export PYTHON_LIB_PATH="$CPYTHON_PATH/lib/python3.11/"
export PYTHON_INSTALL_PATH="$INSTALL_PATH/lib/python3.11/site-packages/"
export SSL_CERT_FILE="$CPYTHON_PATH/lib/python3.11/site-packages/pip/_vendor/certifi/cacert.pem"
chmod +x "$PYTHON_BIN_PATH"
elif [[ -f "$CPYTHON_PATH/include/Python.h" ]]; then
CPYTHON_PATH=$(cygpath $CPYTHON_PATH)
OPENBLAS_PATH=$(cygpath $OPENBLAS_PATH)
NUMPY_PATH=$(cygpath $NUMPY_PATH)
export PATH="$OPENBLAS_PATH:$CPYTHON_PATH:$NUMPY_PATH:$PATH"
export PYTHON_BIN_PATH="$CPYTHON_PATH/bin/python.exe"
export PYTHON_INCLUDE_PATH="$CPYTHON_PATH/include/"
export PYTHON_LIB_PATH="$CPYTHON_PATH/lib/"
export PYTHON_INSTALL_PATH="$INSTALL_PATH/lib/site-packages/"
export SSL_CERT_FILE="$CPYTHON_PATH/lib/pip/_vendor/certifi/cacert.pem"
fi
export PYTHONPATH="$PYTHON_INSTALL_PATH:$NUMPY_PATH/python/"
mkdir -p "$PYTHON_INSTALL_PATH"
export CFLAGS="-I$CPYTHON_PATH/include/ -I$PYTHON_LIB_PATH/include/python/ -L$CPYTHON_PATH/lib/ -L$CPYTHON_PATH/libs/"
export PYTHONNOUSERSITE=1
$PYTHON_BIN_PATH -m pip install --target=$PYTHON_LIB_PATH setuptools==59.1.0 pyyaml==6.0 typing_extensions==4.0.0
case $PLATFORM in
linux-x86)
export CC="gcc -m32"
export CXX="g++ -m32"
;;
linux-x86_64)
export CC="gcc -m64"
export CXX="g++ -m64"
;;
macosx-*)
export CC="clang"
export CXX="clang++"
;;
windows-x86_64)
if which ccache.exe; then
export CC="ccache.exe cl.exe"
export CXX="ccache.exe cl.exe"
# export CUDAHOSTCC="cl.exe"
# export CUDAHOSTCXX="cl.exe"
else
export CC="cl.exe"
export CXX="cl.exe"
fi
if [[ -n "${CUDA_PATH:-}" ]]; then
export CUDACXX="$CUDA_PATH/bin/nvcc"
export CUDA_HOME="$CUDA_PATH"
export CUDNN_HOME="$CUDA_PATH"
fi
export CFLAGS="-I$CPYTHON_PATH/include/ -I$PYTHON_LIB_PATH/include/python/"
;;
*)
echo "Error: Platform \"$PLATFORM\" is not supported"
return 0
;;
esac
# work around issues with the build system
sedinplace '/Werror/d' CMakeLists.txt third_party/fbgemm/CMakeLists.txt third_party/fmt/CMakeLists.txt
sedinplace 's/build_python=True/build_python=False/g' setup.py
sedinplace 's/ build_deps()/ build_deps(); sys.exit()/g' setup.py
sedinplace 's/AND NOT DEFINED ENV{CUDAHOSTCXX}//g' cmake/public/cuda.cmake
sedinplace 's/CMAKE_CUDA_FLAGS "/CMAKE_CUDA_FLAGS " --use-local-env /g' CMakeLists.txt
# work around some compiler bugs
sedinplace 's/!defined(__INTEL_COMPILER))/!defined(__INTEL_COMPILER) \&\& (__GNUC__ < 11))/g' third_party/XNNPACK/src/xnnpack/intrinsics-polyfill.h
sedinplace 's/using ExpandingArrayDouble/public: using ExpandingArrayDouble/g' ./torch/csrc/api/include/torch/nn/options/pooling.h
sedinplace 's/typedef c10::variant/public: typedef c10::variant/g' ./torch/csrc/api/include/torch/nn/options/upsampling.h
sedinplace 's/std::copysign/copysignf/g' aten/src/ATen/native/cuda/*.cu
sedinplace 's/std::trunc/truncf/g' aten/src/ATen/native/cuda/*.cu
sedinplace 's/std::floor/floorf/g' aten/src/ATen/native/cuda/*.cu
sedinplace 's/std::ceil/ceilf/g' aten/src/ATen/native/cuda/*.cu
sedinplace 's/round(/roundf(/g' aten/src/ATen/native/cuda/*.cu
sedinplace 's/floor(/floorf(/g' aten/src/ATen/native/cuda/*.cu
sedinplace 's/ceil(/ceilf(/g' aten/src/ATen/native/cuda/*.cu
sedinplace '/#include <thrust\/device_vector.h>/a\
#include <thrust\/host_vector.h>\
' caffe2/utils/math_gpu.cu
# allow setting the build directory and passing CUDA options
sedinplace "s/BUILD_DIR = .build./BUILD_DIR = os.environ['BUILD_DIR'] if 'BUILD_DIR' in os.environ else 'build'/g" tools/setup_helpers/env.py
sedinplace "s/var.startswith(('BUILD_', 'USE_', 'CMAKE_'))/var.startswith(('BUILD_', 'USE_', 'CMAKE_', 'CUDA_'))/g" tools/setup_helpers/cmake.py
# allow resizing std::vector<at::indexing::TensorIndex>
sedinplace 's/TensorIndex(c10::nullopt_t)/TensorIndex(c10::nullopt_t none = None)/g' aten/src/ATen/TensorIndexing.h
# add missing declarations
sedinplace '/using ExampleType = ExampleType_;/a\
using BatchType = ChunkType;\
using DataType = ExampleType;\
' torch/csrc/api/include/torch/data/datasets/chunk.h
sedinplace '/^};/a\
TORCH_API std::ostream& operator<<(std::ostream& stream, const nn::Module& module);\
' torch/csrc/api/include/torch/nn/module.h
sedinplace 's/char(\(.*\))/\1/g' torch/csrc/jit/serialization/pickler.h
#USE_GLOO=0 USE_MKLDNN=0 \
"$PYTHON_BIN_PATH" setup.py build
rm -Rf ../lib
ln -sf pytorch/torch/include ../include
ln -sf pytorch/torch/lib ../lib
ln -sf pytorch/torch/bin ../bin
# fix library with correct rpath on Mac
case $PLATFORM in
macosx-*)
cp /usr/local/lib/libomp.dylib ../lib/libiomp5.dylib
chmod +w ../lib/libiomp5.dylib
install_name_tool -id @rpath/libiomp5.dylib ../lib/libiomp5.dylib
install_name_tool -change @rpath/libomp.dylib @rpath/libiomp5.dylib ../lib/libtorch_cpu.dylib
;;
esac
cd ../..