[ML] Upgrade to PyTorch 1.11 on Linux (#2238)

Together with #2233 and #2235 this completes the upgrade to
PyTorch 1.11 for all platforms.

A small change to the way things are done is that we now copy
all MKL libraries into the Docker images, then copy the required
subset into our release distribution. This means we can change
the subset of MKL libraries that we redistribute without having
to rebuild our Docker images.

Fixes #2126

Author: droberts195

3rd_party/3rd_party.sh

@@ -70,7 +70,7 @@ case `uname` in
                 MKL_LOCATION=/usr/local/gcc103/lib
                 MKL_EXTENSION=.so
                 MKL_PREFIX=libmkl_
-                MKL_LIBRARIES=`cd "$MKL_LOCATION" && ls $MKL_PREFIX*$MKL_EXTENSION`
+                MKL_LIBRARIES='avx avx2 avx512 avx512_mic core def gnu_thread intel_lp64 mc3 vml_avx vml_avx2 vml_avx512 vml_avx512_mic vml_cmpt vml_def vml_mc3'
             fi
             BOOST_EXTENSION=mt-${BOOST_ARCH}-1_77.so.1.77.0
             BOOST_LIBRARIES='atomic chrono date_time filesystem iostreams log log_setup program_options regex system thread unit_test_framework'
@@ -265,11 +265,11 @@ fi
 if [ ! -z "$MKL_LOCATION" ] ; then
     if [ ! -z "$MKL_LIBRARIES" ] ; then
         if [ -n "$INSTALL_DIR" ] ; then
+            rm -f $INSTALL_DIR/$MKL_PREFIX*$MKL_EXTENSION
             for LIBRARY in $MKL_LIBRARIES
             do
-                rm -f $INSTALL_DIR/$LIBRARY
-                cp $MKL_LOCATION/$LIBRARY $INSTALL_DIR
-                chmod u+wx $INSTALL_DIR/$LIBRARY
+                cp $MKL_LOCATION/$MKL_PREFIX$LIBRARY$MKL_EXTENSION $INSTALL_DIR
+                chmod u+wx $INSTALL_DIR/$MKL_PREFIX$LIBRARY$MKL_EXTENSION
             done
         fi
     else

3rd_party/licenses/pytorch-INFO.csv

@@ -1,2 +1,2 @@
 name,version,revision,url,license,copyright,sourceURL
-PyTorch,1.9.0,d69c22dd61a2f006dcfe1e3ea8468a3ecaf931aa,https://pytorch.org,BSD-3-Clause,,
+PyTorch,1.11.0,bc2c6edaf163b1a1330e37a6e34caf8c553e4755,https://pytorch.org,BSD-3-Clause,,

build-setup/linux.md

@@ -274,18 +274,13 @@ sudo yum-config-manager --add-repo https://yum.repos.intel.com/mkl/setup/intel-m
 sudo yum -y install intel-mkl-2020.4-912
 ```
 
-Then copy the required libraries to the system directory:
+Then copy the shared libraries to the system directory:
 
 ```
-sudo cp /opt/intel/mkl/lib/intel64/libmkl_intel_lp64.so /usr/local/gcc103/lib
-sudo cp /opt/intel/mkl/lib/intel64/libmkl_core.so /usr/local/gcc103/lib
-sudo cp /opt/intel/mkl/lib/intel64/libmkl_def.so /usr/local/gcc103/lib
-sudo cp /opt/intel/mkl/lib/intel64/libmkl_gnu_thread.so /usr/local/gcc103/lib
-sudo cp /opt/intel/mkl/lib/intel64/libmkl_avx*.so /usr/local/gcc103/lib
-sudo cp /opt/intel/mkl/lib/intel64/libmkl_vml*.so /usr/local/gcc103/lib
+sudo cp /opt/intel/mkl/lib/intel64/libmkl*.so /usr/local/gcc103/lib
 ```
 
-### PyTorch 1.9.0
+### PyTorch 1.11.0
 
 PyTorch requires that certain Python modules are installed. Install these modules with `pip` using the same Python version you will build PyTorch with. If you followed the instructions above and built Python from source use `python3.7`:
 
@@ -302,7 +297,7 @@ sudo /usr/local/gcc103/bin/python3.7 -m pip install install numpy pyyaml setupto
 Then obtain the PyTorch code:
 
 ```
-git clone --depth=1 --branch=v1.9.0 git@github.com:pytorch/pytorch.git
+git clone --depth=1 --branch=v1.11.0 git@github.com:pytorch/pytorch.git
 cd pytorch
 git submodule sync
 git submodule update --init --recursive
@@ -330,7 +325,8 @@ export USE_MKLDNN=ON
 export USE_QNNPACK=OFF
 export USE_PYTORCH_QNNPACK=OFF
 [ $(uname -m) = x86_64 ] && export USE_XNNPACK=OFF
-export PYTORCH_BUILD_VERSION=1.9.0
+[ $(uname -m) != x86_64 ] && export USE_BREAKPAD=OFF
+export PYTORCH_BUILD_VERSION=1.11.0
 export PYTORCH_BUILD_NUMBER=1
 /usr/local/gcc103/bin/python3.7 setup.py install
 ```

dev-tools/docker/build_linux_aarch64_cross_build_image.sh

@@ -22,7 +22,7 @@
 HOST=docker.elastic.co
 ACCOUNT=ml-dev
 REPOSITORY=ml-linux-aarch64-cross-build
-VERSION=6
+VERSION=7
 
 set -e
 

dev-tools/docker/build_linux_aarch64_native_build_image.sh

@@ -34,7 +34,7 @@ sleep 5
 HOST=docker.elastic.co
 ACCOUNT=ml-dev
 REPOSITORY=ml-linux-aarch64-native-build
-VERSION=6
+VERSION=7
 
 set -e
 

dev-tools/docker/build_linux_build_image.sh

@@ -34,7 +34,7 @@ sleep 5
 HOST=docker.elastic.co
 ACCOUNT=ml-dev
 REPOSITORY=ml-linux-build
-VERSION=20
+VERSION=21
 
 set -e
 

dev-tools/docker/linux_aarch64_cross_builder/Dockerfile

@@ -10,7 +10,7 @@
 #
 
 # Increment the version here when a new tools/3rd party components image is built
-FROM docker.elastic.co/ml-dev/ml-linux-aarch64-cross-build:6
+FROM docker.elastic.co/ml-dev/ml-linux-aarch64-cross-build:7
 
 MAINTAINER David Roberts <dave.roberts@elastic.co>
 

dev-tools/docker/linux_aarch64_cross_image/Dockerfile

@@ -24,7 +24,7 @@ RUN \
 RUN \
   mkdir -p /usr/local/sysroot-aarch64-linux-gnu/usr && \
   cd /usr/local/sysroot-aarch64-linux-gnu/usr && \
-  wget --quiet -O - https://s3-eu-west-1.amazonaws.com/prelert-artifacts/dependencies/usr-aarch64-linux-gnu-6.tar.bz2 | tar jxf - && \
+  wget --quiet -O - https://s3-eu-west-1.amazonaws.com/prelert-artifacts/dependencies/usr-aarch64-linux-gnu-7.tar.bz2 | tar jxf - && \
   cd .. && \
   ln -s usr/lib lib && \
   ln -s usr/lib64 lib64

dev-tools/docker/linux_aarch64_native_builder/Dockerfile

@@ -10,7 +10,7 @@
 #
 
 # Increment the version here when a new tools/3rd party components image is built
-FROM docker.elastic.co/ml-dev/ml-linux-aarch64-native-build:6
+FROM docker.elastic.co/ml-dev/ml-linux-aarch64-native-build:7
 
 MAINTAINER David Roberts <dave.roberts@elastic.co>
 

dev-tools/docker/linux_aarch64_native_image/Dockerfile

@@ -127,7 +127,7 @@ ENV PATH=${PATH}:/usr/local/cmake/bin
 # If the PyTorch branch is changed also update PYTORCH_BUILD_VERSION
 RUN \
   cd ${build_dir} && \
-  git -c advice.detachedHead=false clone --depth=1 --branch=v1.9.0 https://github.com/pytorch/pytorch.git && \
+  git -c advice.detachedHead=false clone --depth=1 --branch=v1.11.0 https://github.com/pytorch/pytorch.git && \
   cd pytorch && \
   git submodule sync && \
   git submodule update --init --recursive && \
@@ -140,7 +140,8 @@ RUN \
   export USE_MKLDNN=ON && \
   export USE_QNNPACK=OFF && \
   export USE_PYTORCH_QNNPACK=OFF && \
-  export PYTORCH_BUILD_VERSION=1.9.0 && \
+  export USE_BREAKPAD=OFF && \
+  export PYTORCH_BUILD_VERSION=1.11.0 && \
   export PYTORCH_BUILD_NUMBER=1 && \
   /usr/local/bin/python3.7 setup.py install && \
   mkdir /usr/local/gcc103/include/pytorch && \

dev-tools/docker/linux_aarch64_native_tester/Dockerfile

@@ -10,7 +10,7 @@
 #
 
 # Increment the version here when a new tools/3rd party components image is built
-FROM docker.elastic.co/ml-dev/ml-linux-aarch64-native-build:6
+FROM docker.elastic.co/ml-dev/ml-linux-aarch64-native-build:7
 
 MAINTAINER David Roberts <dave.roberts@elastic.co>
 

dev-tools/docker/linux_builder/Dockerfile

@@ -10,7 +10,7 @@
 #
 
 # Increment the version here when a new tools/3rd party components image is built
-FROM docker.elastic.co/ml-dev/ml-linux-build:20
+FROM docker.elastic.co/ml-dev/ml-linux-build:21
 
 MAINTAINER David Roberts <dave.roberts@elastic.co>
 

dev-tools/docker/linux_image/Dockerfile

@@ -128,18 +128,13 @@ ENV PATH=${PATH}:/usr/local/cmake/bin
 RUN \
   yum-config-manager --add-repo https://yum.repos.intel.com/mkl/setup/intel-mkl.repo && \
   yum -y install intel-mkl-2020.4-912 && \
-  cp /opt/intel/mkl/lib/intel64/libmkl_intel_lp64.so /usr/local/gcc103/lib && \
-  cp /opt/intel/mkl/lib/intel64/libmkl_core.so /usr/local/gcc103/lib && \
-  cp /opt/intel/mkl/lib/intel64/libmkl_def.so /usr/local/gcc103/lib && \
-  cp /opt/intel/mkl/lib/intel64/libmkl_gnu_thread.so /usr/local/gcc103/lib && \
-  cp /opt/intel/mkl/lib/intel64/libmkl_avx*.so /usr/local/gcc103/lib && \
-  cp /opt/intel/mkl/lib/intel64/libmkl_vml*.so /usr/local/gcc103/lib
+  cp /opt/intel/mkl/lib/intel64/libmkl*.so /usr/local/gcc103/lib
 
 # Clone PyTorch and build LibTorch
 # If the PyTorch branch is changed also update PYTORCH_BUILD_VERSION
 RUN \
   cd ${build_dir} && \
-  git -c advice.detachedHead=false clone --depth=1 --branch=v1.9.0 https://github.com/pytorch/pytorch.git && \
+  git -c advice.detachedHead=false clone --depth=1 --branch=v1.11.0 https://github.com/pytorch/pytorch.git && \
   cd pytorch && \
   git submodule sync && \
   git submodule update --init --recursive && \
@@ -153,7 +148,7 @@ RUN \
   export USE_QNNPACK=OFF && \
   export USE_PYTORCH_QNNPACK=OFF && \
   export USE_XNNPACK=OFF && \
-  export PYTORCH_BUILD_VERSION=1.9.0 && \
+  export PYTORCH_BUILD_VERSION=1.11.0 && \
   export PYTORCH_BUILD_NUMBER=1 && \
   /usr/local/bin/python3.7 setup.py install && \
   mkdir /usr/local/gcc103/include/pytorch && \

dev-tools/docker/linux_tester/Dockerfile

@@ -10,7 +10,7 @@
 #
 
 # Increment the version here when a new tools/3rd party components image is built
-FROM docker.elastic.co/ml-dev/ml-linux-build:20
+FROM docker.elastic.co/ml-dev/ml-linux-build:21
 
 MAINTAINER David Roberts <dave.roberts@elastic.co>
 

docs/CHANGELOG.asciidoc

@@ -28,6 +28,13 @@
 
 //=== Regressions
 
+== {es} version 8.3.0
+
+=== Enhancements
+
+* Upgrade PyTorch to version 1.11. (See {ml-pull}2233[#2233], {ml-pull}2235[#2235]
+  and {ml-pull}2238[#2238].)
+
 == {es} version 8.2.0
 
 === Enhancements