CI: enable testing for PT 1.11 (#11792)

Borda · carmocca · akihironitta · Borda · commit 0b54a58e53f3 · 2022-03-10T20:29:47.000+01:00
* enable PT 1.11
* horovod
* Apply suggestions from code review

Co-authored-by: Carlos Mocholí &lt;carlossmocholi@gmail.com&gt;
Co-authored-by: Aki Nitta &lt;nitta@akihironitta.com&gt;
diff --git a/.github/workflows/ci_test-conda.yml b/.github/workflows/ci_test-conda.yml
@@ -23,7 +23,10 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ["3.8"]  # previous to last Python version as that one is already used in test-full
-        pytorch-version: ["1.8", "1.9", "1.10"]  # nightly: add when there's a release candidate
+        pytorch-version: ["1.8", "1.9", "1.10"]
+        # nightly: add when there's a release candidate
+        include:
+          - {python-version: "3.9", pytorch-version: "1.11"}
 
     timeout-minutes: 30
     steps:
@@ -36,7 +39,7 @@ jobs:
         # adjust versions according installed Torch version
         python ./requirements/adjust-versions.py requirements/extra.txt
         python ./requirements/adjust-versions.py requirements/examples.txt
-        pip install --requirement requirements/devel.txt --find-links https://download.pytorch.org/whl/nightly/torch_nightly.html
+        pip install --requirement requirements/devel.txt --find-links https://download.pytorch.org/whl/test/torch_test.html
         # set a per-test timeout of 2.5 minutes to fail sooner. this aids with hanging tests
         pip install pytest-timeout
         pip list
diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
@@ -24,9 +24,12 @@ jobs:
         python-version: ["3.7", "3.9"]  # minimum, maximum
         requires: ["oldest", "latest"]
         release: ["stable"]
-        #include:
-          # nightly: add when there's a release candidate
-          #- {os: ubuntu-20.04, python-version: "3.10", requires: "latest", release: "pre"}
+        exclude:
+          # Skip if torch<1.8 and py3.9 on Linux: https://github.com/pytorch/pytorch/issues/50014
+          - {os: ubuntu-20.04, python-version: "3.9", requires: "oldest"}
+        # TODO: re-enable RC testing
+        # include:
+        #   - {os: ubuntu-20.04, python-version: "3.10", requires: "latest", release: "pre"}
 
     timeout-minutes: 40
 
@@ -83,7 +86,8 @@ jobs:
     - name: Install dependencies
       run: |
         flag=$(python -c "print('--pre' if '${{matrix.release}}' == 'pre' else '')" 2>&1)
-        pip install --requirement requirements.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade $flag
+        url=$(python -c "print('test/cpu/torch_test.html' if '${{matrix.release}}' == 'pre' else 'cpu/torch_stable.html')" 2>&1)
+        pip install --requirement requirements.txt --upgrade $flag --find-links "https://download.pytorch.org/whl/${url}"
         # adjust versions according installed Torch version
         python ./requirements/adjust-versions.py requirements/examples.txt
         pip install --requirement requirements/examples.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade
diff --git a/.github/workflows/events-nightly.yml b/.github/workflows/events-nightly.yml
@@ -123,9 +123,14 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # the config used in '.github/workflows/ci_test-conda.yml'
-        python_version: ["3.8"]
-        pytorch_version: ["1.8", "1.9", "1.10"]
+        include:
+          # see: https://pytorch.org/get-started/previous-versions/
+          - {python_version: "3.8", pytorch_version: "1.8", cuda_version: "11.1"}
+          - {python_version: "3.8", pytorch_version: "1.9", cuda_version: "11.1"}
+          - {python_version: "3.8", pytorch_version: "1.10", cuda_version: "11.1"}
+          - {python_version: "3.9", pytorch_version: "1.11", cuda_version: "11.3.1"}
+          #  nightly: add when there's a release candidate
+          #  - {python_version: "3.9", pytorch_version: "1.12"}
 
     steps:
       - name: Checkout
@@ -138,20 +143,14 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
-      # see: https://pytorch.org/get-started/previous-versions/
-      - run: |
-          cuda=$(python -c "from distutils.version import LooseVersion as LVer ; print(11.1 if LVer('${{matrix.pytorch_version}}') > LVer('1.7') else 10.2)" 2>&1)
-          echo "::set-output name=CUDA::$cuda"
-        id: extend
-
       - name: Publish Conda to Docker Hub
         # publish master/release
         uses: docker/build-push-action@v2
         with:
           build-args: |
             PYTHON_VERSION=${{ matrix.python_version }}
             PYTORCH_VERSION=${{ matrix.pytorch_version }}
-            CUDA_VERSION=${{ steps.extend.outputs.CUDA }}
+            CUDA_VERSION=${{ matrix.cuda_version }}
           file: dockers/base-conda/Dockerfile
           push: ${{ env.PUSH_TO_HUB }}
           tags: pytorchlightning/pytorch_lightning:base-conda-py${{ matrix.python_version }}-torch${{ matrix.pytorch_version }}
diff --git a/dockers/base-conda/Dockerfile b/dockers/base-conda/Dockerfile
@@ -12,13 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ARG CUDA_VERSION=11.1
+ARG CUDA_VERSION=11.3.1
 
 FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu18.04
 
-ARG PYTHON_VERSION=3.8
-ARG PYTORCH_VERSION=1.6
-ARG CONDA_VERSION=4.9.2
+ARG PYTHON_VERSION=3.9
+ARG PYTORCH_VERSION=1.8
+ARG CONDA_VERSION=4.11.0
 
 SHELL ["/bin/bash", "-c"]
 # https://techoverflow.net/2019/05/18/how-to-fix-configuring-tzdata-interactive-input-when-building-docker-images/
@@ -68,14 +68,16 @@ COPY environment.yml environment.yml
 
 # conda init
 RUN conda update -n base -c defaults conda && \
-    conda create -y --name $CONDA_ENV python=${PYTHON_VERSION} pytorch=${PYTORCH_VERSION} cudatoolkit=${CUDA_VERSION} -c nvidia -c pytorch -c pytorch-test -c pytorch-nightly && \
+    conda install mamba -n base -c conda-forge && \
+    mamba create -y --name $CONDA_ENV python=${PYTHON_VERSION} pytorch=${PYTORCH_VERSION} torchvision torchtext cudatoolkit=${CUDA_VERSION} -c nvidia -c pytorch -c pytorch-test -c pytorch-nightly && \
     conda init bash && \
-    # NOTE: this requires that the channel is presented in the yaml before packages
-    python -c "import re ; fname = 'environment.yml' ; req = re.sub(r'- python[>=]+[\d\.]+', '# - python=${PYTHON_VERSION}', open(fname).read()) ; open(fname, 'w').write(req)" && \
-    python -c "import re ; fname = 'environment.yml' ; req = re.sub(r'- pytorch[>=]+[\d\.]+', '# - pytorch=${PYTORCH_VERSION}', open(fname).read()) ; open(fname, 'w').write(req)" && \
+    # NOTE: this requires that the channel is presented in the yaml before packages \
+    printf "import re;\nfname = 'environment.yml';\nreq = open(fname).read();\nfor n in ['python', 'pytorch', 'torchtext', 'torchvision']:\n    req = re.sub(rf'- {n}[>=]+', f'# - {n}=', req);\nopen(fname, 'w').write(req)" > prune.py && \
+    python prune.py && \
+    rm prune.py && \
     cat environment.yml && \
-    conda env update --name $CONDA_ENV --file environment.yml && \
-    conda clean -ya && \
+    mamba env update --name $CONDA_ENV --file environment.yml && \
+    mamba clean -ya && \
     rm environment.yml
 
 ENV \
@@ -96,7 +98,7 @@ RUN \
     python requirements_prune_packages.py requirements-extra.txt "horovod" && \
     python requirements_adjust_versions.py requirements-examples.txt && \
     # Install remaining requirements
-    pip install -r requirements-extra.txt --no-cache-dir && \
+    pip install -r requirements-extra.txt --no-cache-dir --find-links https://download.pytorch.org/whl/test/torch_test.html && \
     pip install -r requirements-examples.txt --no-cache-dir --find-links https://download.pytorch.org/whl/test/torch_test.html && \
     pip install -r requirements-test.txt --no-cache-dir
 
