Merge pull request pytorch#217 from pjh5/master

Adding AWS S3Upload Task for wheel Windows jobs Add retry logic for publishing package Fix conda-build by rearranging PATHs Turn off wheels builds Turn on wheels builds Change flag name . . , Unify nightly default variables minor fixes More minor fixes Skip CUDA test if no GPU available Pin vs version to 15.4.5 Fix VS2017 installation Fix variable passing issues Fix lint Fix error codes Disable env fix for legacy VS Test CPU build Revert Fix create dir error
peterjc123 · Nov 30, 2018 · ec14f2e · ec14f2e
2 parents 1f5ecf4 + eb823d3
commit ec14f2e
Show file tree

Hide file tree

Showing 15 changed files with 429 additions and 66 deletions.
diff --git a/.gitignore b/.gitignore
@@ -68,3 +68,5 @@ conda/torchvision-src
 wheel/pytorch-src
 wheel/torchvision-src
 torch_nightly.html
+
+node_modules
diff --git a/conda/build_pytorch.sh b/conda/build_pytorch.sh
@@ -149,6 +149,10 @@ elif [[ "$OSTYPE" == "msys" ]]; then
     mkdir -p "$WIN_PACKAGE_WORK_DIR" || true
     pytorch_rootdir="$(realpath ${WIN_PACKAGE_WORK_DIR})/pytorch"
     git config --system core.longpaths true
+    # The jobs are seperated on Windows, so we don't need to clone again.
+    if [[ -d "$NIGHTLIES_PYTORCH_ROOT" ]]; then
+        cp -R "$NIGHTLIES_PYTORCH_ROOT" "$WIN_PACKAGE_WORK_DIR"
+    fi
 elif [[ -d '/pytorch' ]]; then
     pytorch_rootdir='/pytorch'
 else
@@ -186,7 +190,7 @@ elif [[ "$OSTYPE" == "msys" ]]; then
     curl -k https://repo.continuum.io/miniconda/Miniconda3-latest-Windows-x86_64.exe -o "$miniconda_exe"
     "$SOURCE_DIR/install_conda.bat" && rm "$miniconda_exe"
     pushd $tmp_conda
-    export PATH="$(pwd):$(pwd)/Scripts:$PATH"
+    export PATH="$(pwd):$(pwd)/Library/usr/bin:$(pwd)/Library/bin:$(pwd)/Scripts:$(pwd)/bin:$PATH"
     echo $PATH
     popd
     conda install -y conda-build
@@ -265,10 +269,11 @@ for py_ver in "${DESIRED_PYTHON[@]}"; do
 
     # We need to build the compiler activation scripts first on Windows
     if [[ "$OSTYPE" == "msys" ]]; then
-        conda build -c "$ANACONDA_USER" \
-                     --no-anaconda-upload \
-                     --output-folder "$output_folder" \
-                     vs2017
+        time VSDEVCMD_ARGS=${VSDEVCMD_ARGS[@]} \
+             conda build -c "$ANACONDA_USER" \
+                         --no-anaconda-upload \
+                         --output-folder "$output_folder" \
+                         vs2017
     fi
 
     # Output the meta.yaml for easy debugging

diff --git a/conda/pytorch-nightly/bld.bat b/conda/pytorch-nightly/bld.bat
@@ -55,7 +55,7 @@ if NOT "%build_with_cuda%" == "" (
 
     if "%CUDA_VERSION%" == "80" (
         :: Only if you use Ninja with CUDA 8
-        set "CUDA_HOST_COMPILER=%VS140COMNTOOLS%\..\..\VC\bin\amd64\cl.exe"
+        set "CUDAHOSTCXX=%VS140COMNTOOLS%\..\..\VC\bin\amd64\cl.exe"
     )
 )
 

diff --git a/conda/pytorch-nightly/meta.yaml b/conda/pytorch-nightly/meta.yaml
@@ -42,7 +42,7 @@ build:
     - DEVELOPER_DIR
     - DEBUG
     - NO_FBGEMM
-    - USE_SCCACHE #[win]
+    - USE_SCCACHE # [win]
   # Features go here
 
 test:

diff --git a/conda/vs2017/install_activate.bat b/conda/vs2017/install_activate.bat
@@ -8,8 +8,13 @@ IF "%cross_compiler_target_platform%" == "win-64" (
   set "target_platform=amd64"
   echo SET "CMAKE_GENERATOR=Visual Studio %VER% %YEAR% Win64" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
   echo pushd "%%VSINSTALLDIR%%" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
-  echo CALL "VC\Auxiliary\Build\vcvarsall.bat" x64 >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
-  echo CALL "VC\Auxiliary\Build\vcvarsall.bat" x86_amd64 >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
+  IF "%VSDEVCMD_ARGS%" == "" (
+    echo CALL "VC\Auxiliary\Build\vcvarsall.bat" x64 >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
+    echo CALL "VC\Auxiliary\Build\vcvarsall.bat" x86_amd64 >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
+  ) ELSE (
+    echo CALL "VC\Auxiliary\Build\vcvarsall.bat" x64 %VSDEVCMD_ARGS% >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
+    echo CALL "VC\Auxiliary\Build\vcvarsall.bat" x86_amd64 %VSDEVCMD_ARGS% >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
+  )
   echo popd >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
   ) else (
   set "target_platform=x86"

diff --git a/conda/vs2017/meta.yaml b/conda/vs2017/meta.yaml
@@ -9,6 +9,8 @@ package:
 
 build:
   skip: True  [not win]
+  script_env:
+    - VSDEVCMD_ARGS # [win]
 
 outputs:
   - name: vs{{ vsyear }}_{{ cross_compiler_target_platform }}

diff --git a/run_tests.sh b/run_tests.sh
@@ -74,6 +74,14 @@ else
     python -c 'import torch; exit(0 if torch.backends.mkl.is_available() else 1)'
 fi
 
+if [[ "$OSTYPE" == "msys" ]]; then
+    GPUS=$(wmic path win32_VideoController get name)
+    if [[ ! "$GPUS" == *NVIDIA* ]]; then
+        echo "Skip CUDA tests for machines without a Nvidia GPU card"
+        exit 0
+    fi
+fi
+
 # Test that CUDA builds are setup correctly
 if [[ "$cuda_ver" != 'cpu' ]]; then
     # Test CUDA archs

diff --git a/windows/azure-pipelines.yml b/windows/azure-pipelines.yml
@@ -1,9 +1,9 @@
 
 variables:
-  pytorch.build.version: 1.0
-  pytorch.build.number: 1
-  pytorch.branch: master # override the default of build.version
-  pytorch.repo: pytorch
+  # pytorch.build.version: '1.0'
+  # pytorch.build.number: 1
+  # pytorch.branch: master
+  # pytorch.repo: pytorch
   build.linux.enabled: false
   build.cpu.enabled: true
   build.cuda.enabled: true
@@ -37,11 +37,20 @@ jobs:
         BUILD_PYTHONLESS: 1
 
   pool:
-    vmImage: 'vs2017-win2016'
+    vmImage: 'win1803'
 
   steps:
   - task: BatchScript@1
     condition: eq(variables['self.hosted.agent.cpu'], 'false')
+    displayName: 'Install VS 2017'
+    inputs:
+      filename: 'windows/internal/vs_install.bat'
+
+      modifyEnvironment: true
+
+  - task: BatchScript@1
+    condition: eq(variables['self.hosted.agent.cpu'], 'false')
+    enabled: false
     displayName: 'Environment fix'
     inputs:
       filename: 'windows/internal/env_fix.bat'
@@ -56,6 +65,13 @@ jobs:
 
       modifyEnvironment: true
 
+  - task: BatchScript@1
+    displayName: 'Setup nightly variables'
+    inputs:
+      filename: 'windows/internal/nightly_defaults.bat'
+
+      modifyEnvironment: true
+
   - script: 'call windows/build_pytorch.bat %CUDA_VERSION% %PYTORCH_BUILD_VERSION% %PYTORCH_BUILD_NUMBER%'
     displayName: Build
     env:
@@ -67,12 +83,9 @@ jobs:
       testRunTitle: 'Publish test results'
     enabled: false
 
-  - task: BatchScript@1
-    displayName: 'Upload artifacts'
-    inputs:
-      filename: 'windows/internal/publish.bat'
-
-      modifyEnvironment: false
+  - template: templates/upload_to_s3.yml
+    parameters:
+      cuVer: 'cpu'
 
 - job: 'Windows_CPU_Conda_Build'
   timeoutInMinutes: 0
@@ -81,8 +94,6 @@ jobs:
   variables:
     PACKAGE: conda
     CUDA_VERSION: cpu
-    PYTORCH_REPO: pytorch
-    TORCH_CONDA_BUILD_FOLDER: pytorch-nightly
     USE_SCCACHE: $(use.sccache.cpu)
 
   strategy:
@@ -96,11 +107,20 @@ jobs:
         DESIRED_PYTHON: 3.7
 
   pool:
-    vmImage: 'vs2017-win2016'
+    vmImage: 'win1803'
 
   steps:
   - task: BatchScript@1
     condition: eq(variables['self.hosted.agent.cpu'], 'false')
+    displayName: 'Install VS 2017'
+    inputs:
+      filename: 'windows/internal/vs_install.bat'
+
+      modifyEnvironment: true
+
+  - task: BatchScript@1
+    condition: eq(variables['self.hosted.agent.cpu'], 'false')
+    enabled: false
     displayName: 'Environment fix'
     inputs:
       filename: 'windows/internal/env_fix.bat'
@@ -115,7 +135,14 @@ jobs:
 
       modifyEnvironment: true
 
-  - script: 'mkdir %PYTORCH_FINAL_PACKAGE_DIR%'
+  - task: BatchScript@1
+    displayName: 'Setup nightly variables'
+    inputs:
+      filename: 'windows/internal/nightly_defaults.bat'
+
+      modifyEnvironment: true
+
+  - script: 'if not exist %PYTORCH_FINAL_PACKAGE_DIR% mkdir %PYTORCH_FINAL_PACKAGE_DIR%'
     displayName: 'Create final package directory'
     env:
       PYTORCH_FINAL_PACKAGE_DIR: '$(Build.Repository.LocalPath)\windows\output'
@@ -146,7 +173,6 @@ jobs:
   variables:
     PACKAGE: conda
     CUDA_VERSION: cpu
-    PYTORCH_REPO: pytorch
     TORCH_CONDA_BUILD_FOLDER: pytorch-nightly
 
   strategy:
@@ -178,26 +204,55 @@ jobs:
       MAX_JOBS: 2
       PYTORCH_FINAL_PACKAGE_DIR: '$(Build.Repository.LocalPath)/output'
 
-- job: 'Windows_CUDA90_Wheels_Build'
+- job: 'Windows_CUDA_Wheels_Build'
   timeoutInMinutes: 0
   cancelTimeoutInMinutes: 5
   condition: and(eq(variables['build.cuda.enabled'], 'true'), eq(variables['build.wheels.enabled'], 'true'))
   variables:
     PACKAGE: wheels
-    CUDA_VERSION: 90
     USE_SCCACHE: $(use.sccache.cuda)
   strategy:
     maxParallel: 10
     matrix:
-      PY3.5:
+      PY3.5_90:
         DESIRED_PYTHON: 3.5
-      PY3.6:
+        CUDA_VERSION: 90
+      PY3.6_90:
         DESIRED_PYTHON: 3.6
-      PY3.7:
+        CUDA_VERSION: 90
+      PY3.7_90:
         DESIRED_PYTHON: 3.7
-      LIBTORCH:
+        CUDA_VERSION: 90
+      LIBTORCH_90:
         DESIRED_PYTHON: 3
         BUILD_PYTHONLESS: 1
+        CUDA_VERSION: 90
+      # PY3.5_80:
+      #   DESIRED_PYTHON: 3.5
+      #   CUDA_VERSION: 80
+      # PY3.6_80:
+      #   DESIRED_PYTHON: 3.6
+      #   CUDA_VERSION: 80
+      # PY3.7_80:
+      #   DESIRED_PYTHON: 3.7
+      #   CUDA_VERSION: 80
+      # LIBTORCH_80:
+      #   DESIRED_PYTHON: 3
+      #   BUILD_PYTHONLESS: 1
+      #   CUDA_VERSION: 80
+      # PY3.5_92:
+      #   DESIRED_PYTHON: 3.5
+      #   CUDA_VERSION: 92
+      # PY3.6_92:
+      #   DESIRED_PYTHON: 3.6
+      #   CUDA_VERSION: 92
+      # PY3.7_92:
+      #   DESIRED_PYTHON: 3.7
+      #   CUDA_VERSION: 92
+      # LIBTORCH_92:
+      #   DESIRED_PYTHON: 3
+      #   BUILD_PYTHONLESS: 1
+      #   CUDA_VERSION: 92
 
   pool:
     name: 'pytorch'
@@ -235,6 +290,13 @@ jobs:
 
       modifyEnvironment: false
 
+  - task: BatchScript@1
+    displayName: 'Setup nightly variables'
+    inputs:
+      filename: 'windows/internal/nightly_defaults.bat'
+
+      modifyEnvironment: true
+
   - script: 'call windows/build_pytorch.bat %CUDA_VERSION% %PYTORCH_BUILD_VERSION% %PYTORCH_BUILD_NUMBER%'
     displayName: Build
 
@@ -244,33 +306,49 @@ jobs:
       testRunTitle: 'Publish test results'
     enabled: false
 
-  - task: BatchScript@1
-    displayName: 'Upload artifacts'
-    inputs:
-      filename: 'windows/internal/publish.bat'
-
-      modifyEnvironment: false
+  - template: templates/upload_to_s3.yml
+    parameters:
+      cuVer: 'cu90'
+      cudaVer: 'cuda90'
 
-- job: 'Windows_CUDA90_Conda_Build'
+- job: 'Windows_CUDA_Conda_Build'
   timeoutInMinutes: 0
   condition: and(eq(variables['build.cuda.enabled'], 'true'), eq(variables['build.conda.enabled'], 'true'))
   cancelTimeoutInMinutes: 5
   variables:
     PACKAGE: conda
-    CUDA_VERSION: 90
-    PYTORCH_REPO: pytorch
-    TORCH_CONDA_BUILD_FOLDER: pytorch-nightly
     USE_SCCACHE: $(use.sccache.cuda)
 
   strategy:
     maxParallel: 10
     matrix:
-      PY3.5:
+      PY3.5_90:
         DESIRED_PYTHON: 3.5
-      PY3.6:
+        CUDA_VERSION: 90
+      PY3.6_90:
         DESIRED_PYTHON: 3.6
-      PY3.7:
+        CUDA_VERSION: 90
+      PY3.7_90:
         DESIRED_PYTHON: 3.7
+        CUDA_VERSION: 90
+      # PY3.5_80:
+      #   DESIRED_PYTHON: 3.5
+      #   CUDA_VERSION: 80
+      # PY3.6_80:
+      #   DESIRED_PYTHON: 3.6
+      #   CUDA_VERSION: 80
+      # PY3.7_80:
+      #   DESIRED_PYTHON: 3.7
+      #   CUDA_VERSION: 80
+      # PY3.5_92:
+      #   DESIRED_PYTHON: 3.5
+      #   CUDA_VERSION: 92
+      # PY3.6_92:
+      #   DESIRED_PYTHON: 3.6
+      #   CUDA_VERSION: 92
+      # PY3.7_92:
+      #   DESIRED_PYTHON: 3.7
+      #   CUDA_VERSION: 92
 
   pool:
     name: 'pytorch'
@@ -308,7 +386,14 @@ jobs:
 
       modifyEnvironment: false
 
-  - script: 'mkdir %PYTORCH_FINAL_PACKAGE_DIR%'
+  - task: BatchScript@1
+    displayName: 'Setup nightly variables'
+    inputs:
+      filename: 'windows/internal/nightly_defaults.bat'
+
+      modifyEnvironment: true
+
+  - script: 'if not exist %PYTORCH_FINAL_PACKAGE_DIR% mkdir %PYTORCH_FINAL_PACKAGE_DIR%'
     displayName: 'Create final package directory'
     env:
       PYTORCH_FINAL_PACKAGE_DIR: '$(Build.Repository.LocalPath)\windows\output'