Merge pull request #217 from pjh5/master

Adding AWS S3Upload Task for wheel Windows jobs

.gitignore

@@ -68,3 +68,5 @@ conda/torchvision-src
 wheel/pytorch-src
 wheel/torchvision-src
 torch_nightly.html
+
+node_modules

windows/azure-pipelines.yml

@@ -67,12 +67,9 @@ jobs:
       testRunTitle: 'Publish test results'
     enabled: false
 
-  - task: BatchScript@1
-    displayName: 'Upload artifacts'
-    inputs:
-      filename: 'windows/internal/publish.bat'
-
-      modifyEnvironment: false
+  - template: templates/upload_to_s3.yml
+    parameters:
+      cuVer: 'cpu'
 
 - job: 'Windows_CPU_Conda_Build'
   timeoutInMinutes: 0
@@ -244,12 +241,10 @@ jobs:
       testRunTitle: 'Publish test results'
     enabled: false
 
-  - task: BatchScript@1
-    displayName: 'Upload artifacts'
-    inputs:
-      filename: 'windows/internal/publish.bat'
-
-      modifyEnvironment: false
+  - template: templates/upload_to_s3.yml
+    parameters:
+      cuVer: 'cu90'
+      cudaVer: 'cuda90'
 
 - job: 'Windows_CUDA90_Conda_Build'
   timeoutInMinutes: 0

windows/build_pytorch.bat

@@ -1,5 +1,8 @@
 @echo off
 
+:: This script parses args, installs required libraries (miniconda, MKL,
+:: Magma), and then delegates to cpu.bat, cuda80.bat, etc.
+
 IF NOT "%CUDA_VERSION%" == "" IF NOT "%PYTORCH_BUILD_VERSION%" == "" if NOT "%PYTORCH_BUILD_NUMBER%" == "" goto env_end
 if "%~1"=="" goto arg_error
 if "%~2"=="" goto arg_error
@@ -47,6 +50,7 @@ IF ERRORLEVEL 1 exit /b 1
 set "ORIG_PATH=%PATH%"
 set "PATH=%CONDA_HOME%;%CONDA_HOME%\scripts;%CONDA_HOME%\Library\bin;%PATH%"
 
+:: Create a new conda environment
 setlocal EnableDelayedExpansion
 FOR %%v IN (%DESIRED_PYTHON%) DO (
     set PYTHON_VERSION_STR=%%v
@@ -64,8 +68,8 @@ curl https://s3.amazonaws.com/ossci-windows/mkl_2018.2.185.7z -k -O
 set CMAKE_INCLUDE_PATH=%cd%\\mkl\\include
 set LIB=%cd%\\mkl\\lib;%LIB%
 
+:: Download MAGMA Files on CUDA builds
 if NOT "%CUDA_VERSION%" == "cpu" (
-    :: Download MAGMA Files
     rmdir /s /q magma_%CUDA_PREFIX%_release
     del magma_%CUDA_PREFIX%_release.7z
     curl -k https://s3.amazonaws.com/ossci-windows/magma_%CUDA_PREFIX%_release_mkl_2018.2.185.7z -o magma_%CUDA_PREFIX%_release.7z

windows/internal/setup.bat

@@ -54,6 +54,7 @@ copy /Y libtorch-%VARIANT%-%PYTORCH_BUILD_VERSION%.zip ..\output\%CUDA_PREFIX%\
 goto build_end
 
 :pytorch
+:: This stores in e.g. D:/_work/1/s/windows/output/cpu
 pip wheel -e . --wheel-dir ../output/%CUDA_PREFIX%
 
 :build_end

windows/templates/upload_to_s3.yml

@@ -0,0 +1,15 @@
+parameters:
+  cuVer: ''
+  cudaVer: ''
+
+steps:
+  - task: AmazonWebServices.aws-vsts-tools.S3Upload.S3Upload@1
+    displayName: 'Upload ${{ parameters.cuVer }} wheel to S3'
+    inputs:
+      awsCredentials: 'Pytorch S3 bucket'
+      bucketName: 'pytorch'
+      sourceFolder: 'windows/output/${{ parameters.cudaVer }}'
+      globExpressions: '*.whl'
+      targetFolder: 'whl/nightly/${{ parameters.cuVer }}/'
+      fileAcl: 'public read'
+      flattenFolders: 'true'