From 1fb27443701dac7c03711eada0bd5ac651cad909 Mon Sep 17 00:00:00 2001 From: Nikita Titov Date: Tue, 22 Dec 2020 14:20:01 +0300 Subject: [PATCH] [docs][ci] added docs about GPU support out of the box for Windows wheels and small refactoring for dual test (#3660) * added docs about GPU support out of the box for Windows and small refactoring for dual test * test * Revert "test" This reverts commit 45188103c2c9b3773bc62d97fbe865a2e5dd9120. * fix docs * fix docs * hotfix config * Apply suggestions from code review Co-authored-by: TP Boudreau Co-authored-by: TP Boudreau --- .appveyor.yml | 7 +++-- .ci/install_opencl.ps1 | 4 --- .ci/test_windows.ps1 | 26 ++++++++--------- .vsts-ci.yml | 6 ++-- docs/GPU-Targets.rst | 6 ++-- docs/GPU-Windows.rst | 8 ++++-- python-package/README.rst | 2 ++ tests/python_package_test/test_dual.py | 40 ++++++++++++-------------- 8 files changed, 50 insertions(+), 49 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 696aedccca11..70bfdf1099df 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -26,14 +26,15 @@ install: - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% - set PYTHON_VERSION=%CONFIGURATION% - set CONDA_ENV="test-env" - - ps: >- + - ps: | switch ($env:PYTHON_VERSION) { "3.6" {$env:MINICONDA = "C:\Miniconda36-x64"} "3.7" {$env:MINICONDA = "C:\Miniconda37-x64"} default {$env:MINICONDA = "C:\Miniconda37-x64"} } - $env:PATH="$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH" - - ps: $env:LGB_VER = (Get-Content $env:APPVEYOR_BUILD_FOLDER\VERSION.txt).trim() + $env:PATH = "$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH" + $env:BUILD_SOURCESDIRECTORY = "$env:APPVEYOR_BUILD_FOLDER" + $env:LGB_VER = (Get-Content $env:APPVEYOR_BUILD_FOLDER\VERSION.txt).trim() build: false diff --git a/.ci/install_opencl.ps1 b/.ci/install_opencl.ps1 index 8303629a04b0..f6d4e54ebecf 100644 --- a/.ci/install_opencl.ps1 +++ b/.ci/install_opencl.ps1 @@ -1,4 +1,3 @@ - Write-Output "Installing OpenCL CPU platform" $cache = "$env:PIPELINE_WORKSPACE\opencl_windows-amd_cpu-v3_0_130_135" @@ -6,7 +5,6 @@ $installer = "AMD-APP-SDKInstaller-v3.0.130.135-GA-windows-F-x64.exe" if ($env:OPENCL_INSTALLER_FOUND -ne 'true') { # Pipeline cache miss; download OpenCL platform installer executable into workspace cache - Write-Output "Downloading OpenCL platform installer" Invoke-WebRequest -OutFile "$installer" -Uri "https://github.com/microsoft/LightGBM/releases/download/v2.0.12/$installer" @@ -25,7 +23,6 @@ if ($env:OPENCL_INSTALLER_FOUND -ne 'true') { } # Install OpenCL platform from installer executable expected in workspace cache - Write-Output "Running OpenCL installer" Invoke-Command -ScriptBlock {Start-Process "$cache\$installer" -ArgumentList '/S /V"/quiet /norestart /passive /log opencl.log"' -Wait} @@ -42,4 +39,3 @@ if ($property -eq $null) { Write-Output "Current OpenCL drivers:" Write-Output $property } - diff --git a/.ci/test_windows.ps1 b/.ci/test_windows.ps1 index df05ac10cdf7..ea2e5f99bdfe 100644 --- a/.ci/test_windows.ps1 +++ b/.ci/test_windows.ps1 @@ -6,15 +6,9 @@ function Check-Output { } } -# Import the Chocolatey profile module so that the RefreshEnv command -# invoked below properly updates the current PowerShell session enviroment. -$module = "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" -Import-Module "$module" ; Check-Output $? - -# unify environment variables for Azure devops and AppVeyor +# unify environment variable for Azure devops and AppVeyor if (Test-Path env:APPVEYOR) { $env:APPVEYOR = "true" - $env:BUILD_SOURCESDIRECTORY = $env:APPVEYOR_BUILD_FOLDER } if ($env:TASK -eq "r-package") { @@ -28,9 +22,11 @@ conda activate conda config --set always_yes yes --set changeps1 no conda update -q -y conda conda create -q -y -n $env:CONDA_ENV python=$env:PYTHON_VERSION joblib matplotlib numpy pandas psutil pytest python-graphviz scikit-learn scipy ; Check-Output $? +if ($env:TASK -ne "bdist") { + conda activate $env:CONDA_ENV +} if ($env:TASK -eq "regular") { - conda activate $env:CONDA_ENV mkdir $env:BUILD_SOURCESDIRECTORY/build; cd $env:BUILD_SOURCESDIRECTORY/build cmake -A x64 .. ; cmake --build . --target ALL_BUILD --config Release ; Check-Output $? cd $env:BUILD_SOURCESDIRECTORY/python-package @@ -39,7 +35,6 @@ if ($env:TASK -eq "regular") { cp $env:BUILD_SOURCESDIRECTORY/Release/lightgbm.exe $env:BUILD_ARTIFACTSTAGINGDIRECTORY } elseif ($env:TASK -eq "sdist") { - conda activate $env:CONDA_ENV cd $env:BUILD_SOURCESDIRECTORY/python-package python setup.py sdist --formats gztar ; Check-Output $? cd dist; pip install @(Get-ChildItem *.gz) -v ; Check-Output $? @@ -54,7 +49,12 @@ elseif ($env:TASK -eq "sdist") { cp $env:BUILD_SOURCESDIRECTORY/build/lightgbmlib.jar $env:BUILD_ARTIFACTSTAGINGDIRECTORY/lightgbmlib_win.jar } elseif ($env:TASK -eq "bdist") { + # Import the Chocolatey profile module so that the RefreshEnv command + # invoked below properly updates the current PowerShell session enviroment. + $module = "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" + Import-Module "$module" ; Check-Output $? RefreshEnv + Write-Output "Current OpenCL drivers:" Get-ItemProperty -Path Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Khronos\OpenCL\Vendors @@ -64,7 +64,6 @@ elseif ($env:TASK -eq "bdist") { cd dist; pip install --user @(Get-ChildItem *.whl) ; Check-Output $? cp @(Get-ChildItem *.whl) $env:BUILD_ARTIFACTSTAGINGDIRECTORY } elseif (($env:APPVEYOR -eq "true") -and ($env:TASK -eq "python")) { - conda activate $env:CONDA_ENV cd $env:BUILD_SOURCESDIRECTORY\python-package if ($env:COMPILER -eq "MINGW") { python setup.py install --mingw ; Check-Output $? @@ -76,13 +75,14 @@ elseif ($env:TASK -eq "bdist") { if (($env:TASK -eq "sdist") -or (($env:APPVEYOR -eq "true") -and ($env:TASK -eq "python"))) { # cannot test C API with "sdist" task $tests = $env:BUILD_SOURCESDIRECTORY + "/tests/python_package_test" -} elseif ($env:TASK -eq "bdist") { +} else { $tests = $env:BUILD_SOURCESDIRECTORY + "/tests" +} +if ($env:TASK -eq "bdist") { # Make sure we can do both CPU and GPU; see tests/python_package_test/test_dual.py $env:LIGHTGBM_TEST_DUAL_CPU_GPU = "1" -} else { - $tests = $env:BUILD_SOURCESDIRECTORY + "/tests" } + pytest $tests ; Check-Output $? if (($env:TASK -eq "regular") -or (($env:APPVEYOR -eq "true") -and ($env:TASK -eq "python"))) { diff --git a/.vsts-ci.yml b/.vsts-ci.yml index ff563c1a241f..d6e13e482cb9 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -129,11 +129,11 @@ jobs: key: '"opencl.windows" | "amd.cpu" | "v3.0.130.135" | "1"' path: $(Pipeline.Workspace)/opencl_windows-amd_cpu-v3_0_130_135 cacheHitVar: OPENCL_INSTALLER_FOUND - condition: eq(variables.TASK, 'bdist') - displayName: 'Cache OpenCL' + condition: eq(variables['TASK'], 'bdist') + displayName: 'Cache OpenCL Installer' - script: | cmd /c "powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/install_opencl.ps1" - condition: eq(variables.TASK, 'bdist') + condition: eq(variables['TASK'], 'bdist') displayName: 'Install OpenCL' - script: | cmd /c "conda init powershell" diff --git a/docs/GPU-Targets.rst b/docs/GPU-Targets.rst index 5574e02029d1..626c3dc5d203 100644 --- a/docs/GPU-Targets.rst +++ b/docs/GPU-Targets.rst @@ -24,7 +24,7 @@ You can find below a table of correspondence: Legend: -\* AMD APP SDK is deprecated. On Windows, OpenCL is included in AMD graphics driver. On Linux, newer generation AMD cards are supported by the `ROCm`_ driver. You can download an archived copy of AMD APP SDK for Linux from `our GitHub repo`_. +\* AMD APP SDK is deprecated. On Windows, OpenCL is included in AMD graphics driver. On Linux, newer generation AMD cards are supported by the `ROCm`_ driver. You can download an archived copy of AMD APP SDK from our GitHub repo (`for Linux`_ and `for Windows`_). -------------- @@ -159,7 +159,9 @@ Known issues: .. _ROCm: https://rocmdocs.amd.com/en/latest/ -.. _our GitHub repo: https://github.com/microsoft/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.136-GA-linux64.tar.bz2 +.. _for Linux: https://github.com/microsoft/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.136-GA-linux64.tar.bz2 + +.. _for Windows: https://github.com/microsoft/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.135-GA-windows-F-x64.exe .. _NVIDIA CUDA Toolkit: https://developer.nvidia.com/cuda-downloads diff --git a/docs/GPU-Windows.rst b/docs/GPU-Windows.rst index 528fa94a4df4..65adb169fe45 100644 --- a/docs/GPU-Windows.rst +++ b/docs/GPU-Windows.rst @@ -77,9 +77,9 @@ OpenCL SDK Installation Installing the appropriate OpenCL SDK requires you to download the correct vendor source SDK. You need to know what you are going to use LightGBM! -- For running on Intel, get `Intel SDK for OpenCL`_ (NOT RECOMMENDED). +- For running on Intel, get `Intel SDK for OpenCL`_ (**NOT RECOMMENDED**). -- For running on AMD, get `AMD APP SDK`_ (you may want to replace the ``OpenCL.dll`` from GPU driver package with the one from the SDK, if the one shipped with the driver lacks some functions). +- For running on AMD, get AMD APP SDK (downloads `for Linux`_ and `for Windows`_). You may want to replace the ``OpenCL.dll`` from the GPU driver package with the one from the SDK, if the one shipped with the driver lacks some functions. - For running on NVIDIA, get `CUDA Toolkit`_. @@ -573,7 +573,9 @@ And open an issue in GitHub `here`_ with that log. .. _CUDA Toolkit: https://developer.nvidia.com/cuda-downloads -.. _AMD APP SDK: https://github.com/fireice-uk/xmr-stak/issues/1511 +.. _for Linux: https://github.com/microsoft/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.136-GA-linux64.tar.bz2 + +.. _for Windows: https://github.com/microsoft/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.135-GA-windows-F-x64.exe .. _Khronos official OpenCL headers: https://github.com/KhronosGroup/OpenCL-Headers diff --git a/python-package/README.rst b/python-package/README.rst index 42046746f224..6715bb9f460a 100644 --- a/python-package/README.rst +++ b/python-package/README.rst @@ -23,6 +23,8 @@ Install from `PyPI `_ Using ``pip`` You may need to install `wheel `_ via ``pip install wheel`` first. +Compiled library that is included in the wheel file supports both **GPU** and **CPU** versions out of the box. This feature is experimental and available only for **Windows** currently. To use **GPU** version you only need to install OpenCL Runtime libraries. For NVIDIA and AMD GPU they are included in the ordinary drivers for your graphics card, so no action is required. If you would like your AMD or Intel CPU to act like a GPU (for testing and debugging) you can install `AMD APP SDK `_. + For **Windows** users, `VC runtime `_ is needed if **Visual Studio** (2015 or newer) is not installed. For **Linux** users, **glibc** >= 2.14 is required. diff --git a/tests/python_package_test/test_dual.py b/tests/python_package_test/test_dual.py index f0c5e83618dd..64046d340f8c 100644 --- a/tests/python_package_test/test_dual.py +++ b/tests/python_package_test/test_dual.py @@ -1,3 +1,4 @@ +# coding: utf-8 """Tests for dual GPU+CPU support.""" import os @@ -5,32 +6,29 @@ import lightgbm as lgb import numpy as np -from lightgbm.basic import LightGBMError +from sklearn.metrics import log_loss -@pytest.mark.skipif( - os.environ.get("LIGHTGBM_TEST_DUAL_CPU_GPU", None) is None, - reason="Only run if appropriate env variable is set", -) -def test_cpu_works(): - """If compiled appropriately, the same installation will support both GPU and CPU.""" - data = np.random.rand(500, 10) - label = np.random.randint(2, size=500) - validation_data = train_data = lgb.Dataset(data, label=label) - - param = {"verbosity": 2, "num_leaves": 31, "objective": "binary", "device": "cpu"} - gbm = lgb.train(param, train_data, 10, valid_sets=[validation_data]) +from .utils import load_breast_cancer @pytest.mark.skipif( os.environ.get("LIGHTGBM_TEST_DUAL_CPU_GPU", None) is None, reason="Only run if appropriate env variable is set", ) -def test_gpu_works(): - """If compiled appropriately, the same installation will support both GPU and CPU.""" - data = np.random.rand(500, 10) - label = np.random.randint(2, size=500) - validation_data = train_data = lgb.Dataset(data, label=label) - - param = {"verbosity": 2, "num_leaves": 31, "objective": "binary", "device": "gpu"} - gbm = lgb.train(param, train_data, 10, valid_sets=[validation_data]) +def test_cpu_and_gpu_work(): + # If compiled appropriately, the same installation will support both GPU and CPU. + X, y = load_breast_cancer(return_X_y=True) + data = lgb.Dataset(X, y) + + params_cpu = {"verbosity": -1, "num_leaves": 31, "objective": "binary", "device": "cpu"} + cpu_bst = lgb.train(params_cpu, data, num_boost_round=10) + cpu_score = log_loss(y, cpu_bst.predict(X)) + + params_gpu = params_cpu.copy() + params_gpu["device"] = "gpu" + gpu_bst = lgb.train(params_gpu, data, num_boost_round=10) + gpu_score = log_loss(y, gpu_bst.predict(X)) + + np.testing.assert_allclose(cpu_score, gpu_score, rtol=1e-4) + assert gpu_score < 0.25