Skip to content

Commit

Permalink
[docs][ci] added docs about GPU support out of the box for Windows wh…
Browse files Browse the repository at this point in the history
…eels and small refactoring for dual test (#3660)

* added docs about GPU support out of the box for Windows and small refactoring for dual test

* test

* Revert "test"

This reverts commit 4518810.

* fix docs

* fix docs

* hotfix config

* Apply suggestions from code review

Co-authored-by: TP Boudreau <tpboudreau@gmail.com>

Co-authored-by: TP Boudreau <tpboudreau@gmail.com>
  • Loading branch information
StrikerRUS and tpboudreau authored Dec 22, 2020
1 parent e5554e1 commit 1fb2744
Show file tree
Hide file tree
Showing 8 changed files with 50 additions and 49 deletions.
7 changes: 4 additions & 3 deletions .appveyor.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,14 +26,15 @@ install:
- set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH%
- set PYTHON_VERSION=%CONFIGURATION%
- set CONDA_ENV="test-env"
- ps: >-
- ps: |
switch ($env:PYTHON_VERSION) {
"3.6" {$env:MINICONDA = "C:\Miniconda36-x64"}
"3.7" {$env:MINICONDA = "C:\Miniconda37-x64"}
default {$env:MINICONDA = "C:\Miniconda37-x64"}
}
$env:PATH="$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH"
- ps: $env:LGB_VER = (Get-Content $env:APPVEYOR_BUILD_FOLDER\VERSION.txt).trim()
$env:PATH = "$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH"
$env:BUILD_SOURCESDIRECTORY = "$env:APPVEYOR_BUILD_FOLDER"
$env:LGB_VER = (Get-Content $env:APPVEYOR_BUILD_FOLDER\VERSION.txt).trim()
build: false

Expand Down
4 changes: 0 additions & 4 deletions .ci/install_opencl.ps1
Original file line number Diff line number Diff line change
@@ -1,12 +1,10 @@

Write-Output "Installing OpenCL CPU platform"

$cache = "$env:PIPELINE_WORKSPACE\opencl_windows-amd_cpu-v3_0_130_135"
$installer = "AMD-APP-SDKInstaller-v3.0.130.135-GA-windows-F-x64.exe"

if ($env:OPENCL_INSTALLER_FOUND -ne 'true') {
# Pipeline cache miss; download OpenCL platform installer executable into workspace cache

Write-Output "Downloading OpenCL platform installer"
Invoke-WebRequest -OutFile "$installer" -Uri "https://github.com/microsoft/LightGBM/releases/download/v2.0.12/$installer"

Expand All @@ -25,7 +23,6 @@ if ($env:OPENCL_INSTALLER_FOUND -ne 'true') {
}

# Install OpenCL platform from installer executable expected in workspace cache

Write-Output "Running OpenCL installer"
Invoke-Command -ScriptBlock {Start-Process "$cache\$installer" -ArgumentList '/S /V"/quiet /norestart /passive /log opencl.log"' -Wait}

Expand All @@ -42,4 +39,3 @@ if ($property -eq $null) {
Write-Output "Current OpenCL drivers:"
Write-Output $property
}

26 changes: 13 additions & 13 deletions .ci/test_windows.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,9 @@ function Check-Output {
}
}

# Import the Chocolatey profile module so that the RefreshEnv command
# invoked below properly updates the current PowerShell session enviroment.
$module = "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
Import-Module "$module" ; Check-Output $?

# unify environment variables for Azure devops and AppVeyor
# unify environment variable for Azure devops and AppVeyor
if (Test-Path env:APPVEYOR) {
$env:APPVEYOR = "true"
$env:BUILD_SOURCESDIRECTORY = $env:APPVEYOR_BUILD_FOLDER
}

if ($env:TASK -eq "r-package") {
Expand All @@ -28,9 +22,11 @@ conda activate
conda config --set always_yes yes --set changeps1 no
conda update -q -y conda
conda create -q -y -n $env:CONDA_ENV python=$env:PYTHON_VERSION joblib matplotlib numpy pandas psutil pytest python-graphviz scikit-learn scipy ; Check-Output $?
if ($env:TASK -ne "bdist") {
conda activate $env:CONDA_ENV
}

if ($env:TASK -eq "regular") {
conda activate $env:CONDA_ENV
mkdir $env:BUILD_SOURCESDIRECTORY/build; cd $env:BUILD_SOURCESDIRECTORY/build
cmake -A x64 .. ; cmake --build . --target ALL_BUILD --config Release ; Check-Output $?
cd $env:BUILD_SOURCESDIRECTORY/python-package
Expand All @@ -39,7 +35,6 @@ if ($env:TASK -eq "regular") {
cp $env:BUILD_SOURCESDIRECTORY/Release/lightgbm.exe $env:BUILD_ARTIFACTSTAGINGDIRECTORY
}
elseif ($env:TASK -eq "sdist") {
conda activate $env:CONDA_ENV
cd $env:BUILD_SOURCESDIRECTORY/python-package
python setup.py sdist --formats gztar ; Check-Output $?
cd dist; pip install @(Get-ChildItem *.gz) -v ; Check-Output $?
Expand All @@ -54,7 +49,12 @@ elseif ($env:TASK -eq "sdist") {
cp $env:BUILD_SOURCESDIRECTORY/build/lightgbmlib.jar $env:BUILD_ARTIFACTSTAGINGDIRECTORY/lightgbmlib_win.jar
}
elseif ($env:TASK -eq "bdist") {
# Import the Chocolatey profile module so that the RefreshEnv command
# invoked below properly updates the current PowerShell session enviroment.
$module = "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
Import-Module "$module" ; Check-Output $?
RefreshEnv

Write-Output "Current OpenCL drivers:"
Get-ItemProperty -Path Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Khronos\OpenCL\Vendors

Expand All @@ -64,7 +64,6 @@ elseif ($env:TASK -eq "bdist") {
cd dist; pip install --user @(Get-ChildItem *.whl) ; Check-Output $?
cp @(Get-ChildItem *.whl) $env:BUILD_ARTIFACTSTAGINGDIRECTORY
} elseif (($env:APPVEYOR -eq "true") -and ($env:TASK -eq "python")) {
conda activate $env:CONDA_ENV
cd $env:BUILD_SOURCESDIRECTORY\python-package
if ($env:COMPILER -eq "MINGW") {
python setup.py install --mingw ; Check-Output $?
Expand All @@ -76,13 +75,14 @@ elseif ($env:TASK -eq "bdist") {
if (($env:TASK -eq "sdist") -or (($env:APPVEYOR -eq "true") -and ($env:TASK -eq "python"))) {
# cannot test C API with "sdist" task
$tests = $env:BUILD_SOURCESDIRECTORY + "/tests/python_package_test"
} elseif ($env:TASK -eq "bdist") {
} else {
$tests = $env:BUILD_SOURCESDIRECTORY + "/tests"
}
if ($env:TASK -eq "bdist") {
# Make sure we can do both CPU and GPU; see tests/python_package_test/test_dual.py
$env:LIGHTGBM_TEST_DUAL_CPU_GPU = "1"
} else {
$tests = $env:BUILD_SOURCESDIRECTORY + "/tests"
}

pytest $tests ; Check-Output $?

if (($env:TASK -eq "regular") -or (($env:APPVEYOR -eq "true") -and ($env:TASK -eq "python"))) {
Expand Down
6 changes: 3 additions & 3 deletions .vsts-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -129,11 +129,11 @@ jobs:
key: '"opencl.windows" | "amd.cpu" | "v3.0.130.135" | "1"'
path: $(Pipeline.Workspace)/opencl_windows-amd_cpu-v3_0_130_135
cacheHitVar: OPENCL_INSTALLER_FOUND
condition: eq(variables.TASK, 'bdist')
displayName: 'Cache OpenCL'
condition: eq(variables['TASK'], 'bdist')
displayName: 'Cache OpenCL Installer'
- script: |
cmd /c "powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/install_opencl.ps1"
condition: eq(variables.TASK, 'bdist')
condition: eq(variables['TASK'], 'bdist')
displayName: 'Install OpenCL'
- script: |
cmd /c "conda init powershell"
Expand Down
6 changes: 4 additions & 2 deletions docs/GPU-Targets.rst
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ You can find below a table of correspondence:

Legend:

\* AMD APP SDK is deprecated. On Windows, OpenCL is included in AMD graphics driver. On Linux, newer generation AMD cards are supported by the `ROCm`_ driver. You can download an archived copy of AMD APP SDK for Linux from `our GitHub repo`_.
\* AMD APP SDK is deprecated. On Windows, OpenCL is included in AMD graphics driver. On Linux, newer generation AMD cards are supported by the `ROCm`_ driver. You can download an archived copy of AMD APP SDK from our GitHub repo (`for Linux`_ and `for Windows`_).


--------------
Expand Down Expand Up @@ -159,7 +159,9 @@ Known issues:

.. _ROCm: https://rocmdocs.amd.com/en/latest/

.. _our GitHub repo: https://github.com/microsoft/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.136-GA-linux64.tar.bz2
.. _for Linux: https://github.com/microsoft/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.136-GA-linux64.tar.bz2

.. _for Windows: https://github.com/microsoft/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.135-GA-windows-F-x64.exe

.. _NVIDIA CUDA Toolkit: https://developer.nvidia.com/cuda-downloads

Expand Down
8 changes: 5 additions & 3 deletions docs/GPU-Windows.rst
Original file line number Diff line number Diff line change
Expand Up @@ -77,9 +77,9 @@ OpenCL SDK Installation
Installing the appropriate OpenCL SDK requires you to download the correct vendor source SDK.
You need to know what you are going to use LightGBM!

- For running on Intel, get `Intel SDK for OpenCL`_ (NOT RECOMMENDED).
- For running on Intel, get `Intel SDK for OpenCL`_ (**NOT RECOMMENDED**).

- For running on AMD, get `AMD APP SDK`_ (you may want to replace the ``OpenCL.dll`` from GPU driver package with the one from the SDK, if the one shipped with the driver lacks some functions).
- For running on AMD, get AMD APP SDK (downloads `for Linux`_ and `for Windows`_). You may want to replace the ``OpenCL.dll`` from the GPU driver package with the one from the SDK, if the one shipped with the driver lacks some functions.

- For running on NVIDIA, get `CUDA Toolkit`_.

Expand Down Expand Up @@ -573,7 +573,9 @@ And open an issue in GitHub `here`_ with that log.

.. _CUDA Toolkit: https://developer.nvidia.com/cuda-downloads

.. _AMD APP SDK: https://github.com/fireice-uk/xmr-stak/issues/1511
.. _for Linux: https://github.com/microsoft/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.136-GA-linux64.tar.bz2

.. _for Windows: https://github.com/microsoft/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.135-GA-windows-F-x64.exe

.. _Khronos official OpenCL headers: https://github.com/KhronosGroup/OpenCL-Headers

Expand Down
2 changes: 2 additions & 0 deletions python-package/README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ Install from `PyPI <https://pypi.org/project/lightgbm>`_ Using ``pip``
You may need to install `wheel <https://pythonwheels.com>`_ via ``pip install wheel`` first.

Compiled library that is included in the wheel file supports both **GPU** and **CPU** versions out of the box. This feature is experimental and available only for **Windows** currently. To use **GPU** version you only need to install OpenCL Runtime libraries. For NVIDIA and AMD GPU they are included in the ordinary drivers for your graphics card, so no action is required. If you would like your AMD or Intel CPU to act like a GPU (for testing and debugging) you can install `AMD APP SDK <https://github.com/microsoft/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.135-GA-windows-F-x64.exe>`_.

For **Windows** users, `VC runtime <https://support.microsoft.com/en-us/help/2977003/the-latest-supported-visual-c-downloads>`_ is needed if **Visual Studio** (2015 or newer) is not installed.

For **Linux** users, **glibc** >= 2.14 is required.
Expand Down
40 changes: 19 additions & 21 deletions tests/python_package_test/test_dual.py
Original file line number Diff line number Diff line change
@@ -1,36 +1,34 @@
# coding: utf-8
"""Tests for dual GPU+CPU support."""

import os
import pytest

import lightgbm as lgb
import numpy as np
from lightgbm.basic import LightGBMError

from sklearn.metrics import log_loss

@pytest.mark.skipif(
os.environ.get("LIGHTGBM_TEST_DUAL_CPU_GPU", None) is None,
reason="Only run if appropriate env variable is set",
)
def test_cpu_works():
"""If compiled appropriately, the same installation will support both GPU and CPU."""
data = np.random.rand(500, 10)
label = np.random.randint(2, size=500)
validation_data = train_data = lgb.Dataset(data, label=label)

param = {"verbosity": 2, "num_leaves": 31, "objective": "binary", "device": "cpu"}
gbm = lgb.train(param, train_data, 10, valid_sets=[validation_data])
from .utils import load_breast_cancer


@pytest.mark.skipif(
os.environ.get("LIGHTGBM_TEST_DUAL_CPU_GPU", None) is None,
reason="Only run if appropriate env variable is set",
)
def test_gpu_works():
"""If compiled appropriately, the same installation will support both GPU and CPU."""
data = np.random.rand(500, 10)
label = np.random.randint(2, size=500)
validation_data = train_data = lgb.Dataset(data, label=label)

param = {"verbosity": 2, "num_leaves": 31, "objective": "binary", "device": "gpu"}
gbm = lgb.train(param, train_data, 10, valid_sets=[validation_data])
def test_cpu_and_gpu_work():
# If compiled appropriately, the same installation will support both GPU and CPU.
X, y = load_breast_cancer(return_X_y=True)
data = lgb.Dataset(X, y)

params_cpu = {"verbosity": -1, "num_leaves": 31, "objective": "binary", "device": "cpu"}
cpu_bst = lgb.train(params_cpu, data, num_boost_round=10)
cpu_score = log_loss(y, cpu_bst.predict(X))

params_gpu = params_cpu.copy()
params_gpu["device"] = "gpu"
gpu_bst = lgb.train(params_gpu, data, num_boost_round=10)
gpu_score = log_loss(y, gpu_bst.predict(X))

np.testing.assert_allclose(cpu_score, gpu_score, rtol=1e-4)
assert gpu_score < 0.25

0 comments on commit 1fb2744

Please sign in to comment.