From 72d7010b5ae09818541a749bd02b455b5a08de5b Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 20 Apr 2021 08:00:42 -0500 Subject: [PATCH] [ci] parallelize R package installs in CI jobs (#4198) * [ci] parallelize R package installs in CI jobs * build 2 * build 3 * add ncpus in static_analysis --- .ci/test_r_package.sh | 2 +- .ci/test_r_package_windows.ps1 | 2 +- .github/workflows/r_package.yml | 4 ++-- .github/workflows/r_solaris.yml | 2 +- .github/workflows/r_valgrind.yml | 2 +- .github/workflows/static_analysis.yml | 2 +- R-package/README.md | 4 ++-- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.ci/test_r_package.sh b/.ci/test_r_package.sh index 93bddf3c05a0..1f5029543dc0 100755 --- a/.ci/test_r_package.sh +++ b/.ci/test_r_package.sh @@ -100,7 +100,7 @@ if [[ $OS_NAME == "macos" ]]; then packages+=", type = 'binary'" compile_from_source="never" fi -Rscript --vanilla -e "options(install.packages.compile.from.source = '${compile_from_source}'); install.packages(${packages}, repos = '${CRAN_MIRROR}', lib = '${R_LIB_PATH}', dependencies = c('Depends', 'Imports', 'LinkingTo'))" || exit -1 +Rscript --vanilla -e "options(install.packages.compile.from.source = '${compile_from_source}'); install.packages(${packages}, repos = '${CRAN_MIRROR}', lib = '${R_LIB_PATH}', dependencies = c('Depends', 'Imports', 'LinkingTo'), Ncpus = parallel::detectCores())" || exit -1 cd ${BUILD_DIRECTORY} diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index b07f15299af3..47357fa79821 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -110,7 +110,7 @@ Write-Output "Done installing Rtools" Write-Output "Installing dependencies" $packages = "c('data.table', 'jsonlite', 'Matrix', 'processx', 'R6', 'testthat'), dependencies = c('Imports', 'Depends', 'LinkingTo')" -Run-R-Code-Redirect-Stderr "options(install.packages.check.source = 'no'); install.packages($packages, repos = '$env:CRAN_MIRROR', type = 'binary', lib = '$env:R_LIB_PATH')" ; Check-Output $? +Run-R-Code-Redirect-Stderr "options(install.packages.check.source = 'no'); install.packages($packages, repos = '$env:CRAN_MIRROR', type = 'binary', lib = '$env:R_LIB_PATH', Ncpus = parallel::detectCores())" ; Check-Output $? # MiKTeX and pandoc can be skipped on non-MinGW builds, since we don't # build the package documentation for those. diff --git a/.github/workflows/r_package.yml b/.github/workflows/r_package.yml index 146f03fb90f9..3e0acd8631c2 100644 --- a/.github/workflows/r_package.yml +++ b/.github/workflows/r_package.yml @@ -183,7 +183,7 @@ jobs: - name: Install packages shell: bash run: | - Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'testthat'), repos = 'https://cran.r-project.org')" + Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" sh build-cran-package.sh Rdevel CMD INSTALL lightgbm_*.tar.gz || exit -1 - name: Run tests with sanitizers @@ -213,7 +213,7 @@ jobs: shell: bash run: | export PATH=/opt/R-devel/bin/:${PATH} - Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'testthat'), repos = 'https://cran.r-project.org')" + Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" sh build-cran-package.sh R CMD check --as-cran --run-donttest lightgbm_*.tar.gz || exit -1 if grep -q -E "NOTE|WARNING|ERROR" lightgbm.Rcheck/00check.log; then diff --git a/.github/workflows/r_solaris.yml b/.github/workflows/r_solaris.yml index b5d33d2ec1bd..a297338ba2f6 100644 --- a/.github/workflows/r_solaris.yml +++ b/.github/workflows/r_solaris.yml @@ -44,7 +44,7 @@ jobs: libxml2-dev \ libssl-dev log_file="$GITHUB_WORKSPACE/rhub_logs.txt" - Rscript -e "install.packages('rhub', dependencies = c('Depends', 'Imports', 'LinkingTo'), repos = 'https://cran.r-project.org')" + Rscript -e "install.packages('rhub', dependencies = c('Depends', 'Imports', 'LinkingTo'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" Rscript $GITHUB_WORKSPACE/.ci/run_rhub_solaris_checks.R $(pwd)/lightgbm_*.tar.gz $log_file || exit -1 - name: Send final status if: ${{ always() }} diff --git a/.github/workflows/r_valgrind.yml b/.github/workflows/r_valgrind.yml index e9af3ae6fdc5..678789af3bf0 100644 --- a/.github/workflows/r_valgrind.yml +++ b/.github/workflows/r_valgrind.yml @@ -38,7 +38,7 @@ jobs: - name: Install packages shell: bash run: | - RDscriptvalgrind -e "install.packages(c('R6', 'data.table', 'jsonlite', 'testthat'), repos = 'https://cran.r-project.org')" + RDscriptvalgrind -e "install.packages(c('R6', 'data.table', 'jsonlite', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" sh build-cran-package.sh RDvalgrind CMD INSTALL --preclean --install-tests lightgbm_*.tar.gz || exit -1 - name: Run tests with valgrind diff --git a/.github/workflows/static_analysis.yml b/.github/workflows/static_analysis.yml index 9f0faa12d1fd..851c3f7e56bf 100644 --- a/.github/workflows/static_analysis.yml +++ b/.github/workflows/static_analysis.yml @@ -57,7 +57,7 @@ jobs: - name: Install packages shell: bash run: | - Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'roxygen2', 'testthat'), repos = 'https://cran.r-project.org')" + Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'roxygen2', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" sh build-cran-package.sh || exit -1 R CMD INSTALL --with-keep.source lightgbm_*.tar.gz || exit -1 - name: Test documentation diff --git a/R-package/README.md b/R-package/README.md index d8ed4d938164..3e4c6f50e802 100644 --- a/R-package/README.md +++ b/R-package/README.md @@ -374,7 +374,7 @@ docker run \ -it rhub/rocker-gcc-san \ /bin/bash -Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'testthat'), repos = 'https://cran.rstudio.com')" +Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'testthat'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" sh build-cran-package.sh @@ -396,7 +396,7 @@ docker run \ -it \ wch1/r-debug -RDscriptvalgrind -e "install.packages(c('R6', 'data.table', 'jsonlite', 'testthat'), repos = 'https://cran.rstudio.com')" +RDscriptvalgrind -e "install.packages(c('R6', 'data.table', 'jsonlite', 'testthat'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" sh build-cran-package.sh