[R-package] parallelize compilation in CMake-based builds (#4525)

* [R-package] parallelize compilation in CMake-based builds

* Apply suggestions from code review

Co-authored-by: Nikita Titov <nekit94-08@mail.ru>

* working on adding -j

* pass -j through to install.libs.R

* add docs on -j

* use -j4

* Update R-package/README.md

Co-authored-by: Nikita Titov <nekit94-08@mail.ru>

Co-authored-by: Nikita Titov <nekit94-08@mail.ru>

.ci/test_r_package.sh

@@ -112,7 +112,7 @@ cd ${BUILD_DIRECTORY}
 PKG_TARBALL="lightgbm_*.tar.gz"
 LOG_FILE_NAME="lightgbm.Rcheck/00check.log"
 if [[ $R_BUILD_TYPE == "cmake" ]]; then
-    Rscript build_r.R --skip-install || exit -1
+    Rscript build_r.R -j4 --skip-install || exit -1
 elif [[ $R_BUILD_TYPE == "cran" ]]; then
 
     # on Linux, we recreate configure in CI to test if

.ci/test_r_package_windows.ps1

@@ -153,10 +153,10 @@ if ($env:COMPILER -ne "MSVC") {
   if ($env:R_BUILD_TYPE -eq "cmake") {
     if ($env:TOOLCHAIN -eq "MINGW") {
       Write-Output "Telling R to use MinGW"
-      $env:BUILD_R_FLAGS = "c('--skip-install', '--use-mingw')"
+      $env:BUILD_R_FLAGS = "c('--skip-install', '--use-mingw', '-j4')"
     } elseif ($env:TOOLCHAIN -eq "MSYS") {
       Write-Output "Telling R to use MSYS"
-      $env:BUILD_R_FLAGS = "c('--skip-install', '--use-msys2')"
+      $env:BUILD_R_FLAGS = "c('--skip-install', '--use-msys2', '-j4')"
     } elseif ($env:TOOLCHAIN -eq "MSVC") {
       $env:BUILD_R_FLAGS = "'--skip-install'"
     } else {

R-package/README.md

@@ -152,6 +152,9 @@ Rscript build_r.R
 
 The `build_r.R` script builds the package in a temporary directory called `lightgbm_r`. It will destroy and recreate that directory each time you run the script. That script supports the following command-line options:
 
+- `-j[jobs]`: number of threads to use when compiling LightGBM. E.g., `-j4` will try to compile 4 objects at a time.
+    - by default, this script uses single-thread compilation
+    - for best results, set `-j` to the number of physical CPUs
 - `--skip-install`: Build the package tarball, but do not install it.
 - `--use-gpu`: Build a GPU-enabled version of the library.
 - `--use-mingw`: Force the use of MinGW toolchain, regardless of R version.

R-package/src/install.libs.R

@@ -1,5 +1,6 @@
 # User options
 use_gpu <- FALSE
+make_args_from_build_script <- character(0L)
 
 # For Windows, the package will be built with Visual Studio
 # unless you set one of these to TRUE
@@ -132,7 +133,7 @@ if (WINDOWS && use_visual_studio) {
 # Prepare installation steps
 cmake_args <- NULL
 build_cmd <- "make"
-build_args <- "_lightgbm"
+build_args <- c("_lightgbm", make_args_from_build_script)
 lib_folder <- file.path(source_dir, fsep = "/")
 
 # add in command-line arguments
@@ -194,7 +195,7 @@ if (WINDOWS) {
     cmake_args <- c(cmake_args, "-G", shQuote(windows_makefile_generator))
     .run_shell_command("cmake", c(cmake_args, ".."), strict = FALSE)
     build_cmd <- windows_build_tool
-    build_args <- "_lightgbm"
+    build_args <- c("_lightgbm", make_args_from_build_script)
   } else {
     visual_studio_succeeded <- .generate_vs_makefiles(cmake_args)
     if (!isTRUE(visual_studio_succeeded)) {
@@ -203,7 +204,7 @@ if (WINDOWS) {
       cmake_args <- c(cmake_args, "-G", shQuote(windows_makefile_generator))
       .run_shell_command("cmake", c(cmake_args, ".."), strict = FALSE)
       build_cmd <- windows_build_tool
-      build_args <- "_lightgbm"
+      build_args <- c("_lightgbm", make_args_from_build_script)
     } else {
       build_cmd <- "cmake"
       build_args <- c("--build", ".", "--target", "_lightgbm", "--config", "Release")

build_r.R

@@ -21,9 +21,12 @@ TEMP_SOURCE_DIR <- file.path(TEMP_R_DIR, "src")
   out_list <- list(
     "flags" = character(0L)
     , "keyword_args" = character(0L)
+    , "make_args" = character(0L)
   )
   for (arg in args) {
-    if (any(grepl("=", arg))) {
+    if (any(grepl("^\\-j[0-9]+", arg))) {
+        out_list[["make_args"]] <- arg
+    } else if (any(grepl("=", arg))) {
       split_arg <- strsplit(arg, "=")[[1L]]
       arg_name <- split_arg[[1L]]
       arg_value <- split_arg[[2L]]
@@ -109,6 +112,20 @@ if (length(keyword_args) > 0L) {
   )
 }
 
+# if provided, set '-j' in 'make' commands in install.libs.R
+if (length(parsed_args[["make_args"]]) > 0L) {
+  install_libs_content <- gsub(
+    pattern = "make_args_from_build_script <- character(0L)"
+    , replacement = paste0(
+      "make_args_from_build_script <- c(\""
+      , paste0(parsed_args[["make_args"]], collapse = "\", \"")
+      , "\")"
+    )
+    , x = install_libs_content
+    , fixed = TRUE
+  )
+}
+
 # R returns FALSE (not a non-zero exit code) if a file copy operation
 # breaks. Let's fix that
 .handle_result <- function(res) {