Use rapids-cmake parallel testing feature (#12451)

Converts libcudf over to use rapids-cmake new GPU aware parallel testing feature, which allows tests to run across all the GPUs on a machine without oversubscription. This will allow developers to run `ctest -j<N>` and ctest will figure out given the current machine how many tests it can run in parallel given the current GPU set ( currently 2 tests per GPU ). Authors: - Robert Maynard (https://github.com/robertmaynard) - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) - AJ Schmidt (https://github.com/ajschmidt8) URL: #12451
rapidsai · Mar 22, 2023 · 253f2ab · 253f2ab
1 parent 00c6000
commit 253f2ab
Show file tree

Hide file tree

Showing 6 changed files with 126 additions and 50 deletions.
diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh
@@ -16,28 +16,26 @@ echo "STREAM_IDENTIFY_LIB=${STREAM_IDENTIFY_LIB_MODE_CUDF}"
 # Run libcudf and libcudf_kafka gtests from libcudf-tests package
 rapids-logger "Run gtests"
 
-# TODO: exit code handling is too verbose. Find a cleaner solution.
-
-for gt in "$CONDA_PREFIX"/bin/gtests/{libcudf,libcudf_kafka}/* ; do
-    test_name=$(basename ${gt})
-    echo "Running gtest $test_name"
-
-    # TODO: This strategy for using the stream lib will need to change when we
-    # switch to invoking ctest. For one, we will want to set the test
-    # properties to use the lib (which means that the decision will be made at
-    # CMake-configure time instead of runtime). We may also need to leverage
-    # something like gtest_discover_tests to be able to filter on the
-    # underlying test names.
-    if [[ ${test_name} == "SPAN_TEST" ]]; then
-        # This one test is specifically designed to test using a thrust device
-        # vector, so we expect and allow it to include default stream usage.
-        gtest_filter="SpanTest.CanConstructFromDeviceContainers"
-        GTEST_CUDF_STREAM_MODE="new_cudf_default" LD_PRELOAD=${STREAM_IDENTIFY_LIB_MODE_CUDF} ${gt} --gtest_output=xml:${RAPIDS_TESTS_DIR} --gtest_filter="-${gtest_filter}" && \
-            ${gt} --gtest_output=xml:${RAPIDS_TESTS_DIR} --gtest_filter="${gtest_filter}"
-    else
-        GTEST_CUDF_STREAM_MODE="new_cudf_default" LD_PRELOAD=${STREAM_IDENTIFY_LIB_MODE_CUDF} ${gt} --gtest_output=xml:${RAPIDS_TESTS_DIR}
-    fi
-done
+cd $CONDA_PREFIX/bin/gtests/libcudf/
+export GTEST_CUDF_STREAM_MODE="new_cudf_default"
+export GTEST_OUTPUT=xml:${RAPIDS_TESTS_DIR}/
+export LD_PRELOAD=${STREAM_IDENTIFY_LIB_MODE_CUDF}
+
+ctest -E SPAN_TEST -j20 --output-on-failure
+
+# This one test is specifically designed to test using a thrust device vector,
+# so we expect and allow it to include default stream usage.
+_allowlist_filter="SpanTest.CanConstructFromDeviceContainers"
+GTEST_FILTER="-${_allowlist_filter}" ctest -R SPAN_TEST -VV
+LD_PRELOAD= GTEST_CUDF_STREAM_MODE=default GTEST_FILTER="${_allowlist_filter}" ctest -R SPAN_TEST -VV
+
+SUITEERROR=$?
+
+if (( ${SUITEERROR} == 0 )); then
+    cd $CONDA_PREFIX/bin/gtests/libcudf_kafka/
+    ctest -j20 --output-on-failure
+    SUITEERROR=$?
+fi
 
 rapids-logger "Test script exiting with value: $EXITCODE"
 exit ${EXITCODE}
diff --git a/cpp/cmake/config.json b/cpp/cmake/config.json
@@ -13,7 +13,11 @@
         }
       },
       "ConfigureTest": {
-        "flags": ["TEST_NAME", "TEST_SRC"]
+        "flags": ["TEST_NAME", "TEST_SRC"],
+        "kwargs": {
+          "GPUS": 1,
+          "PERCENT": 1
+        }
       },
       "ConfigureBench": {
         "flags": ["BENCH_NAME", "BENCH_SRC"]

diff --git a/cpp/include/cudf_test/base_fixture.hpp b/cpp/include/cudf_test/base_fixture.hpp
@@ -244,7 +244,10 @@ inline auto make_managed() { return std::make_shared<rmm::mr::managed_memory_res
 
 inline auto make_pool()
 {
-  return rmm::mr::make_owning_wrapper<rmm::mr::pool_memory_resource>(make_cuda());
+  auto const [free, total] = rmm::detail::available_device_memory();
+  auto min_alloc =
+    rmm::detail::align_down(std::min(free, total / 10), rmm::detail::CUDA_ALLOCATION_ALIGNMENT);
+  return rmm::mr::make_owning_wrapper<rmm::mr::pool_memory_resource>(make_cuda(), min_alloc);
 }
 
 inline auto make_arena()

diff --git a/cpp/libcudf_kafka/tests/CMakeLists.txt b/cpp/libcudf_kafka/tests/CMakeLists.txt
@@ -1,5 +1,5 @@
 # =============================================================================
-# Copyright (c) 2018-2022, NVIDIA CORPORATION.
+# Copyright (c) 2018-2023, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 # in compliance with the License. You may obtain a copy of the License at
@@ -13,7 +13,12 @@
 # =============================================================================
 
 # ##################################################################################################
-# * compiler function -----------------------------------------------------------------------------
+# enable testing ################################################################################
+# ##################################################################################################
+enable_testing()
+
+include(rapids-test)
+rapids_test_init()
 
 # This function takes in a test name and test source and handles setting all of the associated
 # properties and linking to build the test
@@ -27,16 +32,18 @@ function(ConfigureTest test_name)
   target_link_libraries(
     ${test_name} PRIVATE GTest::gmock GTest::gmock_main GTest::gtest_main cudf_kafka
   )
-  add_test(NAME ${test_name} COMMAND ${test_name})
-  install(
-    TARGETS ${test_name}
-    COMPONENT testing
-    DESTINATION bin/gtests/libcudf_kafka
-    EXCLUDE_FROM_ALL
+  rapids_test_add(
+    NAME ${test_name}
+    COMMAND ${test_name}
+    GPUS 1
+    PERCENT 25
+    INSTALL_COMPONENT_SET testing
   )
 endfunction()
 
 # ##################################################################################################
 # * Kafka host tests
 # ----------------------------------------------------------------------------------
 ConfigureTest(KAFKA_HOST_TEST kafka_consumer_tests.cpp)
+
+rapids_test_install_relocatable(INSTALL_COMPONENT_SET testing DESTINATION bin/gtests/libcudf_kafka)
diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt
@@ -13,12 +13,32 @@
 # =============================================================================
 
 # ##################################################################################################
-# * compiler function -----------------------------------------------------------------------------
+# enable testing ################################################################################
+# ##################################################################################################
+enable_testing()
+
+include(rapids-test)
+rapids_test_init()
 
 # This function takes in a test name and test source and handles setting all of the associated
 # properties and linking to build the test
 function(ConfigureTest CMAKE_TEST_NAME)
-  add_executable(${CMAKE_TEST_NAME} ${ARGN})
+  set(options)
+  set(one_value GPUS PERCENT)
+  set(multi_value)
+  cmake_parse_arguments(_CUDF_TEST "${options}" "${one_value}" "${multi_value}" ${ARGN})
+  if(NOT DEFINED _CUDF_TEST_GPUS AND NOT DEFINED _CUDF_TEST_PERCENT)
+    set(_CUDF_TEST_GPUS 1)
+    set(_CUDF_TEST_PERCENT 15)
+  endif()
+  if(NOT DEFINED _CUDF_TEST_GPUS)
+    set(_CUDF_TEST_GPUS 1)
+  endif()
+  if(NOT DEFINED _CUDF_TEST_PERCENT)
+    set(_CUDF_TEST_PERCENT 100)
+  endif()
+
+  add_executable(${CMAKE_TEST_NAME} ${_CUDF_TEST_UNPARSED_ARGUMENTS})
   set_target_properties(
     ${CMAKE_TEST_NAME}
     PROPERTIES RUNTIME_OUTPUT_DIRECTORY "$<BUILD_INTERFACE:${CUDF_BINARY_DIR}/gtests>"
@@ -35,12 +55,12 @@ function(ConfigureTest CMAKE_TEST_NAME)
     ${CMAKE_TEST_NAME} PRIVATE cudftestutil GTest::gmock_main GTest::gtest_main
                                $<TARGET_NAME_IF_EXISTS:conda_env>
   )
-  add_test(NAME ${CMAKE_TEST_NAME} COMMAND ${CMAKE_TEST_NAME})
-  install(
-    TARGETS ${CMAKE_TEST_NAME}
-    COMPONENT testing
-    DESTINATION bin/gtests/libcudf
-    EXCLUDE_FROM_ALL
+  rapids_test_add(
+    NAME ${CMAKE_TEST_NAME}
+    COMMAND ${CMAKE_TEST_NAME}
+    GPUS ${_CUDF_TEST_GPUS}
+    PERCENT ${_CUDF_TEST_PERCENT}
+    INSTALL_COMPONENT_SET testing
   )
 endfunction()
 
@@ -112,6 +132,8 @@ ConfigureTest(
   groupby/sum_tests.cpp
   groupby/tdigest_tests.cu
   groupby/var_tests.cpp
+  GPUS 1
+  PERCENT 100
 )
 
 # ##################################################################################################
@@ -138,6 +160,8 @@ ConfigureTest(HASHING_TEST hashing/hash_test.cpp)
 ConfigureTest(
   PARTITIONING_TEST partitioning/hash_partition_test.cpp partitioning/round_robin_test.cpp
   partitioning/partition_test.cpp
+  GPUS 1
+  PERCENT 70
 )
 
 # ##################################################################################################
@@ -149,6 +173,8 @@ ConfigureTest(HASH_MAP_TEST hash_map/map_test.cu)
 ConfigureTest(
   QUANTILES_TEST quantiles/percentile_approx_test.cpp quantiles/quantile_test.cpp
   quantiles/quantiles_test.cpp
+  GPUS 1
+  PERCENT 70
 )
 
 # ##################################################################################################
@@ -162,6 +188,8 @@ ConfigureTest(
   reductions/segmented_reduction_tests.cpp
   reductions/list_rank_test.cpp
   reductions/tdigest_tests.cu
+  GPUS 1
+  PERCENT 70
 )
 
 # ##################################################################################################
@@ -221,17 +249,41 @@ ConfigureTest(
 # * io tests --------------------------------------------------------------------------------------
 ConfigureTest(DECOMPRESSION_TEST io/comp/decomp_test.cpp)
 
-ConfigureTest(CSV_TEST io/csv_test.cpp)
-ConfigureTest(FILE_IO_TEST io/file_io_test.cpp)
-ConfigureTest(ORC_TEST io/orc_test.cpp)
-ConfigureTest(PARQUET_TEST io/parquet_test.cpp io/parquet_chunked_reader_test.cpp)
-ConfigureTest(JSON_TEST io/json_test.cpp io/json_chunked_reader.cpp)
+ConfigureTest(
+  CSV_TEST io/csv_test.cpp
+  GPUS 1
+  PERCENT 30
+)
+ConfigureTest(
+  FILE_IO_TEST io/file_io_test.cpp
+  GPUS 1
+  PERCENT 30
+)
+ConfigureTest(
+  ORC_TEST io/orc_test.cpp
+  GPUS 1
+  PERCENT 30
+)
+ConfigureTest(
+  PARQUET_TEST io/parquet_test.cpp io/parquet_chunked_reader_test.cpp
+  GPUS 1
+  PERCENT 30
+)
+ConfigureTest(
+  JSON_TEST io/json_test.cpp io/json_chunked_reader.cpp
+  GPUS 1
+  PERCENT 30
+)
 ConfigureTest(JSON_WRITER_TEST io/json_writer.cpp)
 ConfigureTest(JSON_TYPE_CAST_TEST io/json_type_cast_test.cu)
 ConfigureTest(NESTED_JSON_TEST io/nested_json_test.cpp io/json_tree.cpp)
 ConfigureTest(ARROW_IO_SOURCE_TEST io/arrow_io_source_test.cpp)
 ConfigureTest(MULTIBYTE_SPLIT_TEST io/text/multibyte_split_test.cpp)
-ConfigureTest(DATA_CHUNK_SOURCE_TEST io/text/data_chunk_source_test.cpp)
+ConfigureTest(
+  DATA_CHUNK_SOURCE_TEST io/text/data_chunk_source_test.cpp
+  GPUS 1
+  PERCENT 30
+)
 target_link_libraries(DATA_CHUNK_SOURCE_TEST PRIVATE ZLIB::ZLIB)
 ConfigureTest(LOGICAL_STACK_TEST io/fst/logical_stack_test.cu)
 ConfigureTest(FST_TEST io/fst/fst_test.cu)
@@ -245,6 +297,8 @@ endif()
 ConfigureTest(
   SORT_TEST sort/segmented_sort_tests.cpp sort/sort_test.cpp sort/stable_sort_tests.cpp
   sort/rank_test.cpp
+  GPUS 1
+  PERCENT 70
 )
 
 # ##################################################################################################
@@ -275,6 +329,8 @@ ConfigureTest(
   copying/split_tests.cpp
   copying/utility_tests.cpp
   copying/reverse_tests.cpp
+  GPUS 1
+  PERCENT 70
 )
 
 # ##################################################################################################
@@ -318,7 +374,11 @@ ConfigureTest(DEVICE_ATOMICS_TEST device_atomics/device_atomics_test.cu)
 
 # ##################################################################################################
 # * transpose tests -------------------------------------------------------------------------------
-ConfigureTest(TRANSPOSE_TEST transpose/transpose_test.cpp)
+ConfigureTest(
+  TRANSPOSE_TEST transpose/transpose_test.cpp
+  GPUS 1
+  PERCENT 70
+)
 
 # ##################################################################################################
 # * table tests -----------------------------------------------------------------------------------
@@ -359,6 +419,8 @@ ConfigureTest(
   rolling/range_rolling_window_test.cpp
   rolling/range_window_bounds_test.cpp
   rolling/rolling_test.cpp
+  GPUS 1
+  PERCENT 70
 )
 
 # ##################################################################################################
@@ -504,6 +566,8 @@ ConfigureTest(
   lists/sort_lists_tests.cpp
   lists/stream_compaction/apply_boolean_mask_tests.cpp
   lists/stream_compaction/distinct_tests.cpp
+  GPUS 1
+  PERCENT 70
 )
 
 # ##################################################################################################
@@ -524,7 +588,6 @@ set_tests_properties(
 )
 
 # ##################################################################################################
-# enable testing ################################################################################
+# Install tests ####################################################################################
 # ##################################################################################################
-
-enable_testing()
+rapids_test_install_relocatable(INSTALL_COMPONENT_SET testing DESTINATION bin/gtests/libcudf)
diff --git a/dependencies.yaml b/dependencies.yaml
@@ -410,6 +410,7 @@ dependencies:
     common:
       - output_types: conda
         packages:
+          - *cmake_ver
           - *gtest
           - *gmock
     specific: