diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index 3f65399d3af..846b90c78e5 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -16,28 +16,26 @@ echo "STREAM_IDENTIFY_LIB=${STREAM_IDENTIFY_LIB_MODE_CUDF}" # Run libcudf and libcudf_kafka gtests from libcudf-tests package rapids-logger "Run gtests" -# TODO: exit code handling is too verbose. Find a cleaner solution. - -for gt in "$CONDA_PREFIX"/bin/gtests/{libcudf,libcudf_kafka}/* ; do - test_name=$(basename ${gt}) - echo "Running gtest $test_name" - - # TODO: This strategy for using the stream lib will need to change when we - # switch to invoking ctest. For one, we will want to set the test - # properties to use the lib (which means that the decision will be made at - # CMake-configure time instead of runtime). We may also need to leverage - # something like gtest_discover_tests to be able to filter on the - # underlying test names. - if [[ ${test_name} == "SPAN_TEST" ]]; then - # This one test is specifically designed to test using a thrust device - # vector, so we expect and allow it to include default stream usage. - gtest_filter="SpanTest.CanConstructFromDeviceContainers" - GTEST_CUDF_STREAM_MODE="new_cudf_default" LD_PRELOAD=${STREAM_IDENTIFY_LIB_MODE_CUDF} ${gt} --gtest_output=xml:${RAPIDS_TESTS_DIR} --gtest_filter="-${gtest_filter}" && \ - ${gt} --gtest_output=xml:${RAPIDS_TESTS_DIR} --gtest_filter="${gtest_filter}" - else - GTEST_CUDF_STREAM_MODE="new_cudf_default" LD_PRELOAD=${STREAM_IDENTIFY_LIB_MODE_CUDF} ${gt} --gtest_output=xml:${RAPIDS_TESTS_DIR} - fi -done +cd $CONDA_PREFIX/bin/gtests/libcudf/ +export GTEST_CUDF_STREAM_MODE="new_cudf_default" +export GTEST_OUTPUT=xml:${RAPIDS_TESTS_DIR}/ +export LD_PRELOAD=${STREAM_IDENTIFY_LIB_MODE_CUDF} + +ctest -E SPAN_TEST -j20 --output-on-failure + +# This one test is specifically designed to test using a thrust device vector, +# so we expect and allow it to include default stream usage. +_allowlist_filter="SpanTest.CanConstructFromDeviceContainers" +GTEST_FILTER="-${_allowlist_filter}" ctest -R SPAN_TEST -VV +LD_PRELOAD= GTEST_CUDF_STREAM_MODE=default GTEST_FILTER="${_allowlist_filter}" ctest -R SPAN_TEST -VV + +SUITEERROR=$? + +if (( ${SUITEERROR} == 0 )); then + cd $CONDA_PREFIX/bin/gtests/libcudf_kafka/ + ctest -j20 --output-on-failure + SUITEERROR=$? +fi rapids-logger "Test script exiting with value: $EXITCODE" exit ${EXITCODE} diff --git a/cpp/cmake/config.json b/cpp/cmake/config.json index f7d7b001856..a65afe9e58d 100644 --- a/cpp/cmake/config.json +++ b/cpp/cmake/config.json @@ -13,7 +13,11 @@ } }, "ConfigureTest": { - "flags": ["TEST_NAME", "TEST_SRC"] + "flags": ["TEST_NAME", "TEST_SRC"], + "kwargs": { + "GPUS": 1, + "PERCENT": 1 + } }, "ConfigureBench": { "flags": ["BENCH_NAME", "BENCH_SRC"] diff --git a/cpp/include/cudf_test/base_fixture.hpp b/cpp/include/cudf_test/base_fixture.hpp index 6bdfc7bfe98..1477314c592 100644 --- a/cpp/include/cudf_test/base_fixture.hpp +++ b/cpp/include/cudf_test/base_fixture.hpp @@ -244,7 +244,10 @@ inline auto make_managed() { return std::make_shared(make_cuda()); + auto const [free, total] = rmm::detail::available_device_memory(); + auto min_alloc = + rmm::detail::align_down(std::min(free, total / 10), rmm::detail::CUDA_ALLOCATION_ALIGNMENT); + return rmm::mr::make_owning_wrapper(make_cuda(), min_alloc); } inline auto make_arena() diff --git a/cpp/libcudf_kafka/tests/CMakeLists.txt b/cpp/libcudf_kafka/tests/CMakeLists.txt index afa10f02c16..68a5327b455 100644 --- a/cpp/libcudf_kafka/tests/CMakeLists.txt +++ b/cpp/libcudf_kafka/tests/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2018-2022, NVIDIA CORPORATION. +# Copyright (c) 2018-2023, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -13,7 +13,12 @@ # ============================================================================= # ################################################################################################## -# * compiler function ----------------------------------------------------------------------------- +# enable testing ################################################################################ +# ################################################################################################## +enable_testing() + +include(rapids-test) +rapids_test_init() # This function takes in a test name and test source and handles setting all of the associated # properties and linking to build the test @@ -27,12 +32,12 @@ function(ConfigureTest test_name) target_link_libraries( ${test_name} PRIVATE GTest::gmock GTest::gmock_main GTest::gtest_main cudf_kafka ) - add_test(NAME ${test_name} COMMAND ${test_name}) - install( - TARGETS ${test_name} - COMPONENT testing - DESTINATION bin/gtests/libcudf_kafka - EXCLUDE_FROM_ALL + rapids_test_add( + NAME ${test_name} + COMMAND ${test_name} + GPUS 1 + PERCENT 25 + INSTALL_COMPONENT_SET testing ) endfunction() @@ -40,3 +45,5 @@ endfunction() # * Kafka host tests # ---------------------------------------------------------------------------------- ConfigureTest(KAFKA_HOST_TEST kafka_consumer_tests.cpp) + +rapids_test_install_relocatable(INSTALL_COMPONENT_SET testing DESTINATION bin/gtests/libcudf_kafka) diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 0d58b19de6a..bd4077aff4e 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -13,12 +13,32 @@ # ============================================================================= # ################################################################################################## -# * compiler function ----------------------------------------------------------------------------- +# enable testing ################################################################################ +# ################################################################################################## +enable_testing() + +include(rapids-test) +rapids_test_init() # This function takes in a test name and test source and handles setting all of the associated # properties and linking to build the test function(ConfigureTest CMAKE_TEST_NAME) - add_executable(${CMAKE_TEST_NAME} ${ARGN}) + set(options) + set(one_value GPUS PERCENT) + set(multi_value) + cmake_parse_arguments(_CUDF_TEST "${options}" "${one_value}" "${multi_value}" ${ARGN}) + if(NOT DEFINED _CUDF_TEST_GPUS AND NOT DEFINED _CUDF_TEST_PERCENT) + set(_CUDF_TEST_GPUS 1) + set(_CUDF_TEST_PERCENT 15) + endif() + if(NOT DEFINED _CUDF_TEST_GPUS) + set(_CUDF_TEST_GPUS 1) + endif() + if(NOT DEFINED _CUDF_TEST_PERCENT) + set(_CUDF_TEST_PERCENT 100) + endif() + + add_executable(${CMAKE_TEST_NAME} ${_CUDF_TEST_UNPARSED_ARGUMENTS}) set_target_properties( ${CMAKE_TEST_NAME} PROPERTIES RUNTIME_OUTPUT_DIRECTORY "$" @@ -35,12 +55,12 @@ function(ConfigureTest CMAKE_TEST_NAME) ${CMAKE_TEST_NAME} PRIVATE cudftestutil GTest::gmock_main GTest::gtest_main $ ) - add_test(NAME ${CMAKE_TEST_NAME} COMMAND ${CMAKE_TEST_NAME}) - install( - TARGETS ${CMAKE_TEST_NAME} - COMPONENT testing - DESTINATION bin/gtests/libcudf - EXCLUDE_FROM_ALL + rapids_test_add( + NAME ${CMAKE_TEST_NAME} + COMMAND ${CMAKE_TEST_NAME} + GPUS ${_CUDF_TEST_GPUS} + PERCENT ${_CUDF_TEST_PERCENT} + INSTALL_COMPONENT_SET testing ) endfunction() @@ -112,6 +132,8 @@ ConfigureTest( groupby/sum_tests.cpp groupby/tdigest_tests.cu groupby/var_tests.cpp + GPUS 1 + PERCENT 100 ) # ################################################################################################## @@ -138,6 +160,8 @@ ConfigureTest(HASHING_TEST hashing/hash_test.cpp) ConfigureTest( PARTITIONING_TEST partitioning/hash_partition_test.cpp partitioning/round_robin_test.cpp partitioning/partition_test.cpp + GPUS 1 + PERCENT 70 ) # ################################################################################################## @@ -149,6 +173,8 @@ ConfigureTest(HASH_MAP_TEST hash_map/map_test.cu) ConfigureTest( QUANTILES_TEST quantiles/percentile_approx_test.cpp quantiles/quantile_test.cpp quantiles/quantiles_test.cpp + GPUS 1 + PERCENT 70 ) # ################################################################################################## @@ -162,6 +188,8 @@ ConfigureTest( reductions/segmented_reduction_tests.cpp reductions/list_rank_test.cpp reductions/tdigest_tests.cu + GPUS 1 + PERCENT 70 ) # ################################################################################################## @@ -221,17 +249,41 @@ ConfigureTest( # * io tests -------------------------------------------------------------------------------------- ConfigureTest(DECOMPRESSION_TEST io/comp/decomp_test.cpp) -ConfigureTest(CSV_TEST io/csv_test.cpp) -ConfigureTest(FILE_IO_TEST io/file_io_test.cpp) -ConfigureTest(ORC_TEST io/orc_test.cpp) -ConfigureTest(PARQUET_TEST io/parquet_test.cpp io/parquet_chunked_reader_test.cpp) -ConfigureTest(JSON_TEST io/json_test.cpp io/json_chunked_reader.cpp) +ConfigureTest( + CSV_TEST io/csv_test.cpp + GPUS 1 + PERCENT 30 +) +ConfigureTest( + FILE_IO_TEST io/file_io_test.cpp + GPUS 1 + PERCENT 30 +) +ConfigureTest( + ORC_TEST io/orc_test.cpp + GPUS 1 + PERCENT 30 +) +ConfigureTest( + PARQUET_TEST io/parquet_test.cpp io/parquet_chunked_reader_test.cpp + GPUS 1 + PERCENT 30 +) +ConfigureTest( + JSON_TEST io/json_test.cpp io/json_chunked_reader.cpp + GPUS 1 + PERCENT 30 +) ConfigureTest(JSON_WRITER_TEST io/json_writer.cpp) ConfigureTest(JSON_TYPE_CAST_TEST io/json_type_cast_test.cu) ConfigureTest(NESTED_JSON_TEST io/nested_json_test.cpp io/json_tree.cpp) ConfigureTest(ARROW_IO_SOURCE_TEST io/arrow_io_source_test.cpp) ConfigureTest(MULTIBYTE_SPLIT_TEST io/text/multibyte_split_test.cpp) -ConfigureTest(DATA_CHUNK_SOURCE_TEST io/text/data_chunk_source_test.cpp) +ConfigureTest( + DATA_CHUNK_SOURCE_TEST io/text/data_chunk_source_test.cpp + GPUS 1 + PERCENT 30 +) target_link_libraries(DATA_CHUNK_SOURCE_TEST PRIVATE ZLIB::ZLIB) ConfigureTest(LOGICAL_STACK_TEST io/fst/logical_stack_test.cu) ConfigureTest(FST_TEST io/fst/fst_test.cu) @@ -245,6 +297,8 @@ endif() ConfigureTest( SORT_TEST sort/segmented_sort_tests.cpp sort/sort_test.cpp sort/stable_sort_tests.cpp sort/rank_test.cpp + GPUS 1 + PERCENT 70 ) # ################################################################################################## @@ -275,6 +329,8 @@ ConfigureTest( copying/split_tests.cpp copying/utility_tests.cpp copying/reverse_tests.cpp + GPUS 1 + PERCENT 70 ) # ################################################################################################## @@ -318,7 +374,11 @@ ConfigureTest(DEVICE_ATOMICS_TEST device_atomics/device_atomics_test.cu) # ################################################################################################## # * transpose tests ------------------------------------------------------------------------------- -ConfigureTest(TRANSPOSE_TEST transpose/transpose_test.cpp) +ConfigureTest( + TRANSPOSE_TEST transpose/transpose_test.cpp + GPUS 1 + PERCENT 70 +) # ################################################################################################## # * table tests ----------------------------------------------------------------------------------- @@ -359,6 +419,8 @@ ConfigureTest( rolling/range_rolling_window_test.cpp rolling/range_window_bounds_test.cpp rolling/rolling_test.cpp + GPUS 1 + PERCENT 70 ) # ################################################################################################## @@ -504,6 +566,8 @@ ConfigureTest( lists/sort_lists_tests.cpp lists/stream_compaction/apply_boolean_mask_tests.cpp lists/stream_compaction/distinct_tests.cpp + GPUS 1 + PERCENT 70 ) # ################################################################################################## @@ -524,7 +588,6 @@ set_tests_properties( ) # ################################################################################################## -# enable testing ################################################################################ +# Install tests #################################################################################### # ################################################################################################## - -enable_testing() +rapids_test_install_relocatable(INSTALL_COMPONENT_SET testing DESTINATION bin/gtests/libcudf) diff --git a/dependencies.yaml b/dependencies.yaml index 12a6d1b370e..5f72b8b6dea 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -410,6 +410,7 @@ dependencies: common: - output_types: conda packages: + - *cmake_ver - *gtest - *gmock specific: