Improved adapter selection in CTS testing

The CTS tests now have an additional --backend option, and the --platform option allows specifying platforms per-device. In addition, new targets have been added that test the "loader"; that is, where the adapter isn't forced.
oneapi-src · Oct 9, 2024 · dcc2b12 · dcc2b12
1 parent 0112320
commit dcc2b12
Show file tree

Hide file tree

Showing 11 changed files with 257 additions and 123 deletions.
diff --git a/.github/workflows/build-hw-reusable.yml b/.github/workflows/build-hw-reusable.yml
@@ -82,6 +82,7 @@ jobs:
         -DUR_ENABLE_TRACING=ON
         -DUR_DEVELOPER_MODE=ON
         -DUR_BUILD_TESTS=ON
+        -DUR_CONFORMANCE_TEST_LOADER=OFF
         -DUR_BUILD_ADAPTER_${{matrix.adapter.name}}=ON
         -DUR_STATIC_LOADER=${{matrix.adapter.static_Loader}}
         -DUR_STATIC_ADAPTER_${{matrix.adapter.name}}=${{matrix.adapter.static_adapter}}

diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml
@@ -92,6 +92,7 @@ jobs:
         -DUR_BUILD_TESTS=ON
         -DUR_FORMAT_CPP_STYLE=OFF
         -DUR_DPCXX=${{github.workspace}}/dpcpp_compiler/bin/clang++
+        -DUR_CONFORMANCE_TEST_LOADER=OFF
         ${{matrix.libbacktrace}}
         ${{matrix.pool_tracking}}
         ${{matrix.latency_tracking}}
@@ -259,6 +260,7 @@ jobs:
         -DUR_DEVELOPER_MODE=ON
         -DUR_BUILD_TESTS=ON
         -DUR_FORMAT_CPP_STYLE=OFF
+        -DUR_CONFORMANCE_TEST_LOADER=OFF
         ${{matrix.adapter.var}}
 
     - name: Build all

diff --git a/.github/workflows/multi_device.yml b/.github/workflows/multi_device.yml
@@ -47,6 +47,7 @@ jobs:
         -DUR_DEVELOPER_MODE=ON
         -DUR_BUILD_TESTS=ON
         -DUR_BUILD_ADAPTER_${{matrix.adapter.name}}=ON
+        -DUR_CONFORMANCE_TEST_LOADER=OFF
         -DUR_TEST_DEVICES_COUNT=2
         -DUR_DPCXX=${{github.workspace}}/dpcpp_compiler/bin/clang++
         -DUR_SYCL_LIBRARY_DIR=${{github.workspace}}/dpcpp_compiler/lib

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -68,6 +68,7 @@ set(UR_CONFORMANCE_TARGET_TRIPLES "" CACHE STRING
     "List of sycl targets to build CTS device binaries for")
 set(UR_CONFORMANCE_AMD_ARCH "" CACHE STRING "AMD device target ID to build CTS binaries for")
 option(UR_CONFORMANCE_ENABLE_MATCH_FILES "Enable CTS match files" ON)
+option(UR_CONFORMANCE_TEST_LOADER "Also test the loader in the conformance tests" ON)
 set(UR_ADAPTER_LEVEL_ZERO_SOURCE_DIR "" CACHE PATH
     "Path to external 'level_zero' adapter source dir")
 set(UR_ADAPTER_OPENCL_SOURCE_DIR "" CACHE PATH

diff --git a/README.md b/README.md
@@ -133,6 +133,7 @@ List of options provided by CMake:
 | UR_CONFORMANCE_TARGET_TRIPLES | SYCL triples to build CTS device binaries for | Comma-separated list | spir64 |
 | UR_CONFORMANCE_AMD_ARCH | AMD device target ID to build CTS binaries for | string | `""` |
 | UR_CONFORMANCE_ENABLE_MATCH_FILES | Enable CTS match files | ON/OFF | ON |
+| UR_CONFORMANCE_TEST_LOADER | Build and run "loader" tests for the CTS | ON/OFF | ON |
 | UR_BUILD_ADAPTER_L0     | Build the Level-Zero adapter            | ON/OFF     | OFF     |
 | UR_BUILD_ADAPTER_OPENCL | Build the OpenCL adapter                | ON/OFF     | OFF     |
 | UR_BUILD_ADAPTER_CUDA   | Build the CUDA adapter                  | ON/OFF     | OFF     |

diff --git a/scripts/core/INTRO.rst b/scripts/core/INTRO.rst
@@ -404,6 +404,32 @@ Specific environment variables can be set to control the behavior of unified run
 
     This environment variable is default enabled on Linux, but default disabled on Windows.
 
+CTS Environment Variables
+-------------------------
+
+The following environment variables are used by the CTS runner and can be used to specify the platform that the test
+framework should run on. This can be used during development and testing to run CTS tests in case multiple platforms
+are available. If both filters are specified, then they both must match a platform for it to be selected. If there are
+no valid platforms, then the tests will fail. Command line arguments take priority over these variables.
+
+.. envvar:: UR_CTS_ADAPTER_PLATFORM
+
+    A specifier list in the form of `[(backend):](platform name)[;[(backend)]:(platform name)]...`. If a backend
+    specific specifier is present in the list and the test is running for that backend, the device with the given name
+    is chosen. Otherwise, it must match the name of the specifier from the list with no backend. Backend names are case-
+    insensitive.
+
+    For example, if the test device has multiple platforms and you want to run tests on the "ABC Corp" backend when
+    testing OpenCL and "XYZ Org" when testing level zero, you'd use `OPENCL:ABC Corp;LEVEL_ZERO:XYZ Org`. This form is
+    useful when running the `build` target with a build with multiple backends.
+
+    For testing only one platform, the backend can be omitted. For example, just `ABC Corp` is sufficient if the tests
+    are only going to be testing OpenCL.
+
+.. envvar:: UR_CTS_BACKEND
+
+    A (case insensitive) backend to force the test to use. For example, `opencl`, `level_zero`, `hip` and so on.
+
 Service identifiers
 ---------------------
 

diff --git a/test/adapters/CMakeLists.txt b/test/adapters/CMakeLists.txt
@@ -63,7 +63,7 @@ function(add_adapter_memcheck_test name)
         add_test(NAME ${test_name}
                 COMMAND ${CMAKE_COMMAND}
                 -D TEST_FILE=valgrind
-                -D TEST_ARGS="--tool=memcheck --leak-check=full $<TARGET_FILE:${target}> --devices_count=${UR_TEST_DEVICES_COUNT} --platforms_count=${UR_TEST_DEVICES_COUNT}"
+                -D TEST_ARGS="--tool=memcheck --leak-check=full $<TARGET_FILE:${target}> --backend=${backend} --devices_count=${UR_TEST_DEVICES_COUNT} --platforms_count=${UR_TEST_DEVICES_COUNT}"
                 -D MODE=stderr
                 -D MATCH_FILE=${CMAKE_CURRENT_SOURCE_DIR}/${name}_memcheck.match
                 -P ${PROJECT_SOURCE_DIR}/cmake/match.cmake

diff --git a/test/conformance/CMakeLists.txt b/test/conformance/CMakeLists.txt
@@ -9,45 +9,48 @@ set(UR_CONFORMANCE_DEVICE_BINARIES_DIR
     "${CMAKE_CURRENT_BINARY_DIR}/device_binaries" CACHE INTERNAL
     "Internal cache variable for device binaries directory")
 
-function(add_test_adapter name adapter)
-    if(NOT "${ARGN}" STREQUAL "")
-        set(EXTRA_NAME "-${ARGN}")
-    endif()
+function(add_test_adapter name adapter backend)
     set(TEST_TARGET_NAME test-${name})
-    set(TEST_NAME ${name}-${adapter}${EXTRA_NAME})
+    set(TEST_NAME ${name}-${adapter})
 
     set(TEST_COMMAND
-        "${PROJECT_BINARY_DIR}/bin/${TEST_TARGET_NAME} --devices_count=${UR_TEST_DEVICES_COUNT} --platforms_count=${UR_TEST_PLATFORMS_COUNT}"
+        "${PROJECT_BINARY_DIR}/bin/${TEST_TARGET_NAME} --backend=${backend} --devices_count=${UR_TEST_DEVICES_COUNT} --platforms_count=${UR_TEST_PLATFORMS_COUNT}"
     )
-    set(MATCH_FILE "${CMAKE_CURRENT_SOURCE_DIR}/${name}_${adapter}${EXTRA_NAME}.match")
-
-    if(${UR_CONFORMANCE_ENABLE_MATCH_FILES} AND EXISTS ${MATCH_FILE})
-        add_test(NAME ${TEST_NAME}
-            COMMAND ${CMAKE_COMMAND}
-            -D TEST_FILE=${Python3_EXECUTABLE}
-            -D TEST_ARGS="${UR_CONFORMANCE_TEST_DIR}/cts_exe.py --test_command ${TEST_COMMAND}"
-            -D MODE=stdout
-            -D MATCH_FILE=${MATCH_FILE}
-            -P ${PROJECT_SOURCE_DIR}/cmake/match.cmake
-            DEPENDS ${TEST_TARGET_NAME}
-            WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
-        )
-    else()
-        separate_arguments(TEST_COMMAND)
-        add_test(NAME ${TEST_NAME}
-            COMMAND ${TEST_COMMAND}
-            DEPENDS ${TEST_TARGET_NAME}
-            WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
-        )
-    endif()
+    set(MATCH_FILE "${CMAKE_CURRENT_SOURCE_DIR}/${name}_${adapter}.match")
+
+    function(do_add_test tname env)
+        if(${UR_CONFORMANCE_ENABLE_MATCH_FILES} AND EXISTS ${MATCH_FILE})
+            add_test(NAME ${tname}
+                COMMAND ${CMAKE_COMMAND}
+                -D TEST_FILE=${Python3_EXECUTABLE}
+                -D TEST_ARGS="${UR_CONFORMANCE_TEST_DIR}/cts_exe.py --test_command ${TEST_COMMAND}"
+                -D MODE=stdout
+                -D MATCH_FILE=${MATCH_FILE}
+                -P ${PROJECT_SOURCE_DIR}/cmake/match.cmake
+                DEPENDS ${TEST_TARGET_NAME}
+                WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+            )
+        else()
+            separate_arguments(TEST_COMMAND)
+            add_test(NAME ${tname}
+                COMMAND ${TEST_COMMAND}
+                DEPENDS ${TEST_TARGET_NAME}
+                WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+            )
+        endif()
 
-    set(TEST_ENV UR_ADAPTERS_FORCE_LOAD="$<TARGET_FILE:ur_${adapter}>")
-    if(UR_CONFORMANCE_ENABLE_MATCH_FILES)
-        list(APPEND TEST_ENV GTEST_COLOR=no)
+        if(UR_CONFORMANCE_ENABLE_MATCH_FILES)
+            list(APPEND env GTEST_COLOR=no)
+        endif()
+        set_tests_properties(${TEST_NAME} PROPERTIES
+            ENVIRONMENT "${env}"
+            LABELS "conformance;${adapter}")
+    endfunction()
+
+    do_add_test(${TEST_NAME} UR_ADAPTERS_FORCE_LOAD="$<TARGET_FILE:ur_${adapter}>")
+    if(UR_CONFORMANCE_TEST_LOADER)
+        do_add_test(${TEST_NAME}-loader "")
     endif()
-    set_tests_properties(${TEST_NAME} PROPERTIES
-        ENVIRONMENT "${TEST_ENV}"
-        LABELS "conformance;${adapter}")
 endfunction()
 
 function(add_conformance_test name)
@@ -66,29 +69,29 @@ function(add_conformance_test name)
         unit_tests_helpers)
 
     if(UR_BUILD_ADAPTER_CUDA OR UR_BUILD_ADAPTER_ALL)
-        add_test_adapter(${name} adapter_cuda)
+        add_test_adapter(${name} adapter_cuda CUDA)
     endif()
     if(UR_BUILD_ADAPTER_HIP OR UR_BUILD_ADAPTER_ALL)
-        add_test_adapter(${name} adapter_hip)
+        add_test_adapter(${name} adapter_hip HIP)
     endif()
     if(UR_BUILD_ADAPTER_L0 OR UR_BUILD_ADAPTER_ALL)
-        add_test_adapter(${name} adapter_level_zero)
+        add_test_adapter(${name} adapter_level_zero LEVEL_ZERO)
     endif()
     if(UR_BUILD_ADAPTER_L0_V2)
-        add_test_adapter(${name} adapter_level_zero_v2)
+        add_test_adapter(${name} adapter_level_zero_v2 LEVEL_ZERO)
     endif()
     if(UR_BUILD_ADAPTER_OPENCL OR UR_BUILD_ADAPTER_ALL)
-        add_test_adapter(${name} adapter_opencl)
+        add_test_adapter(${name} adapter_opencl OPENCL)
     endif()
     if(UR_BUILD_ADAPTER_NATIVE_CPU OR UR_BUILD_ADAPTER_ALL)
-        add_test_adapter(${name} adapter_native_cpu)
+        add_test_adapter(${name} adapter_native_cpu NATIVE_CPU)
     endif()
 
     if(NOT (UR_BUILD_ADAPTER_CUDA OR UR_BUILD_ADAPTER_HIP
             OR UR_BUILD_ADAPTER_L0 OR UR_BUILD_ADAPTER_OPENCL
             OR UR_BUILD_ADAPTER_NATIVE_CPU OR UR_BUILD_ADAPTER_L0_V2
             OR UR_BUILD_ADAPTER_ALL))
-        add_test_adapter(${name} adapter_mock)
+        add_test_adapter(${name} adapter_mock MOCK)
     endif()
 endfunction()
 

diff --git a/test/conformance/cts_exe.py b/test/conformance/cts_exe.py
@@ -23,11 +23,13 @@
     parser.add_argument("--test_command", help="Ctest test case")
     parser.add_argument("--devices_count", type=str, help="Number of devices on which tests will be run")
     parser.add_argument("--platforms_count", type=str, help="Number of platforms on which tests will be run")
+    parser.add_argument("--backend", type=str, help="Number of platforms on which tests will be run")
     args = parser.parse_args()
 
     result = subprocess.Popen([args.test_command, '--gtest_brief=1',        # nosec B603
                                f'--devices_count={args.devices_count}',
-                               f'--platforms_count={args.platforms_count}'],
+                               f'--platforms_count={args.platforms_count}',
+                               f'--backend={args.backend}'],
                                stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
 
     pat = re.compile(r'\[( )*FAILED( )*\]')