Improved adapter selection in CTS testing

The CTS tests now have an additional --backend option, and the --platform option allows specifying platforms per-device. In addition, new targets have been added that test the "loader"; that is, where the adapter isn't forced.
oneapi-src · Oct 3, 2024 · 90b9a46 · 90b9a46
1 parent 32e8de4
commit 90b9a46
Show file tree

Hide file tree

Showing 72 changed files with 284 additions and 65 deletions.
diff --git a/.github/workflows/build-hw-reusable.yml b/.github/workflows/build-hw-reusable.yml
@@ -82,6 +82,7 @@ jobs:
         -DUR_ENABLE_TRACING=ON
         -DUR_DEVELOPER_MODE=ON
         -DUR_BUILD_TESTS=ON
+        -DUR_CONFORMANCE_TEST_LOADER=OFF
         -DUR_BUILD_ADAPTER_${{matrix.adapter.name}}=ON
         -DUR_STATIC_LOADER=${{matrix.adapter.static_Loader}}
         -DUR_STATIC_ADAPTER_${{matrix.adapter.name}}=${{matrix.adapter.static_adapter}}

diff --git a/.github/workflows/multi_device.yml b/.github/workflows/multi_device.yml
@@ -47,6 +47,7 @@ jobs:
         -DUR_DEVELOPER_MODE=ON
         -DUR_BUILD_TESTS=ON
         -DUR_BUILD_ADAPTER_${{matrix.adapter.name}}=ON
+        -DUR_CONFORMANCE_TEST_LOADER=OFF
         -DUR_TEST_DEVICES_COUNT=2
         -DUR_DPCXX=${{github.workspace}}/dpcpp_compiler/bin/clang++
         -DUR_SYCL_LIBRARY_DIR=${{github.workspace}}/dpcpp_compiler/lib

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -68,6 +68,7 @@ set(UR_CONFORMANCE_TARGET_TRIPLES "" CACHE STRING
     "List of sycl targets to build CTS device binaries for")
 set(UR_CONFORMANCE_AMD_ARCH "" CACHE STRING "AMD device target ID to build CTS binaries for")
 option(UR_CONFORMANCE_ENABLE_MATCH_FILES "Enable CTS match files" ON)
+option(UR_CONFORMANCE_TEST_LOADER "Also test the loader in the conformance tests" ON)
 set(UR_ADAPTER_LEVEL_ZERO_SOURCE_DIR "" CACHE PATH
     "Path to external 'level_zero' adapter source dir")
 set(UR_ADAPTER_OPENCL_SOURCE_DIR "" CACHE PATH

diff --git a/test/adapters/CMakeLists.txt b/test/adapters/CMakeLists.txt
@@ -63,7 +63,7 @@ function(add_adapter_memcheck_test name)
         add_test(NAME ${test_name}
                 COMMAND ${CMAKE_COMMAND}
                 -D TEST_FILE=valgrind
-                -D TEST_ARGS="--tool=memcheck --leak-check=full $<TARGET_FILE:${target}> --devices_count=${UR_TEST_DEVICES_COUNT} --platforms_count=${UR_TEST_DEVICES_COUNT}"
+                -D TEST_ARGS="--tool=memcheck --leak-check=full $<TARGET_FILE:${target}> --backend=${backend} --devices_count=${UR_TEST_DEVICES_COUNT} --platforms_count=${UR_TEST_DEVICES_COUNT}"
                 -D MODE=stderr
                 -D MATCH_FILE=${CMAKE_CURRENT_SOURCE_DIR}/${name}_memcheck.match
                 -P ${PROJECT_SOURCE_DIR}/cmake/match.cmake

diff --git a/test/conformance/CMakeLists.txt b/test/conformance/CMakeLists.txt
@@ -9,15 +9,15 @@ set(UR_CONFORMANCE_DEVICE_BINARIES_DIR
     "${CMAKE_CURRENT_BINARY_DIR}/device_binaries" CACHE INTERNAL
     "Internal cache variable for device binaries directory")
 
-function(add_test_adapter name adapter)
+function(add_test_adapter name adapter backend force)
     if(NOT "${ARGN}" STREQUAL "")
         set(EXTRA_NAME "-${ARGN}")
     endif()
     set(TEST_TARGET_NAME test-${name})
     set(TEST_NAME ${name}-${adapter}${EXTRA_NAME})
 
     set(TEST_COMMAND
-        "${PROJECT_BINARY_DIR}/bin/${TEST_TARGET_NAME} --devices_count=${UR_TEST_DEVICES_COUNT} --platforms_count=${UR_TEST_PLATFORMS_COUNT}"
+        "${PROJECT_BINARY_DIR}/bin/${TEST_TARGET_NAME} --backend=${backend} --devices_count=${UR_TEST_DEVICES_COUNT} --platforms_count=${UR_TEST_PLATFORMS_COUNT}"
     )
     set(MATCH_FILE "${CMAKE_CURRENT_SOURCE_DIR}/${name}_${adapter}${EXTRA_NAME}.match")
 
@@ -41,7 +41,9 @@ function(add_test_adapter name adapter)
         )
     endif()
 
-    set(TEST_ENV UR_ADAPTERS_FORCE_LOAD="$<TARGET_FILE:ur_${adapter}>")
+    if(${force})
+        set(TEST_ENV UR_ADAPTERS_FORCE_LOAD="$<TARGET_FILE:ur_${adapter}>")
+    endif()
     if(UR_CONFORMANCE_ENABLE_MATCH_FILES)
         list(APPEND TEST_ENV GTEST_COLOR=no)
     endif()
@@ -66,29 +68,50 @@ function(add_conformance_test name)
         unit_tests_helpers)
 
     if(UR_BUILD_ADAPTER_CUDA OR UR_BUILD_ADAPTER_ALL)
-        add_test_adapter(${name} adapter_cuda)
+        add_test_adapter(${name} adapter_cuda CUDA ON)
     endif()
     if(UR_BUILD_ADAPTER_HIP OR UR_BUILD_ADAPTER_ALL)
-        add_test_adapter(${name} adapter_hip)
+        add_test_adapter(${name} adapter_hip HIP ON)
     endif()
     if(UR_BUILD_ADAPTER_L0 OR UR_BUILD_ADAPTER_ALL)
-        add_test_adapter(${name} adapter_level_zero)
+        add_test_adapter(${name} adapter_level_zero LEVEL_ZERO ON)
     endif()
     if(UR_BUILD_ADAPTER_L0_V2)
-        add_test_adapter(${name} adapter_level_zero_v2)
+        add_test_adapter(${name} adapter_level_zero_v2 LEVEL_ZERO ON)
     endif()
     if(UR_BUILD_ADAPTER_OPENCL OR UR_BUILD_ADAPTER_ALL)
-        add_test_adapter(${name} adapter_opencl)
+        add_test_adapter(${name} adapter_opencl OPENCL ON)
     endif()
     if(UR_BUILD_ADAPTER_NATIVE_CPU OR UR_BUILD_ADAPTER_ALL)
-        add_test_adapter(${name} adapter_native_cpu)
+        add_test_adapter(${name} adapter_native_cpu NATIVE_CPU ON)
+    endif()
+
+    if(UR_CONFORMANCE_TEST_LOADER)
+        if(UR_BUILD_ADAPTER_CUDA OR UR_BUILD_ADAPTER_ALL)
+            add_test_adapter(${name} adapter_cuda_loader CUDA OFF)
+        endif()
+        if(UR_BUILD_ADAPTER_HIP OR UR_BUILD_ADAPTER_ALL)
+            add_test_adapter(${name} adapter_hip_loader HIP OFF)
+        endif()
+        if(UR_BUILD_ADAPTER_L0 OR UR_BUILD_ADAPTER_ALL)
+            add_test_adapter(${name} adapter_level_zero_loader LEVEL_ZERO OFF)
+        endif()
+        if(UR_BUILD_ADAPTER_L0_V2)
+            add_test_adapter(${name} adapter_level_zero_v2_loader LEVEL_ZERO OFF)
+        endif()
+        if(UR_BUILD_ADAPTER_OPENCL OR UR_BUILD_ADAPTER_ALL)
+            add_test_adapter(${name} adapter_opencl_loader OPENCL OFF)
+        endif()
+        if(UR_BUILD_ADAPTER_NATIVE_CPU OR UR_BUILD_ADAPTER_ALL)
+            add_test_adapter(${name} adapter_native_cpu_loader NATIVE_CPU OFF)
+        endif()
     endif()
 
     if(NOT (UR_BUILD_ADAPTER_CUDA OR UR_BUILD_ADAPTER_HIP
             OR UR_BUILD_ADAPTER_L0 OR UR_BUILD_ADAPTER_OPENCL
             OR UR_BUILD_ADAPTER_NATIVE_CPU OR UR_BUILD_ADAPTER_L0_V2
             OR UR_BUILD_ADAPTER_ALL))
-        add_test_adapter(${name} adapter_mock)
+        add_test_adapter(${name} adapter_mock MOCK ON)
     endif()
 endfunction()
 

diff --git a/test/conformance/adapter/adapter_adapter_native_cpu_loader.match b/test/conformance/adapter/adapter_adapter_native_cpu_loader.match
@@ -0,0 +1 @@
+adapter_adapter_native_cpu.match
diff --git a/test/conformance/context/context_adapter_level_zero_loader.match b/test/conformance/context/context_adapter_level_zero_loader.match
@@ -0,0 +1 @@
+context_adapter_level_zero.match
diff --git a/test/conformance/context/context_adapter_level_zero_v2_loader.match b/test/conformance/context/context_adapter_level_zero_v2_loader.match
@@ -0,0 +1 @@
+context_adapter_level_zero_v2.match
diff --git a/test/conformance/context/context_adapter_native_cpu_loader.match b/test/conformance/context/context_adapter_native_cpu_loader.match
@@ -0,0 +1 @@
+context_adapter_native_cpu.match
diff --git a/test/conformance/cts_exe.py b/test/conformance/cts_exe.py
@@ -23,11 +23,13 @@
     parser.add_argument("--test_command", help="Ctest test case")
     parser.add_argument("--devices_count", type=str, help="Number of devices on which tests will be run")
     parser.add_argument("--platforms_count", type=str, help="Number of platforms on which tests will be run")
+    parser.add_argument("--backend", type=str, help="Number of platforms on which tests will be run")
     args = parser.parse_args()
 
     result = subprocess.Popen([args.test_command, '--gtest_brief=1',        # nosec B603
                                f'--devices_count={args.devices_count}',
-                               f'--platforms_count={args.platforms_count}'],
+                               f'--platforms_count={args.platforms_count}',
+                               f'--backend={args.backend}'],
                                stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
 
     pat = re.compile(r'\[( )*FAILED( )*\]')

diff --git a/test/conformance/device/device_adapter_cuda_loader.match b/test/conformance/device/device_adapter_cuda_loader.match
@@ -0,0 +1 @@
+device_adapter_cuda.match
diff --git a/test/conformance/device/device_adapter_hip_loader.match b/test/conformance/device/device_adapter_hip_loader.match
@@ -0,0 +1 @@
+device_adapter_hip.match
diff --git a/test/conformance/device/device_adapter_level_zero_loader.match b/test/conformance/device/device_adapter_level_zero_loader.match
@@ -0,0 +1 @@
+device_adapter_level_zero.match
diff --git a/test/conformance/device/device_adapter_level_zero_v2_loader.match b/test/conformance/device/device_adapter_level_zero_v2_loader.match
@@ -0,0 +1 @@
+device_adapter_level_zero_v2.match
diff --git a/test/conformance/device/device_adapter_native_cpu_loader.match b/test/conformance/device/device_adapter_native_cpu_loader.match
@@ -0,0 +1 @@
+device_adapter_native_cpu.match
diff --git a/test/conformance/device/device_adapter_opencl_loader.match b/test/conformance/device/device_adapter_opencl_loader.match
@@ -0,0 +1 @@
+device_adapter_opencl.match
diff --git a/test/conformance/enqueue/enqueue_adapter_cuda_loader.match b/test/conformance/enqueue/enqueue_adapter_cuda_loader.match
@@ -0,0 +1 @@
+enqueue_adapter_cuda.match
diff --git a/test/conformance/enqueue/enqueue_adapter_hip_loader.match b/test/conformance/enqueue/enqueue_adapter_hip_loader.match
@@ -0,0 +1 @@
+enqueue_adapter_hip.match
diff --git a/test/conformance/enqueue/enqueue_adapter_level_zero_loader.match b/test/conformance/enqueue/enqueue_adapter_level_zero_loader.match
@@ -0,0 +1 @@
+enqueue_adapter_level_zero.match
diff --git a/test/conformance/enqueue/enqueue_adapter_level_zero_v2_loader.match b/test/conformance/enqueue/enqueue_adapter_level_zero_v2_loader.match
@@ -0,0 +1 @@
+enqueue_adapter_level_zero_v2.match
diff --git a/test/conformance/enqueue/enqueue_adapter_native_cpu_loader.match b/test/conformance/enqueue/enqueue_adapter_native_cpu_loader.match
@@ -0,0 +1 @@
+enqueue_adapter_native_cpu.match
diff --git a/test/conformance/enqueue/enqueue_adapter_opencl_loader.match b/test/conformance/enqueue/enqueue_adapter_opencl_loader.match
@@ -0,0 +1 @@
+enqueue_adapter_opencl.match
diff --git a/test/conformance/event/event_adapter_cuda_loader.match b/test/conformance/event/event_adapter_cuda_loader.match
@@ -0,0 +1 @@
+event_adapter_cuda.match
diff --git a/test/conformance/event/event_adapter_hip_loader.match b/test/conformance/event/event_adapter_hip_loader.match
@@ -0,0 +1 @@
+event_adapter_hip.match
diff --git a/test/conformance/event/event_adapter_level_zero_loader.match b/test/conformance/event/event_adapter_level_zero_loader.match
@@ -0,0 +1 @@
+event_adapter_level_zero.match
diff --git a/test/conformance/event/event_adapter_level_zero_v2_loader.match b/test/conformance/event/event_adapter_level_zero_v2_loader.match
@@ -0,0 +1 @@
+event_adapter_level_zero_v2.match
diff --git a/test/conformance/event/event_adapter_native_cpu_loader.match b/test/conformance/event/event_adapter_native_cpu_loader.match
@@ -0,0 +1 @@
+event_adapter_native_cpu.match
diff --git a/test/conformance/exp_command_buffer/exp_command_buffer_adapter_cuda_loader.match b/test/conformance/exp_command_buffer/exp_command_buffer_adapter_cuda_loader.match
@@ -0,0 +1 @@
+exp_command_buffer_adapter_cuda.match
diff --git a/test/conformance/exp_command_buffer/exp_command_buffer_adapter_hip_loader.match b/test/conformance/exp_command_buffer/exp_command_buffer_adapter_hip_loader.match
@@ -0,0 +1 @@
+exp_command_buffer_adapter_hip.match
diff --git a/test/conformance/exp_command_buffer/exp_command_buffer_adapter_level_zero_v2_loader.match b/test/conformance/exp_command_buffer/exp_command_buffer_adapter_level_zero_v2_loader.match
@@ -0,0 +1 @@
+exp_command_buffer_adapter_level_zero_v2.match
diff --git a/test/conformance/exp_command_buffer/exp_command_buffer_adapter_native_cpu_loader.match b/test/conformance/exp_command_buffer/exp_command_buffer_adapter_native_cpu_loader.match
@@ -0,0 +1 @@
+exp_command_buffer_adapter_native_cpu.match
diff --git a/test/conformance/exp_enqueue_native/exp_enqueue_native_adapter_level_zero_v2_loader.match b/test/conformance/exp_enqueue_native/exp_enqueue_native_adapter_level_zero_v2_loader.match
@@ -0,0 +1 @@
+exp_enqueue_native_adapter_level_zero_v2.match
diff --git a/test/conformance/exp_launch_properties/exp_launch_properties_adapter_native_cpu_loader.match b/test/conformance/exp_launch_properties/exp_launch_properties_adapter_native_cpu_loader.match
@@ -0,0 +1 @@
+exp_launch_properties_adapter_native_cpu.match
diff --git a/test/conformance/integration/integration_adapter_level_zero_loader.match b/test/conformance/integration/integration_adapter_level_zero_loader.match
@@ -0,0 +1 @@
+integration_adapter_level_zero.match
diff --git a/test/conformance/integration/integration_adapter_level_zero_v2_loader.match b/test/conformance/integration/integration_adapter_level_zero_v2_loader.match
@@ -0,0 +1 @@
+integration_adapter_level_zero_v2.match
diff --git a/test/conformance/integration/integration_adapter_native_cpu_loader.match b/test/conformance/integration/integration_adapter_native_cpu_loader.match
@@ -0,0 +1 @@
+integration_adapter_native_cpu.match
diff --git a/test/conformance/integration/integration_adapter_opencl_loader.match b/test/conformance/integration/integration_adapter_opencl_loader.match
@@ -0,0 +1 @@
+integration_adapter_opencl.match
diff --git a/test/conformance/kernel/kernel_adapter_cuda_loader.match b/test/conformance/kernel/kernel_adapter_cuda_loader.match
@@ -0,0 +1 @@
+kernel_adapter_cuda.match
diff --git a/test/conformance/kernel/kernel_adapter_hip_loader.match b/test/conformance/kernel/kernel_adapter_hip_loader.match
@@ -0,0 +1 @@
+kernel_adapter_hip.match
diff --git a/test/conformance/kernel/kernel_adapter_level_zero_loader.match b/test/conformance/kernel/kernel_adapter_level_zero_loader.match
@@ -0,0 +1 @@
+kernel_adapter_level_zero.match
diff --git a/test/conformance/kernel/kernel_adapter_level_zero_v2_loader.match b/test/conformance/kernel/kernel_adapter_level_zero_v2_loader.match
@@ -0,0 +1 @@
+kernel_adapter_level_zero_v2.match
diff --git a/test/conformance/kernel/kernel_adapter_native_cpu_loader.match b/test/conformance/kernel/kernel_adapter_native_cpu_loader.match
@@ -0,0 +1 @@
+kernel_adapter_native_cpu.match
diff --git a/test/conformance/kernel/kernel_adapter_opencl_loader.match b/test/conformance/kernel/kernel_adapter_opencl_loader.match
@@ -0,0 +1 @@
+kernel_adapter_opencl.match
diff --git a/test/conformance/memory/memory_adapter_cuda_loader.match b/test/conformance/memory/memory_adapter_cuda_loader.match
@@ -0,0 +1 @@
+memory_adapter_cuda.match
diff --git a/test/conformance/memory/memory_adapter_hip_loader.match b/test/conformance/memory/memory_adapter_hip_loader.match
@@ -0,0 +1 @@
+memory_adapter_hip.match
diff --git a/test/conformance/memory/memory_adapter_level_zero_loader.match b/test/conformance/memory/memory_adapter_level_zero_loader.match
@@ -0,0 +1 @@
+memory_adapter_level_zero.match
diff --git a/test/conformance/memory/memory_adapter_level_zero_v2_loader.match b/test/conformance/memory/memory_adapter_level_zero_v2_loader.match
@@ -0,0 +1 @@
+memory_adapter_level_zero_v2.match
diff --git a/test/conformance/memory/memory_adapter_native_cpu_loader.match b/test/conformance/memory/memory_adapter_native_cpu_loader.match
@@ -0,0 +1 @@
+memory_adapter_native_cpu.match
diff --git a/test/conformance/memory/memory_adapter_opencl_loader.match b/test/conformance/memory/memory_adapter_opencl_loader.match
@@ -0,0 +1 @@
+memory_adapter_opencl.match
diff --git a/test/conformance/platform/platform_adapter_cuda_loader.match b/test/conformance/platform/platform_adapter_cuda_loader.match
@@ -0,0 +1 @@
+platform_adapter_cuda.match
diff --git a/test/conformance/platform/platform_adapter_hip_loader.match b/test/conformance/platform/platform_adapter_hip_loader.match
@@ -0,0 +1 @@
+platform_adapter_hip.match
diff --git a/test/conformance/platform/platform_adapter_native_cpu_loader.match b/test/conformance/platform/platform_adapter_native_cpu_loader.match
@@ -0,0 +1 @@
+platform_adapter_native_cpu.match
diff --git a/test/conformance/program/program_adapter_cuda_loader.match b/test/conformance/program/program_adapter_cuda_loader.match
@@ -0,0 +1 @@
+program_adapter_cuda.match
diff --git a/test/conformance/program/program_adapter_hip_loader.match b/test/conformance/program/program_adapter_hip_loader.match
@@ -0,0 +1 @@
+program_adapter_hip.match
diff --git a/test/conformance/program/program_adapter_level_zero_loader.match b/test/conformance/program/program_adapter_level_zero_loader.match
@@ -0,0 +1 @@
+program_adapter_level_zero.match
diff --git a/test/conformance/program/program_adapter_level_zero_v2_loader.match b/test/conformance/program/program_adapter_level_zero_v2_loader.match
@@ -0,0 +1 @@
+program_adapter_level_zero_v2.match
diff --git a/test/conformance/program/program_adapter_native_cpu_loader.match b/test/conformance/program/program_adapter_native_cpu_loader.match
@@ -0,0 +1 @@
+program_adapter_native_cpu.match
diff --git a/test/conformance/program/program_adapter_opencl_loader.match b/test/conformance/program/program_adapter_opencl_loader.match
@@ -0,0 +1 @@
+program_adapter_opencl.match
diff --git a/test/conformance/queue/queue_adapter_level_zero_loader.match b/test/conformance/queue/queue_adapter_level_zero_loader.match
@@ -0,0 +1 @@
+queue_adapter_level_zero.match
diff --git a/test/conformance/queue/queue_adapter_native_cpu_loader.match b/test/conformance/queue/queue_adapter_native_cpu_loader.match
@@ -0,0 +1 @@
+queue_adapter_native_cpu.match
diff --git a/test/conformance/sampler/sampler_adapter_level_zero_loader.match b/test/conformance/sampler/sampler_adapter_level_zero_loader.match
@@ -0,0 +1 @@
+sampler_adapter_level_zero.match
diff --git a/test/conformance/sampler/sampler_adapter_level_zero_v2_loader.match b/test/conformance/sampler/sampler_adapter_level_zero_v2_loader.match
@@ -0,0 +1 @@
+sampler_adapter_level_zero_v2.match