ITT stubs and wrappers for SPIR-V devices.

libdevice/cmake/modules/SYCLLibdevice.cmake

@@ -167,12 +167,46 @@ add_custom_command(OUTPUT ${obj_binary_dir}/libsycl-fallback-cmath-fp64.${lib-su
                    DEPENDS device_math.h device.h clang clang-offload-bundler
                    VERBATIM)
 
+add_custom_command(OUTPUT ${obj_binary_dir}/libsycl-itt-stubs.${lib-suffix}
+                   COMMAND ${clang} -fsycl -c
+                           ${compile_opts} ${sycl_targets_opt}
+                           ${CMAKE_CURRENT_SOURCE_DIR}/itt_stubs.cpp
+                           -o ${obj_binary_dir}/libsycl-itt-stubs.${lib-suffix}
+                   MAIN_DEPENDENCY itt_stubs.cpp
+                   DEPENDS device_itt.h spirv_vars.h device.h clang clang-offload-bundler
+                   VERBATIM)
+
+add_custom_command(OUTPUT ${obj_binary_dir}/libsycl-itt-compiler-wrappers.${lib-suffix}
+                   COMMAND ${clang} -fsycl -c
+                           ${compile_opts} ${sycl_targets_opt}
+                           ${CMAKE_CURRENT_SOURCE_DIR}/itt_compiler_wrappers.cpp
+                           -o ${obj_binary_dir}/libsycl-itt-compiler-wrappers.${lib-suffix}
+                   MAIN_DEPENDENCY itt_compiler_wrappers.cpp
+                   DEPENDS device_itt.h spirv_vars.h device.h clang clang-offload-bundler
+                   VERBATIM)
+
+add_custom_command(OUTPUT ${obj_binary_dir}/libsycl-itt-user-wrappers.${lib-suffix}
+                   COMMAND ${clang} -fsycl -c
+                           ${compile_opts} ${sycl_targets_opt}
+                           ${CMAKE_CURRENT_SOURCE_DIR}/itt_user_wrappers.cpp
+                           -o ${obj_binary_dir}/libsycl-itt-user-wrappers.${lib-suffix}
+                   MAIN_DEPENDENCY itt_user_wrappers.cpp
+                   DEPENDS device_itt.h spirv_vars.h device.h clang clang-offload-bundler
+                   VERBATIM)
+
+set(devicelib-obj-itt-files
+  ${obj_binary_dir}/libsycl-itt-stubs.${lib-suffix}
+  ${obj_binary_dir}/libsycl-itt-compiler-wrappers.${lib-suffix}
+  ${obj_binary_dir}/libsycl-itt-user-wrappers.${lib-suffix}
+  )
+
 add_custom_target(libsycldevice-obj DEPENDS
   ${devicelib-obj-file}
   ${devicelib-obj-complex}
   ${devicelib-obj-complex-fp64}
   ${devicelib-obj-cmath}
   ${devicelib-obj-cmath-fp64}
+  ${devicelib-obj-itt-files}
 )
 add_custom_target(libsycldevice-spv DEPENDS
   ${spv_binary_dir}/libsycl-fallback-cassert.spv
@@ -212,7 +246,8 @@ install(FILES ${devicelib-obj-file}
               ${devicelib-obj-cmath}
               ${obj_binary_dir}/libsycl-fallback-cmath.${lib-suffix}
               ${devicelib-obj-cmath-fp64}
-	      ${obj_binary_dir}/libsycl-fallback-cmath-fp64.${lib-suffix}
+              ${obj_binary_dir}/libsycl-fallback-cmath-fp64.${lib-suffix}
+              ${devicelib-obj-itt-files}
         DESTINATION ${install_dest_lib}
         COMPONENT libsycldevice)
 

libdevice/device_itt.h

@@ -0,0 +1,103 @@
+//==------- device_itt.h - ITT devicelib functions declarations ------------==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//==------------------------------------------------------------------------==//
+
+#ifndef __LIBDEVICE_DEVICE_ITT_H__
+#define __LIBDEVICE_DEVICE_ITT_H__
+
+#include "device.h"
+
+#ifdef __SPIR__
+#include "spirv_vars.h"
+
+#define ITT_STUB_ATTRIBUTES __attribute__((noinline, optnone))
+
+/// Atomic operation type
+enum __itt_atomic_mem_op_t {
+  __itt_mem_load = 0,
+  __itt_mem_store = 1,
+  __itt_mem_update = 2
+};
+
+/// Memory operation ordering semantic type
+enum __itt_atomic_mem_order_t {
+  __itt_mem_order_relaxed = 0,
+  __itt_mem_order_acquire = 1,
+  __itt_mem_order_release = 2,
+  __itt_mem_order_acquire_release = 3
+};
+
+// FIXME: must be enabled via -fdeclare-spirv-builtins
+DEVICE_EXTERN_C char __spirv_SpecConstant(int, char);
+
+#define ITT_SPEC_CONSTANT 0xFF747469
+
+static inline bool isITTEnabled() {
+  return __spirv_SpecConstant(ITT_SPEC_CONSTANT, 0) != 0;
+}
+
+// Wrapper APIs that may be called by compiler-generated code.
+// These are just parameterless helper APIs that call the corresponding
+// stub APIs after preparing the arguments for them.
+//
+// Note that we do not provide compiler wrappers for all stub APIs.
+// For example, there is no compiler wrapper for
+// __itt_offload_sync_acquired_stub, since the API's parameter cannot
+// be computed in the wrapper itself and has to be passed from outside.
+// If a compiler needs to invoke such an API, it has to use the user
+// visible API directly (i.e. __itt_offload_sync_acquired).
+DEVICE_EXTERN_C
+void __itt_offload_wi_start_wrapper();
+DEVICE_EXTERN_C
+void __itt_offload_wi_finish_wrapper();
+DEVICE_EXTERN_C
+void __itt_offload_wg_barrier_wrapper();
+DEVICE_EXTERN_C
+void __itt_offload_wi_resume_wrapper();
+
+// Non-inlinable and non-optimizable APIs that are recognized
+// by profiling tools.
+DEVICE_EXTERN_C ITT_STUB_ATTRIBUTES void
+__itt_offload_wi_start_stub(size_t *group_id, size_t wi_id, uint32_t wg_size);
+DEVICE_EXTERN_C ITT_STUB_ATTRIBUTES void
+__itt_offload_wi_finish_stub(size_t *group_id, size_t wi_id);
+DEVICE_EXTERN_C ITT_STUB_ATTRIBUTES void
+__itt_offload_wg_barrier_stub(uintptr_t barrier_id);
+DEVICE_EXTERN_C ITT_STUB_ATTRIBUTES void
+__itt_offload_wi_resume_stub(size_t *group_id, size_t wi_id);
+DEVICE_EXTERN_C ITT_STUB_ATTRIBUTES void
+__itt_offload_sync_acquired_stub(uintptr_t sync_id);
+DEVICE_EXTERN_C ITT_STUB_ATTRIBUTES void
+__itt_offload_sync_releasing_stub(uintptr_t sync_id);
+DEVICE_EXTERN_C ITT_STUB_ATTRIBUTES void
+__itt_offload_wg_local_range_stub(void *ptr, size_t size);
+DEVICE_EXTERN_C ITT_STUB_ATTRIBUTES void
+__itt_offload_atomic_op_start_stub(void *object, __itt_atomic_mem_op_t op_type,
+                                   __itt_atomic_mem_order_t mem_order);
+DEVICE_EXTERN_C ITT_STUB_ATTRIBUTES void
+__itt_offload_atomic_op_finish_stub(void *object, __itt_atomic_mem_op_t op_type,
+                                    __itt_atomic_mem_order_t mem_order);
+
+// User visible APIs. These may called both from user code and from
+// compiler generated code.
+DEVICE_EXTERN_C void __itt_offload_wi_start(size_t *group_id, size_t wi_id,
+                                            uint32_t wg_size);
+DEVICE_EXTERN_C void __itt_offload_wi_finish(size_t *group_id, size_t wi_id);
+DEVICE_EXTERN_C void __itt_offload_wg_barrier(uintptr_t barrier_id);
+DEVICE_EXTERN_C void __itt_offload_wi_resume(size_t *group_id, size_t wi_id);
+DEVICE_EXTERN_C void __itt_offload_sync_acquired(uintptr_t sync_id);
+DEVICE_EXTERN_C void __itt_offload_sync_releasing(uintptr_t sync_id);
+DEVICE_EXTERN_C void __itt_offload_wg_local_range(void *ptr, size_t size);
+DEVICE_EXTERN_C void
+__itt_offload_atomic_op_start(void *object, __itt_atomic_mem_op_t op_type,
+                              __itt_atomic_mem_order_t mem_order);
+DEVICE_EXTERN_C void
+__itt_offload_atomic_op_finish(void *object, __itt_atomic_mem_op_t op_type,
+                               __itt_atomic_mem_order_t mem_order);
+
+#endif // __SPIR__
+#endif // __LIBDEVICE_DEVICE_ITT_H__

libdevice/itt_compiler_wrappers.cpp

@@ -0,0 +1,60 @@
+//==--- itt_compiler_wrappers.cpp - compiler wrappers for ITT --------------==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "device_itt.h"
+
+#ifdef __SPIR__
+
+DEVICE_EXTERN_C
+void __itt_offload_wi_start_wrapper() {
+  if (!isITTEnabled())
+    return;
+
+  size_t GroupID[3] = {__spirv_BuiltInWorkgroupId.x,
+                       __spirv_BuiltInWorkgroupId.y,
+                       __spirv_BuiltInWorkgroupId.z};
+  size_t WIID = __spirv_BuiltInGlobalLinearId;
+  uint32_t WGSize = static_cast<uint32_t>(__spirv_BuiltInWorkgroupSize.x *
+                                          __spirv_BuiltInWorkgroupSize.y *
+                                          __spirv_BuiltInWorkgroupSize.z);
+  __itt_offload_wi_start_stub(GroupID, WIID, WGSize);
+}
+
+DEVICE_EXTERN_C
+void __itt_offload_wi_finish_wrapper() {
+  if (!isITTEnabled())
+    return;
+
+  size_t GroupID[3] = {__spirv_BuiltInWorkgroupId.x,
+                       __spirv_BuiltInWorkgroupId.y,
+                       __spirv_BuiltInWorkgroupId.z};
+  size_t WIID = __spirv_BuiltInGlobalLinearId;
+  __itt_offload_wi_finish_stub(GroupID, WIID);
+}
+
+DEVICE_EXTERN_C
+void __itt_offload_wg_barrier_wrapper() {
+  if (!isITTEnabled())
+    return;
+
+  __itt_offload_wg_barrier_stub(0);
+}
+
+DEVICE_EXTERN_C
+void __itt_offload_wi_resume_wrapper() {
+  if (!isITTEnabled())
+    return;
+
+  size_t GroupID[3] = {__spirv_BuiltInWorkgroupId.x,
+                       __spirv_BuiltInWorkgroupId.y,
+                       __spirv_BuiltInWorkgroupId.z};
+  size_t WIID = __spirv_BuiltInGlobalLinearId;
+  __itt_offload_wi_resume_stub(GroupID, WIID);
+}
+
+#endif // __SPIR__

libdevice/itt_stubs.cpp

@@ -0,0 +1,38 @@
+//==--- itt_stubs.cpp - stub functions for ITT  ----------------------------==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "device_itt.h"
+
+#ifdef __SPIR__
+
+DEVICE_EXTERN_C ITT_STUB_ATTRIBUTES void
+__itt_offload_wi_start_stub(size_t *group_id, size_t wi_id, uint32_t wg_size) {}
+
+DEVICE_EXTERN_C ITT_STUB_ATTRIBUTES void
+__itt_offload_wi_finish_stub(size_t *group_id, size_t wi_id) {}
+
+DEVICE_EXTERN_C ITT_STUB_ATTRIBUTES void
+__itt_offload_wg_barrier_stub(uintptr_t barrier_id) {}
+
+DEVICE_EXTERN_C ITT_STUB_ATTRIBUTES void
+__itt_offload_wi_resume_stub(size_t *group_id, size_t wi_id) {}
+
+DEVICE_EXTERN_C ITT_STUB_ATTRIBUTES void
+__itt_offload_sync_acquired_stub(uintptr_t sync_id) {}
+DEVICE_EXTERN_C ITT_STUB_ATTRIBUTES void
+__itt_offload_sync_releasing_stub(uintptr_t sync_id) {}
+DEVICE_EXTERN_C ITT_STUB_ATTRIBUTES void
+__itt_offload_wg_local_range_stub(void *ptr, size_t size) {}
+DEVICE_EXTERN_C ITT_STUB_ATTRIBUTES void
+__itt_offload_atomic_op_start_stub(void *object, __itt_atomic_mem_op_t op_type,
+                                   __itt_atomic_mem_order_t mem_order) {}
+DEVICE_EXTERN_C ITT_STUB_ATTRIBUTES void
+__itt_offload_atomic_op_finish_stub(void *object, __itt_atomic_mem_op_t op_type,
+                                    __itt_atomic_mem_order_t mem_order) {}
+
+#endif // __SPIR__

libdevice/itt_user_wrappers.cpp

@@ -0,0 +1,63 @@
+//==--- itt_user_wrappers.cpp - user visible functions for ITT  ------------==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "device_itt.h"
+
+#ifdef __SPIR__
+
+DEVICE_EXTERN_C void __itt_offload_wi_start(size_t *group_id, size_t wi_id,
+                                            uint32_t wg_size) {
+  if (isITTEnabled())
+    __itt_offload_wi_start_stub(group_id, wi_id, wg_size);
+}
+
+DEVICE_EXTERN_C void __itt_offload_wi_finish(size_t *group_id, size_t wi_id) {
+  if (isITTEnabled())
+    __itt_offload_wi_finish_stub(group_id, wi_id);
+}
+
+DEVICE_EXTERN_C void __itt_offload_wg_barrier(uintptr_t barrier_id) {
+  if (isITTEnabled())
+    __itt_offload_wg_barrier_stub(barrier_id);
+}
+
+DEVICE_EXTERN_C void __itt_offload_wi_resume(size_t *group_id, size_t wi_id) {
+  if (isITTEnabled())
+    __itt_offload_wi_resume_stub(group_id, wi_id);
+}
+
+DEVICE_EXTERN_C void __itt_offload_sync_acquired(uintptr_t sync_id) {
+  if (isITTEnabled())
+    __itt_offload_sync_acquired_stub(sync_id);
+}
+
+DEVICE_EXTERN_C void __itt_offload_sync_releasing(uintptr_t sync_id) {
+  if (isITTEnabled())
+    __itt_offload_sync_releasing_stub(sync_id);
+}
+
+DEVICE_EXTERN_C void __itt_offload_wg_local_range(void *ptr, size_t size) {
+  if (isITTEnabled())
+    __itt_offload_wg_local_range_stub(ptr, size);
+}
+
+DEVICE_EXTERN_C void
+__itt_offload_atomic_op_start(void *object, __itt_atomic_mem_op_t op_type,
+                              __itt_atomic_mem_order_t mem_order) {
+  if (isITTEnabled())
+    __itt_offload_atomic_op_start_stub(object, op_type, mem_order);
+}
+
+DEVICE_EXTERN_C void
+__itt_offload_atomic_op_finish(void *object, __itt_atomic_mem_op_t op_type,
+                               __itt_atomic_mem_order_t mem_order) {
+  if (isITTEnabled())
+    __itt_offload_atomic_op_finish_stub(object, op_type, mem_order);
+}
+
+#endif // __SPIR__

libdevice/spirv_vars.h

@@ -16,10 +16,21 @@
 #include <cstddef>
 #include <cstdint>
 
+#define __SPIRV_VAR_QUALIFIERS EXTERN_C const
 typedef size_t size_t_vec __attribute__((ext_vector_type(3)));
-extern "C" const size_t_vec __spirv_BuiltInGlobalInvocationId;
-extern "C" const size_t_vec __spirv_BuiltInLocalInvocationId;
-
+__SPIRV_VAR_QUALIFIERS size_t_vec __spirv_BuiltInGlobalInvocationId;
+__SPIRV_VAR_QUALIFIERS size_t __spirv_BuiltInGlobalLinearId;
+__SPIRV_VAR_QUALIFIERS size_t_vec __spirv_BuiltInLocalInvocationId;
+__SPIRV_VAR_QUALIFIERS size_t_vec __spirv_BuiltInWorkgroupId;
+__SPIRV_VAR_QUALIFIERS size_t_vec __spirv_BuiltInWorkgroupSize;
+
+// FIXME: change DEVICE_EXTERNAL to static and rename the functions,
+//        when #3311 is fixed.
+//        These are just internal functions used within libdevice.
+//        We must not intrude the __spirv "namespace", so we'd better
+//        use names like getGlobalInvocationIdX.
+//        Libdevice must not export these APIs either, but it currently
+//        exports them due to DEVICE_EXTERNAL.
 DEVICE_EXTERNAL inline size_t __spirv_GlobalInvocationId_x() {
   return __spirv_BuiltInGlobalInvocationId.x;
 }