Skip to content

[SYCL][ESIMD]Fix regression in atomic_update #8673

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Mar 22, 2023
Merged

[SYCL][ESIMD]Fix regression in atomic_update #8673

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 65 additions & 40 deletions sycl/include/sycl/ext/intel/experimental/esimd/memory.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2285,7 +2285,9 @@ template <__ESIMD_NS::atomic_op Op, typename T, int N,
lsc_data_size DS = lsc_data_size::default_size,
cache_hint L1H = cache_hint::none, cache_hint L3H = cache_hint::none,
typename Toffset>
__ESIMD_API __ESIMD_NS::simd<T, N>
__ESIMD_API std::enable_if_t<
__ESIMD_DNS::get_num_args<__ESIMD_DNS::to_lsc_atomic_op<Op>()>() == 0,
__ESIMD_NS::simd<T, N>>
lsc_atomic_update(T *p, __ESIMD_NS::simd<Toffset, N> offsets,
__ESIMD_NS::simd_mask<N> pred) {
static_assert(std::is_integral_v<Toffset>, "Unsupported offset type");
Expand Down Expand Up @@ -2317,7 +2319,9 @@ template <__ESIMD_NS::atomic_op Op, typename T, int N,
cache_hint L1H = cache_hint::none, cache_hint L3H = cache_hint::none,
typename Toffset,
typename RegionTy = __ESIMD_NS::region1d_t<Toffset, N, 1>>
__ESIMD_API __ESIMD_NS::simd<T, N>
__ESIMD_API std::enable_if_t<
__ESIMD_DNS::get_num_args<__ESIMD_DNS::to_lsc_atomic_op<Op>()>() == 0,
__ESIMD_NS::simd<T, N>>
lsc_atomic_update(T *p, __ESIMD_NS::simd_view<Toffset, RegionTy> offsets,
__ESIMD_NS::simd_mask<N> pred = 1) {
using Ty = typename __ESIMD_NS::simd_view<Toffset, RegionTy>::element_type;
Expand All @@ -2329,8 +2333,10 @@ template <__ESIMD_NS::atomic_op Op, typename T, int N,
lsc_data_size DS = lsc_data_size::default_size,
cache_hint L1H = cache_hint::none, cache_hint L3H = cache_hint::none,
typename Toffset>
__ESIMD_API
std::enable_if_t<std::is_integral_v<Toffset>, __ESIMD_NS::simd<T, N>>
__ESIMD_API std::enable_if_t<
std::is_integral_v<Toffset> &&
__ESIMD_DNS::get_num_args<__ESIMD_DNS::to_lsc_atomic_op<Op>()>() == 0,
__ESIMD_NS::simd<T, N>>
lsc_atomic_update(T *p, Toffset offset, __ESIMD_NS::simd_mask<N> pred = 1) {
return lsc_atomic_update<Op, T, N, DS, L1H, L3H>(
p, __ESIMD_NS::simd<Toffset, N>(offset), pred);
Expand All @@ -2355,7 +2361,9 @@ template <__ESIMD_NS::atomic_op Op, typename T, int N,
lsc_data_size DS = lsc_data_size::default_size,
cache_hint L1H = cache_hint::none, cache_hint L3H = cache_hint::none,
typename Toffset>
__ESIMD_API __ESIMD_NS::simd<T, N>
__ESIMD_API std::enable_if_t<
__ESIMD_DNS::get_num_args<__ESIMD_DNS::to_lsc_atomic_op<Op>()>() == 1,
__ESIMD_NS::simd<T, N>>
lsc_atomic_update(T *p, __ESIMD_NS::simd<Toffset, N> offsets,
__ESIMD_NS::simd<T, N> src0, __ESIMD_NS::simd_mask<N> pred) {
static_assert(std::is_integral_v<Toffset>, "Unsupported offset type");
Expand Down Expand Up @@ -2388,7 +2396,9 @@ template <__ESIMD_NS::atomic_op Op, typename T, int N,
cache_hint L1H = cache_hint::none, cache_hint L3H = cache_hint::none,
typename Toffset,
typename RegionTy = __ESIMD_NS::region1d_t<Toffset, N, 1>>
__ESIMD_API __ESIMD_NS::simd<T, N>
__ESIMD_API std::enable_if_t<
__ESIMD_DNS::get_num_args<__ESIMD_DNS::to_lsc_atomic_op<Op>()>() == 1,
__ESIMD_NS::simd<T, N>>
lsc_atomic_update(T *p, __ESIMD_NS::simd_view<Toffset, RegionTy> offsets,
__ESIMD_NS::simd<T, N> src0,
__ESIMD_NS::simd_mask<N> pred = 1) {
Expand All @@ -2401,11 +2411,13 @@ template <__ESIMD_NS::atomic_op Op, typename T, int N,
lsc_data_size DS = lsc_data_size::default_size,
cache_hint L1H = cache_hint::none, cache_hint L3H = cache_hint::none,
typename Toffset>
__ESIMD_API std::enable_if_t<std::is_integral_v<Toffset> &&
((Op != __ESIMD_NS::atomic_op::store &&
Op != __ESIMD_NS::atomic_op::xchg) ||
N == 1),
__ESIMD_NS::simd<T, N>>
__ESIMD_API std::enable_if_t<
std::is_integral_v<Toffset> &&
__ESIMD_DNS::get_num_args<__ESIMD_DNS::to_lsc_atomic_op<Op>()>() == 1 &&
((Op != __ESIMD_NS::atomic_op::store &&
Op != __ESIMD_NS::atomic_op::xchg) ||
N == 1),
__ESIMD_NS::simd<T, N>>
lsc_atomic_update(T *p, Toffset offset, __ESIMD_NS::simd<T, N> src0,
__ESIMD_NS::simd_mask<N> pred = 1) {
return lsc_atomic_update<Op, T, N, DS, L1H, L3H>(
Expand All @@ -2431,7 +2443,9 @@ template <__ESIMD_NS::atomic_op Op, typename T, int N,
lsc_data_size DS = lsc_data_size::default_size,
cache_hint L1H = cache_hint::none, cache_hint L3H = cache_hint::none,
typename Toffset>
__ESIMD_API __ESIMD_NS::simd<T, N>
__ESIMD_API std::enable_if_t<
__ESIMD_DNS::get_num_args<__ESIMD_DNS::to_lsc_atomic_op<Op>()>() == 2,
__ESIMD_NS::simd<T, N>>
lsc_atomic_update(T *p, __ESIMD_NS::simd<Toffset, N> offsets,
__ESIMD_NS::simd<T, N> src0, __ESIMD_NS::simd<T, N> src1,
__ESIMD_NS::simd_mask<N> pred) {
Expand Down Expand Up @@ -2466,7 +2480,9 @@ template <__ESIMD_NS::atomic_op Op, typename T, int N,
cache_hint L1H = cache_hint::none, cache_hint L3H = cache_hint::none,
typename Toffset,
typename RegionTy = __ESIMD_NS::region1d_t<Toffset, N, 1>>
__ESIMD_API __ESIMD_NS::simd<T, N>
__ESIMD_API std::enable_if_t<
__ESIMD_DNS::get_num_args<__ESIMD_DNS::to_lsc_atomic_op<Op>()>() == 2,
__ESIMD_NS::simd<T, N>>
lsc_atomic_update(T *p, __ESIMD_NS::simd_view<Toffset, RegionTy> offsets,
__ESIMD_NS::simd<T, N> src0, __ESIMD_NS::simd<T, N> src1,
__ESIMD_NS::simd_mask<N> pred = 1) {
Expand All @@ -2479,8 +2495,10 @@ template <__ESIMD_NS::atomic_op Op, typename T, int N,
lsc_data_size DS = lsc_data_size::default_size,
cache_hint L1H = cache_hint::none, cache_hint L3H = cache_hint::none,
typename Toffset>
__ESIMD_API
std::enable_if_t<std::is_integral_v<Toffset>, __ESIMD_NS::simd<T, N>>
__ESIMD_API std::enable_if_t<
std::is_integral_v<Toffset> &&
__ESIMD_DNS::get_num_args<__ESIMD_DNS::to_lsc_atomic_op<Op>()>() == 2,
__ESIMD_NS::simd<T, N>>
lsc_atomic_update(T *p, Toffset offset, __ESIMD_NS::simd<T, N> src0,
__ESIMD_NS::simd<T, N> src1,
__ESIMD_NS::simd_mask<N> pred = 1) {
Expand Down Expand Up @@ -2702,58 +2720,64 @@ namespace esimd {
/// <tt>native::lsc::atomic_op</tt> instead of <tt>atomic_op</tt> as atomic
/// operation template argument.
template <native::lsc::atomic_op Op, typename T, int N, typename Toffset>
__ESIMD_API simd<T, N> atomic_update(T *p, simd<Toffset, N> offset,
simd_mask<N> mask) {
__ESIMD_API std::enable_if_t<__ESIMD_DNS::get_num_args<Op>() == 0, simd<T, N>>
atomic_update(T *p, simd<Toffset, N> offset, simd_mask<N> mask) {
return __ESIMD_ENS::lsc_atomic_update<detail::to_atomic_op<Op>(), T, N>(
p, offset, mask);
}

template <native::lsc::atomic_op Op, typename T, int N, typename Toffset,
typename RegionTy = __ESIMD_NS::region1d_t<Toffset, N, 1>>
__ESIMD_API simd<T, N> atomic_update(T *p, simd_view<Toffset, RegionTy> offsets,
simd_mask<N> mask = 1) {
__ESIMD_API std::enable_if_t<__ESIMD_DNS::get_num_args<Op>() == 0, simd<T, N>>
atomic_update(T *p, simd_view<Toffset, RegionTy> offsets,
simd_mask<N> mask = 1) {
return __ESIMD_ENS::lsc_atomic_update<detail::to_atomic_op<Op>(), T, N>(
p, offsets, mask);
}

template <native::lsc::atomic_op Op, typename T, int N, typename Toffset>
__ESIMD_API
std::enable_if_t<std::is_integral_v<Toffset>, __ESIMD_NS::simd<T, N>>
atomic_update(T *p, Toffset offset, simd_mask<N> mask = 1) {
__ESIMD_API std::enable_if_t<std::is_integral_v<Toffset> &&
__ESIMD_DNS::get_num_args<Op>() == 0,
simd<T, N>>
atomic_update(T *p, Toffset offset, simd_mask<N> mask = 1) {
return __ESIMD_ENS::lsc_atomic_update<detail::to_atomic_op<Op>(), T, N>(
p, offset, mask);
}

/// LSC version of the single-argument atomic update.
template <native::lsc::atomic_op Op, typename T, int N, typename Toffset>
__ESIMD_API simd<T, N> atomic_update(T *p, simd<Toffset, N> offset,
simd<T, N> src0, simd_mask<N> mask) {
__ESIMD_API __ESIMD_API
std::enable_if_t<__ESIMD_DNS::get_num_args<Op>() == 1, simd<T, N>>
atomic_update(T *p, simd<Toffset, N> offset, simd<T, N> src0,
simd_mask<N> mask) {
return __ESIMD_ENS::lsc_atomic_update<detail::to_atomic_op<Op>(), T, N>(
p, offset, src0, mask);
}

template <native::lsc::atomic_op Op, typename T, int N, typename Toffset,
typename RegionTy = __ESIMD_NS::region1d_t<Toffset, N, 1>>
__ESIMD_API simd<T, N> atomic_update(T *p, simd_view<Toffset, RegionTy> offsets,
simd<T, N> src0, simd_mask<N> mask = 1) {
__ESIMD_API __ESIMD_API
std::enable_if_t<__ESIMD_DNS::get_num_args<Op>() == 1, simd<T, N>>
atomic_update(T *p, simd_view<Toffset, RegionTy> offsets, simd<T, N> src0,
simd_mask<N> mask = 1) {
return __ESIMD_ENS::lsc_atomic_update<detail::to_atomic_op<Op>(), T, N>(
p, offsets, src0, mask);
}

template <native::lsc::atomic_op Op, typename T, int N, typename Toffset>
__ESIMD_API
std::enable_if_t<std::is_integral_v<Toffset>, __ESIMD_NS::simd<T, N>>
atomic_update(T *p, Toffset offset, simd<T, N> src0,
simd_mask<N> mask = 1) {
__ESIMD_API std::enable_if_t<std::is_integral_v<Toffset> &&
__ESIMD_DNS::get_num_args<Op>() == 1,
simd<T, N>>
atomic_update(T *p, Toffset offset, simd<T, N> src0, simd_mask<N> mask = 1) {
return __ESIMD_ENS::lsc_atomic_update<detail::to_atomic_op<Op>(), T, N>(
p, offset, src0, mask);
}

/// LSC version of the two-argument atomic update.
template <native::lsc::atomic_op Op, typename T, int N, typename Toffset>
__ESIMD_API simd<T, N> atomic_update(T *p, simd<Toffset, N> offset,
simd<T, N> src0, simd<T, N> src1,
simd_mask<N> mask) {
__ESIMD_API std::enable_if_t<__ESIMD_DNS::get_num_args<Op>() == 2, simd<T, N>>
atomic_update(T *p, simd<Toffset, N> offset, simd<T, N> src0, simd<T, N> src1,
simd_mask<N> mask) {
// 2-argument lsc_atomic_update arguments order matches the standard one -
// expected value first, then new value. But atomic_update uses reverse
// order, hence the src1/src0 swap.
Expand All @@ -2763,18 +2787,19 @@ __ESIMD_API simd<T, N> atomic_update(T *p, simd<Toffset, N> offset,

template <native::lsc::atomic_op Op, typename T, int N, typename Toffset,
typename RegionTy = __ESIMD_NS::region1d_t<Toffset, N, 1>>
__ESIMD_API simd<T, N> atomic_update(T *p, simd_view<Toffset, RegionTy> offsets,
simd<T, N> src0, simd<T, N> src1,
simd_mask<N> mask = 1) {
__ESIMD_API std::enable_if_t<__ESIMD_DNS::get_num_args<Op>() == 2, simd<T, N>>
atomic_update(T *p, simd_view<Toffset, RegionTy> offsets, simd<T, N> src0,
simd<T, N> src1, simd_mask<N> mask = 1) {
return __ESIMD_ENS::lsc_atomic_update<detail::to_atomic_op<Op>(), T, N>(
p, offsets, src1, src0, mask);
}

template <native::lsc::atomic_op Op, typename T, int N, typename Toffset>
__ESIMD_API
std::enable_if_t<std::is_integral_v<Toffset>, __ESIMD_NS::simd<T, N>>
atomic_update(T *p, Toffset offset, simd<T, N> src0, simd<T, N> src1,
simd_mask<N> mask = 1) {
__ESIMD_API std::enable_if_t<std::is_integral_v<Toffset> &&
__ESIMD_DNS::get_num_args<Op>() == 2,
__ESIMD_NS::simd<T, N>>
atomic_update(T *p, Toffset offset, simd<T, N> src0, simd<T, N> src1,
simd_mask<N> mask = 1) {
return __ESIMD_ENS::lsc_atomic_update<detail::to_atomic_op<Op>(), T, N>(
p, offset, src1, src0, mask);
}
Expand Down