Translate atomicrmw LLVM instruction to SPIR-V

atomicrmw with nand, fadd and fsub operation can't be represented in
SPIR-V.

Signed-off-by: Alexey Sotkin <alexey.sotkin@intel.com>

Author: AlexeySotkin

llvm-spirv/lib/SPIRV/OCLUtil.h

@@ -42,6 +42,7 @@
 #include "SPIRVInternal.h"
 #include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Instructions.h"
 #include "llvm/Support/Path.h"
 
 #include <functional>
@@ -984,6 +985,20 @@ template <> inline void SPIRVMap<std::string, Op, OCLOpaqueType>::init() {
   add("opencl.sampler_t", OpTypeSampler);
 }
 
+typedef SPIRVMap<AtomicRMWInst::BinOp, Op> LLVMSPIRVAtomicRmwOpCodeMap;
+template <> inline void LLVMSPIRVAtomicRmwOpCodeMap::init() {
+  add(llvm::AtomicRMWInst::Xchg, OpAtomicExchange);
+  add(llvm::AtomicRMWInst::Add, OpAtomicIAdd);
+  add(llvm::AtomicRMWInst::Sub, OpAtomicISub);
+  add(llvm::AtomicRMWInst::And, OpAtomicAnd);
+  add(llvm::AtomicRMWInst::Or, OpAtomicOr);
+  add(llvm::AtomicRMWInst::Xor, OpAtomicXor);
+  add(llvm::AtomicRMWInst::Max, OpAtomicSMax);
+  add(llvm::AtomicRMWInst::Min, OpAtomicSMin);
+  add(llvm::AtomicRMWInst::UMax, OpAtomicUMax);
+  add(llvm::AtomicRMWInst::UMin, OpAtomicUMin);
+}
+
 } // namespace SPIRV
 
 #endif // SPIRV_OCLUTIL_H

llvm-spirv/lib/SPIRV/SPIRVWriter.cpp

@@ -1223,6 +1223,36 @@ SPIRVValue *LLVMToSPIRV::transValueWithoutDecoration(Value *V,
                            transValue(SF->getOperand(1), BB), Comp, BB));
   }
 
+  if (AtomicRMWInst *ARMW = dyn_cast<AtomicRMWInst>(V)) {
+    AtomicRMWInst::BinOp Op = ARMW->getOperation();
+    if (!BM->getErrorLog().checkError(
+            !AtomicRMWInst::isFPOperation(Op) && Op != AtomicRMWInst::Nand,
+            SPIRVEC_InvalidInstruction,
+            OCLUtil::toString(V) + "\nAtomic " +
+                AtomicRMWInst::getOperationName(Op).str() +
+                " is not supported in SPIR-V!\n"))
+      return nullptr;
+
+    spv::Op OC = LLVMSPIRVAtomicRmwOpCodeMap::map(Op);
+    AtomicOrderingCABI Ordering = llvm::toCABI(ARMW->getOrdering());
+    auto MemSem = OCLMemOrderMap::map(static_cast<OCLMemOrderKind>(Ordering));
+    std::vector<Value *> Operands(4);
+    Operands[0] = ARMW->getPointerOperand();
+    // To get the memory scope argument we might use ARMW->getSyncScopeID(), but
+    // atomicrmw LLVM instruction is not aware of OpenCL(or SPIR-V) memory scope
+    // enumeration. And assuming the produced SPIR-V module will be consumed in
+    // an OpenCL environment, we can use the same memory scope as OpenCL atomic
+    // functions that don't have memory_scope argument i.e. memory_scope_device.
+    // See the OpenCL C specification p6.13.11. "Atomic Functions"
+    Operands[1] = getUInt32(M, spv::ScopeDevice);
+    Operands[2] = getUInt32(M, MemSem);
+    Operands[3] = ARMW->getValOperand();
+    std::vector<SPIRVId> Ops = BM->getIds(transValue(Operands, BB));
+    SPIRVType *Ty = transType(ARMW->getType());
+
+    return mapValue(V, BM->addInstTemplate(OC, Ops, BB, Ty));
+  }
+
   if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(V)) {
     SPIRVValue *BV = transIntrinsicInst(II, BB);
     return BV ? mapValue(V, BV) : nullptr;

llvm-spirv/lib/SPIRV/libSPIRV/SPIRVErrorEnum.h

@@ -12,3 +12,4 @@ _SPIRV_OP(InvalidBitWidth, "Invalid bit width in input:")
 _SPIRV_OP(InvalidModule, "Invalid SPIR-V module:")
 _SPIRV_OP(UnimplementedOpCode, "Unimplemented opcode")
 _SPIRV_OP(FunctionPointers, "Can't translate function pointer:\n")
+_SPIRV_OP(InvalidInstruction, "Can't translate llvm instruction:\n")

llvm-spirv/test/atomicrmw.ll

@@ -0,0 +1,70 @@
+; RUN: llvm-as < %s -o %t.bc
+; RUN: llvm-spirv %t.bc -o %t.spv
+; RUN: spirv-val %t.spv
+; RUN: llvm-spirv -to-text %t.spv -o - | FileCheck %s
+
+; CHECK: TypeInt [[Int:[0-9]+]] 32 0
+; CHECK-DAG: Constant [[Int]] [[Scope_Device:[0-9]+]] 1 {{$}}
+; CHECK-DAG: Constant [[Int]] [[MemSem_Relaxed:[0-9]+]] 0
+; CHECK-DAG: Constant [[Int]] [[MemSem_Acquire:[0-9]+]] 2
+; CHECK-DAG: Constant [[Int]] [[MemSem_Release:[0-9]+]] 4 {{$}}
+; CHECK-DAG: Constant [[Int]] [[MemSem_AcquireRelease:[0-9]+]] 8
+; CHECK-DAG: Constant [[Int]] [[MemSem_SequentiallyConsistent:[0-9]+]] 16
+; CHECK-DAG: Constant [[Int]] [[Value:[0-9]+]] 42
+; CHECK: TypeFloat [[Float:[0-9]+]] 32
+; CHECK: Variable {{[0-9]+}} [[Pointer:[0-9]+]]
+; CHECK: Variable {{[0-9]+}} [[FPPointer:[0-9]+]]
+; CHECK: Constant [[Float]] [[FPValue:[0-9]+]] 1109917696
+
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024"
+target triple = "spir64"
+
+@ui = common dso_local addrspace(1) global i32 0, align 4
+@f = common dso_local local_unnamed_addr addrspace(1) global float 0.000000e+00, align 4
+
+; Function Attrs: nounwind
+define dso_local spir_func void @test_atomicrmw() local_unnamed_addr #0 {
+entry:
+  %0 = atomicrmw xchg i32 addrspace(1)* @ui, i32 42 acq_rel
+; CHECK: AtomicExchange [[Int]] {{[0-9]+}} [[Pointer]] [[Scope_Device]] [[MemSem_AcquireRelease]] [[Value]]
+
+  %1 = atomicrmw xchg float addrspace(1)* @f, float 42.000000e+00 seq_cst
+; CHECK: AtomicExchange [[Float]] {{[0-9]+}} [[FPPointer]] [[Scope_Device]] [[MemSem_SequentiallyConsistent]] [[FPValue]]
+
+  %2 = atomicrmw add i32 addrspace(1)* @ui, i32 42 monotonic
+; CHECK: AtomicIAdd [[Int]] {{[0-9]+}} [[Pointer]] [[Scope_Device]] [[MemSem_Relaxed]] [[Value]]
+
+  %3 = atomicrmw sub i32 addrspace(1)* @ui, i32 42 acquire
+; CHECK: AtomicISub [[Int]] {{[0-9]+}} [[Pointer]] [[Scope_Device]] [[MemSem_Acquire]] [[Value]]
+
+  %4 = atomicrmw or i32 addrspace(1)* @ui, i32 42 release
+; CHECK: AtomicOr [[Int]] {{[0-9]+}} [[Pointer]] [[Scope_Device]] [[MemSem_Release]] [[Value]]
+
+  %5 = atomicrmw xor i32 addrspace(1)* @ui, i32 42 acq_rel
+; CHECK: AtomicXor [[Int]] {{[0-9]+}} [[Pointer]] [[Scope_Device]] [[MemSem_AcquireRelease]] [[Value]]
+
+  %6 = atomicrmw and i32 addrspace(1)* @ui, i32 42 seq_cst
+; CHECK: AtomicAnd [[Int]] {{[0-9]+}} [[Pointer]] [[Scope_Device]] [[MemSem_SequentiallyConsistent]] [[Value]]
+
+  %7 = atomicrmw max i32 addrspace(1)* @ui, i32 42 monotonic
+; CHECK: AtomicSMax [[Int]] {{[0-9]+}} [[Pointer]] [[Scope_Device]] [[MemSem_Relaxed]] [[Value]]
+
+  %8 = atomicrmw min i32 addrspace(1)* @ui, i32 42 acquire
+; CHECK: AtomicSMin [[Int]] {{[0-9]+}} [[Pointer]] [[Scope_Device]] [[MemSem_Acquire]] [[Value]]
+
+  %9 = atomicrmw umax i32 addrspace(1)* @ui, i32 42 release
+; CHECK: AtomicUMax [[Int]] {{[0-9]+}} [[Pointer]] [[Scope_Device]] [[MemSem_Release]] [[Value]]
+
+  %10 = atomicrmw umin i32 addrspace(1)* @ui, i32 42 acq_rel
+; CHECK: AtomicUMin [[Int]] {{[0-9]+}} [[Pointer]] [[Scope_Device]] [[MemSem_AcquireRelease]] [[Value]]
+
+  ret void
+}
+
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 11.0.0 (https://github.com/llvm/llvm-project.git 20c5968e0953d978be4d9d1062801e8758c393b5)"}

llvm-spirv/test/negative/atomicrmw-unsupported-operation.ll

@@ -0,0 +1,29 @@
+; RUN: llvm-as < %s -o %t.bc
+; RUN: not --crash llvm-spirv %t.bc -o %t.spv 2>&1 | FileCheck %s
+
+; CHECK: InvalidInstruction: Can't translate llvm instruction:
+; CHECK: atomicrmw nand i32 addrspace(1)* @ui, i32 42 acq_rel
+; CHECK: Atomic nand is not supported in SPIR-V!
+
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024"
+target triple = "spir64"
+
+@ui = common dso_local addrspace(1) global i32 0, align 4
+@f = common dso_local local_unnamed_addr addrspace(1) global float 0.000000e+00, align 4
+
+; Function Attrs: nounwind
+define dso_local spir_func void @test_atomicrmw() local_unnamed_addr #0 {
+entry:
+  %0 = atomicrmw nand i32 addrspace(1)* @ui, i32 42 acq_rel
+  %1 = atomicrmw fadd float addrspace(1)* @f, float 42.000000e+00 seq_cst
+  %2 = atomicrmw fsub float addrspace(1)* @f, float 42.000000e+00 seq_cst
+  ret void
+}
+
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 11.0.0 (https://github.com/llvm/llvm-project.git 20c5968e0953d978be4d9d1062801e8758c393b5)"}