Re-implement Intel cache controls decoration (#2587)

The patch adds CacheControls(Load/Store)INTEL decorations representation as metadata placed on memory accessing instruction with the following form: !spirv.DecorationCacheControlINTEL !X !X = !{i32 %decoration_kind%, i32 %level%, i32 %control%, i32 %operand of the instruction to decorate%} Also patch creates a dummy GEP accessing pointer operand of the instruction to perform SSA copy of the pointer and attaches !spirv.Decorations metadata to the GEP. Few notes about this implementation and other options. It is sub-optimal as it requires iteration over all instructions in the module. Alternatives are: 1. In SPIRVWriter during transDecorations rewrite already translated instruction (lets say load) with the new one, but with GEP operand. But while the translator provides API to erase instructions and rewriting SPIRVBasicBlocks, yet unfortunately it's not really usable outside of of SPIRVModule context; 2. In SPIRVWriter during transValueWithoutDecoration add special handling of the instructions with spirv.DecorationCacheControlINTEL metadata. And it would work with one exception - it also would require to unprivate some methods of SPIRVScavenger to create GEP with the correct type, which I'm not sure if it's a good idea. Note, in this implementation in the worst case scenario SPIRVScavenger would create a bitcast later during translation. Signed-off-by: Sidorov, Dmitry <dmitry.sidorov@intel.com>
KhronosGroup · May 28, 2024 · c8bfc33 · c8bfc33
1 parent 0fd9882
commit c8bfc33
Show file tree

Hide file tree

Showing 4 changed files with 172 additions and 0 deletions.
diff --git a/lib/SPIRV/SPIRVInternal.h b/lib/SPIRV/SPIRVInternal.h
@@ -271,6 +271,7 @@ typedef SPIRVMap<SPIRVExtInstSetKind, std::string, SPIRVExtSetShortName>
 
 #define SPIRV_MD_PARAMETER_DECORATIONS "spirv.ParameterDecorations"
 #define SPIRV_MD_DECORATIONS "spirv.Decorations"
+#define SPIRV_MD_INTEL_CACHE_DECORATIONS "spirv.DecorationCacheControlINTEL"
 
 #define OCL_TYPE_NAME_SAMPLER_T "sampler_t"
 #define SPIR_TYPE_NAME_EVENT_T "opencl.event_t"

diff --git a/lib/SPIRV/SPIRVRegularizeLLVM.cpp b/lib/SPIRV/SPIRVRegularizeLLVM.cpp
@@ -495,6 +495,65 @@ void regularizeWithOverflowInstrinsics(StringRef MangledName, CallInst *Call,
   }
   ToErase.push_back(Call);
 }
+
+// CacheControls(Load/Store)INTEL decorations can be represented as metadata
+// placed on memory accessing instruction with the following form:
+// !spirv.DecorationCacheControlINTEL !X
+// !X = !{i32 %decoration_kind%, i32 %level%, i32 %control%,
+//        i32 %operand of the instruction to decorate%}
+// This function creates a dummy GEP accessing pointer operand of the
+// instruction and creates !spirv.Decorations metadata attached to it.
+void prepareCacheControlsTranslation(Metadata *MD, Instruction *Inst) {
+  if (!Inst->mayReadOrWriteMemory())
+    return;
+  auto *ArgDecoMD = dyn_cast<MDNode>(MD);
+  assert(ArgDecoMD && "Decoration list must be a metadata node");
+  for (unsigned I = 0, E = ArgDecoMD->getNumOperands(); I != E; ++I) {
+    auto *DecoMD = dyn_cast<MDNode>(ArgDecoMD->getOperand(I));
+    if (!DecoMD) {
+      assert(!"Decoration does not name metadata");
+      return;
+    }
+
+    constexpr size_t CacheControlsNumOps = 4;
+    if (DecoMD->getNumOperands() != CacheControlsNumOps) {
+      assert(!"Cache controls metadata on instruction must have 4 operands");
+      return;
+    }
+
+    auto *const KindMD = cast<ConstantAsMetadata>(DecoMD->getOperand(0));
+    auto *const LevelMD = cast<ConstantAsMetadata>(DecoMD->getOperand(1));
+    auto *const ControlMD = cast<ConstantAsMetadata>(DecoMD->getOperand(2));
+
+    const size_t TargetArgNo =
+        mdconst::dyn_extract<ConstantInt>(DecoMD->getOperand(3))
+            ->getZExtValue();
+    Value *PtrInstOp = Inst->getOperand(TargetArgNo);
+    if (!PtrInstOp->getType()->isPointerTy()) {
+      assert(!"Cache controls must decorate a pointer");
+      return;
+    }
+
+    // Create dummy GEP for SSA copy of the pointer operand. Lets do our best
+    // to guess pointee type here, but if we won't - just pointer is also fine,
+    // if necessary TypeScavenger will adjust types and create bitcasts.
+    IRBuilder Builder(Inst);
+    Type *GEPTy = PtrInstOp->getType();
+    if (auto *LI = dyn_cast<LoadInst>(Inst))
+      GEPTy = LI->getType();
+    else if (auto *SI = dyn_cast<StoreInst>(Inst))
+      GEPTy = SI->getValueOperand()->getType();
+    auto *GEP =
+        cast<Instruction>(Builder.CreateConstGEP1_32(GEPTy, PtrInstOp, 0));
+    Inst->setOperand(TargetArgNo, GEP);
+
+    SmallVector<Metadata *, 4> MDs;
+    std::vector<Metadata *> OPs = {KindMD, LevelMD, ControlMD};
+    MDs.push_back(MDNode::get(Inst->getContext(), OPs));
+    MDNode *MDList = MDNode::get(Inst->getContext(), MDs);
+    GEP->setMetadata(SPIRV_MD_DECORATIONS, MDList);
+  }
+}
 } // namespace
 
 /// Remove entities not representable by SPIR-V
@@ -511,9 +570,12 @@ bool SPIRVRegularizeLLVMBase::regularize() {
       continue;
     }
 
+    // TODO: query intrinsic calls from their declarations
     std::vector<Instruction *> ToErase;
     for (BasicBlock &BB : *F) {
       for (Instruction &II : BB) {
+        if (auto *MD = II.getMetadata(SPIRV_MD_INTEL_CACHE_DECORATIONS))
+          prepareCacheControlsTranslation(MD, &II);
         if (auto *Call = dyn_cast<CallInst>(&II)) {
           Call->setTailCall(false);
           Function *CF = Call->getCalledFunction();

diff --git a/test/extensions/INTEL/SPV_INTEL_cache_controls/basic_load_store_arg_dec.ll b/test/extensions/INTEL/SPV_INTEL_cache_controls/basic_load_store_arg_dec.ll
@@ -0,0 +1,66 @@
+; RUN: llvm-as %s -o %t.bc
+; RUN: llvm-spirv --spirv-ext=+SPV_INTEL_cache_controls -spirv-text %t.bc -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+; RUN: llvm-spirv --spirv-ext=+SPV_INTEL_cache_controls %t.bc -o %t.spv
+; RUN: llvm-spirv -r %t.spv --spirv-target-env=SPV-IR -o - | llvm-dis -o - | FileCheck %s --check-prefix=CHECK-LLVM
+
+; CHECK-SPIRV-DAG: TypeInt [[#Int32:]] 32 0
+; CHECK-SPIRV-DAG: Constant [[#Int32]] [[#Zero:]] 0
+; CHECK-SPIRV-DAG: Decorate [[#Load1GEPPtr:]] CacheControlLoadINTEL 0 1
+; CHECK-SPIRV-DAG: Decorate [[#Load2GEPPtr:]] CacheControlLoadINTEL 1 1
+; CHECK-SPIRV-DAG: Decorate [[#Store1GEPPtr:]] CacheControlStoreINTEL 0 1
+; CHECK-SPIRV-DAG: Decorate [[#Store2GEPPtr:]] CacheControlStoreINTEL 1 1
+
+; CHECK-SPIRV: FunctionParameter [[#]] [[#Buffer:]]
+; CHECK-SPIRV: PtrAccessChain [[#]] [[#Load1GEPPtr:]] [[#Buffer]] [[#Zero]]
+; CHECK-SPIRV: Load [[#]] [[#]] [[#Load1GEPPtr]]
+; CHECK-SPIRV: PtrAccessChain [[#]] [[#Load2GEPPtr:]] [[#]] [[#Zero]]
+; CHECK-SPIRV: Load [[#]] [[#]] [[#Load2GEPPtr]]
+; CHECK-SPIRV: PtrAccessChain [[#]] [[#Store1GEPPtr:]] [[#]] [[#Zero]]
+; CHECK-SPIRV: Store [[#Store1GEPPtr]]
+; CHECK-SPIRV: PtrAccessChain [[#]] [[#Store2GEPPtr:]] [[#]] [[#Zero]]
+; CHECK-SPIRV: Store [[#Store2GEPPtr]]
+
+; CHECK-LLVM: %[[#GEPLoad1:]] = getelementptr i32, ptr addrspace(1) %{{.*}}, i32 0, !spirv.Decorations ![[#Cache1:]]
+; CHECK-LLVM: load i32, ptr addrspace(1) %[[#GEPLoad1]], align 4
+; CHECK-LLVM: %[[#GEPLoad2:]] = getelementptr i32, ptr addrspace(1) %{{.*}}, i32 0, !spirv.Decorations ![[#Cache2:]]
+; CHECK-LLVM: load i32, ptr addrspace(1) %[[#GEPLoad2]], align 4
+; CHECK-LLVM: %[[#GEPStore1:]] = getelementptr i32, ptr addrspace(1) %{{.*}}, i32 0, !spirv.Decorations ![[#Cache3:]]
+; CHECK-LLVM: store i32 %[[#]], ptr addrspace(1) %[[#GEPStore1]], align 4
+; CHECK-LLVM: %[[#GEPStore2:]] = getelementptr i32, ptr addrspace(1) %{{.*}}, i32 0, !spirv.Decorations ![[#Cache4:]]
+; CHECK-LLVM: store i32 %[[#]], ptr addrspace(1) %[[#GEPStore2]], align 4
+; CHECK-LLVM: ![[#Cache1]] = !{![[#DecLoad1:]]}
+; CHECK-LLVM: ![[#DecLoad1]] = !{i32 6442, i32 0, i32 1}
+; CHECK-LLVM: ![[#Cache2]] = !{![[#DecLoad2:]]}
+; CHECK-LLVM: ![[#DecLoad2]] = !{i32 6442, i32 1, i32 1}
+; CHECK-LLVM: ![[#Cache3:]] = !{![[#DecStore1:]]}
+; CHECK-LLVM: ![[#DecStore1]] = !{i32 6443, i32 0, i32 1}
+; CHECK-LLVM: ![[#Cache4:]] = !{![[#DecStore2:]]}
+; CHECK-LLVM: ![[#DecStore2]] = !{i32 6443, i32 1, i32 1}
+
+target triple = "spir64-unknown-unknown"
+
+define spir_kernel void @test(ptr addrspace(1) %buffer) {
+entry:
+  %0 = load i32, ptr addrspace(1) %buffer, align 4, !spirv.DecorationCacheControlINTEL !3
+  %1 = load i32, ptr addrspace(1) %buffer, align 4, !spirv.DecorationCacheControlINTEL !5
+  store i32 %0, ptr addrspace(1) %buffer, align 4, !spirv.DecorationCacheControlINTEL !7
+  store i32 %1, ptr addrspace(1) %buffer, align 4, !spirv.DecorationCacheControlINTEL !9
+  ret void
+}
+
+!spirv.MemoryModel = !{!0}
+!spirv.Source = !{!1}
+!opencl.spir.version = !{!2}
+!opencl.ocl.version = !{!2}
+
+!0 = !{i32 2, i32 2}
+!1 = !{i32 3, i32 102000}
+!2 = !{i32 1, i32 2}
+!3 = !{!4}
+!4 = !{i32 6442, i32 0, i32 1, i32 0}
+!5 = !{!6}
+!6 = !{i32 6442, i32 1, i32 1, i32 0}
+!7 = !{!8}
+!8 = !{i32 6443, i32 0, i32 1, i32 1}
+!9 = !{!10}
+!10 = !{i32 6443, i32 1, i32 1, i32 1}
diff --git a/test/extensions/INTEL/SPV_INTEL_cache_controls/call_inst_arg_dec.ll b/test/extensions/INTEL/SPV_INTEL_cache_controls/call_inst_arg_dec.ll
@@ -0,0 +1,43 @@
+; RUN: llvm-as %s -o %t.bc
+; RUN: llvm-spirv --spirv-ext=+SPV_INTEL_cache_controls -spirv-text %t.bc -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+; RUN: llvm-spirv --spirv-ext=+SPV_INTEL_cache_controls %t.bc -o %t.spv
+; RUN: llvm-spirv -r %t.spv --spirv-target-env=SPV-IR -o - | llvm-dis -o - | FileCheck %s --check-prefix=CHECK-LLVM
+
+; CHECK-SPIRV: Decorate [[#GEP1:]] CacheControlLoadINTEL 0 1
+; CHECK-SPIRV: Decorate [[#GEP2:]] CacheControlStoreINTEL 0 1
+; CHECK-SPIRV: TypeInt [[#Int32Ty:]] 32 0
+; CHECK-SPIRV: Constant [[#Int32Ty]] [[#Zero:]] 0
+; CHECK-SPIRV: PtrAccessChain [[#]] [[#GEP1]] [[#]] [[#Zero]]
+; CHECK-SPIRV: PtrAccessChain [[#]] [[#GEP2]] [[#]] [[#Zero]]
+; CHECK-SPIRV: FunctionCall [[#]] [[#]] [[#]] [[#GEP1]] [[#GEP2]]
+
+; CHECK-LLVM: %[[#GEP1:]] = getelementptr ptr addrspace(1), ptr addrspace(1) %{{.*}}, i32 0, !spirv.Decorations ![[#Cache1:]]
+; CHECK-LLVM: %[[#GEP2:]] = getelementptr ptr addrspace(1), ptr addrspace(1) %{{.*}}, i32 0, !spirv.Decorations ![[#Cache2:]]
+; CHECK-LLVM: call spir_func void @foo(ptr addrspace(1) %[[#GEP1]], ptr addrspace(1) %[[#GEP2]])
+; CHECK-LLVM: ![[#Cache1]] = !{![[#LoadCache:]]}
+; CHECK-LLVM: ![[#LoadCache]] = !{i32 6442, i32 0, i32 1}
+; CHECK-LLVM: ![[#Cache2]] = !{![[#StoreCache:]]}
+; CHECK-LLVM: ![[#StoreCache]] = !{i32 6443, i32 0, i32 1}
+
+target triple = "spir64-unknown-unknown"
+
+define spir_kernel void @test(ptr addrspace(1) %buffer1, ptr addrspace(1) %buffer2) {
+entry:
+  call void @foo(ptr addrspace(1) %buffer1, ptr addrspace(1) %buffer2), !spirv.DecorationCacheControlINTEL !3
+  ret void
+}
+
+declare void @foo(ptr addrspace(1), ptr addrspace(1))
+
+!spirv.MemoryModel = !{!0}
+!spirv.Source = !{!1}
+!opencl.spir.version = !{!2}
+!opencl.ocl.version = !{!2}
+
+!0 = !{i32 2, i32 2}
+!1 = !{i32 3, i32 102000}
+!2 = !{i32 1, i32 2}
+!3 = !{!4, !5}
+!4 = !{i32 6442, i32 0, i32 1, i32 0}
+!5 = !{i32 6443, i32 0, i32 1, i32 1}
+