Implement SPV_INTEL_memory_access_aliasing extension

* Implement SPV_INTEL_memory_access_aliasing extension

This extension adds analogs of alias.scope and noalias metadata
in SPIR-V. These metadata is being used to annotate non-aliasing
memory accesses and being generated, for example, during inlining
of functions with restrict pointer parameters.

Spec: #3426

This patch also casually adds OpDecorateId in the translator flow.

Signed-off-by: Dmitry Sidorov <dmitry.sidorov@intel.com>

Original commit:
KhronosGroup/SPIRV-LLVM-Translator@974749c

Author: MrSidims

llvm-spirv/include/LLVMSPIRVExtensions.inc

@@ -35,3 +35,4 @@ EXT(SPV_INTEL_fpga_cluster_attributes)
 EXT(SPV_INTEL_loop_fuse)
 EXT(SPV_INTEL_long_constant_composite)
 EXT(SPV_INTEL_optnone)
+EXT(SPV_INTEL_memory_access_aliasing)

llvm-spirv/lib/SPIRV/SPIRVReader.cpp

@@ -45,6 +45,7 @@
 #include "SPIRVInstruction.h"
 #include "SPIRVInternal.h"
 #include "SPIRVMDBuilder.h"
+#include "SPIRVMemAliasingINTEL.h"
 #include "SPIRVModule.h"
 #include "SPIRVToLLVMDbgTran.h"
 #include "SPIRVType.h"
@@ -64,6 +65,7 @@
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/LegacyPassManager.h"
+#include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Type.h"
@@ -1510,6 +1512,58 @@ static void replaceOperandWithAnnotationIntrinsicCallResult(Value *&V) {
   }
 }
 
+// Translate aliasing memory access masks for SPIRVLoad and SPIRVStore
+// instructions. These masks are mapped on alias.scope and noalias
+// metadata in LLVM. Translation of optional string operand isn't yet supported
+// in the translator.
+template <typename SPIRVInstType>
+void SPIRVToLLVM::transAliasingMemAccess(SPIRVInstType *BI, Instruction *I) {
+  static_assert(std::is_same<SPIRVInstType, SPIRVStore>::value ||
+                std::is_same<SPIRVInstType, SPIRVLoad>::value,
+                "Only stores and loads can be aliased by memory access mask");
+  bool IsAliasScope = BI->SPIRVMemoryAccess::isAliasScope();
+  bool IsNoAlias = BI->SPIRVMemoryAccess::isNoAlias();
+  if (!(IsAliasScope || IsNoAlias))
+    return;
+  uint32_t AliasMDKind = IsAliasScope ? LLVMContext::MD_alias_scope
+                                      : LLVMContext::MD_noalias;
+  SPIRVId AliasListId = BI->SPIRVMemoryAccess::getAliasing();
+  addMemAliasMetadata(I, AliasListId, AliasMDKind);
+}
+
+// Create and apply alias.scope/noalias metadata
+void SPIRVToLLVM::addMemAliasMetadata(Instruction *I, SPIRVId AliasListId,
+                                      uint32_t AliasMDKind) {
+  SPIRVAliasScopeListDeclINTEL *AliasList =
+      BM->get<SPIRVAliasScopeListDeclINTEL>(AliasListId);
+  std::vector<SPIRVId> AliasScopeIds = AliasList->getArguments();
+  MDBuilder MDB(*Context);
+  SmallVector<Metadata *, 4> MDScopes;
+  for (const auto ScopeId : AliasScopeIds) {
+    SPIRVAliasScopeDeclINTEL *AliasScope =
+        BM->get<SPIRVAliasScopeDeclINTEL>(ScopeId);
+    std::vector<SPIRVId> AliasDomainIds = AliasScope->getArguments();
+    // Currently we expect exactly one argument for aliasing scope
+    // instruction.
+    // TODO: add translation of string scope and domain operand.
+    assert(AliasDomainIds.size() == 1 &&
+           "AliasScopeDeclINTEL must have exactly one argument");
+    SPIRVId AliasDomainId = AliasDomainIds[0];
+    // Create and store unique domain and scope metadata
+    MDAliasDomainMap.emplace(AliasDomainId,
+                             MDB.createAnonymousAliasScopeDomain());
+    MDAliasScopeMap.emplace(ScopeId, MDB.createAnonymousAliasScope(
+                                         MDAliasDomainMap[AliasDomainId]));
+    MDScopes.emplace_back(MDAliasScopeMap[ScopeId]);
+  }
+  // Create and store unique alias.scope/noalias metadata
+  MDAliasListMap.emplace(
+      AliasListId,
+      MDNode::concatenate(I->getMetadata(LLVMContext::MD_alias_scope),
+                          MDNode::get(*Context, MDScopes)));
+  I->setMetadata(AliasMDKind, MDAliasListMap[AliasListId]);
+}
+
 /// For instructions, this function assumes they are created in order
 /// and appended to the given basic block. An instruction may use a
 /// instruction from another BB which has not been translated. Such
@@ -1907,6 +1961,7 @@ Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *BV, Function *F,
       SI = new StoreInst(Src, Dst, isVolatile, Align(AlignValue), BB);
     if (BS->SPIRVMemoryAccess::isNonTemporal())
       transNonTemporalMetadata(SI);
+    transAliasingMemAccess<SPIRVStore>(BS, SI);
     return mapValue(BV, SI);
   }
 
@@ -1929,6 +1984,7 @@ Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *BV, Function *F,
     }
     if (BL->SPIRVMemoryAccess::isNonTemporal())
       transNonTemporalMetadata(LI);
+    transAliasingMemAccess<SPIRVLoad>(BL, LI);
     return mapValue(BV, LI);
   }
 
@@ -3749,6 +3805,32 @@ void SPIRVToLLVM::transIntelFPGADecorations(SPIRVValue *BV, Value *V) {
   }
 }
 
+// Translate aliasing decorations applied to instructions. These decorations
+// are mapped on alias.scope and noalias metadata in LLVM. Translation of
+// optional string operand isn't yet supported in the translator.
+void SPIRVToLLVM::transMemAliasingINTELDecorations(SPIRVValue *BV, Value *V) {
+  if (!BV->isInst())
+    return;
+  Instruction *Inst = dyn_cast<Instruction>(V);
+  if (!Inst)
+    return;
+  std::vector<SPIRVId> AliasListIds;
+  uint32_t AliasMDKind;
+  if (BV->hasDecorateId(internal::DecorationAliasScopeINTEL)) {
+    AliasMDKind = LLVMContext::MD_alias_scope;
+    AliasListIds =
+        BV->getDecorationIdLiterals(internal::DecorationAliasScopeINTEL);
+  } else if (BV->hasDecorateId(internal::DecorationNoAliasINTEL)) {
+    AliasMDKind = LLVMContext::MD_noalias;
+    AliasListIds =
+        BV->getDecorationIdLiterals(internal::DecorationNoAliasINTEL);
+  } else
+    return;
+  assert(AliasListIds.size() == 1 &&
+         "Memory aliasing decorations must have one argument");
+  addMemAliasMetadata(Inst, AliasListIds[0], AliasMDKind);
+}
+
 // Having UserSemantic decoration on Function is against the spec, but we allow
 // this for various purposes (like prototyping new features when we need to
 // attach some information on function and propagate that through SPIR-V and
@@ -3800,6 +3882,7 @@ bool SPIRVToLLVM::transDecoration(SPIRVValue *BV, Value *V) {
     return false;
 
   transIntelFPGADecorations(BV, V);
+  transMemAliasingINTELDecorations(BV, V);
 
   DbgTran->transDbgInfo(BV, V);
   return true;

llvm-spirv/lib/SPIRV/SPIRVReader.h

@@ -115,6 +115,10 @@ class SPIRVToLLVM {
   CallInst *transArbFloatInst(SPIRVInstruction *BI, BasicBlock *BB,
                               bool IsBinaryInst = false);
   bool transNonTemporalMetadata(Instruction *I);
+  template <typename SPIRVInstType>
+  void transAliasingMemAccess(SPIRVInstType *BI, Instruction *I);
+  void addMemAliasMetadata(Instruction *I, SPIRVId AliasListId,
+                           uint32_t AliasMDKind);
   void transSourceLanguage();
   bool transSourceExtension();
   void transGeneratorMD();
@@ -183,6 +187,7 @@ class SPIRVToLLVM {
   typedef DenseMap<SPIRVValue *, Value *> SPIRVBlockToLLVMStructMap;
   typedef DenseMap<SPIRVFunction *, Function *> SPIRVToLLVMFunctionMap;
   typedef DenseMap<GlobalVariable *, SPIRVBuiltinVariableKind> BuiltinVarMap;
+  typedef std::unordered_map<SPIRVId, MDNode *> SPIRVToLLVMMDAliasInstMap;
 
   // A SPIRV value may be translated to a load instruction of a placeholder
   // global variable. This map records load instruction of these placeholders
@@ -210,6 +215,12 @@ class SPIRVToLLVM {
   // metadata SPIR-V instruction in SPIR-V representation of this basic block.
   SPIRVToLLVMLoopMetadataMap FuncLoopMetadataMap;
 
+  // These storages are used to prevent duplication of alias.scope/noalias
+  // metadata
+  SPIRVToLLVMMDAliasInstMap MDAliasDomainMap;
+  SPIRVToLLVMMDAliasInstMap MDAliasScopeMap;
+  SPIRVToLLVMMDAliasInstMap MDAliasListMap;
+
   Type *mapType(SPIRVType *BT, Type *T);
 
   // If a value is mapped twice, the existing mapped value is a placeholder,
@@ -282,6 +293,7 @@ class SPIRVToLLVM {
   void createCXXStructor(const char *ListName,
                          SmallVectorImpl<Function *> &Funcs);
   void transIntelFPGADecorations(SPIRVValue *BV, Value *V);
+  void transMemAliasingINTELDecorations(SPIRVValue *BV, Value *V);
 }; // class SPIRVToLLVM
 
 } // namespace SPIRV

llvm-spirv/lib/SPIRV/SPIRVWriter.cpp

@@ -50,6 +50,7 @@
 #include "SPIRVInternal.h"
 #include "SPIRVLLVMUtil.h"
 #include "SPIRVMDWalker.h"
+#include "SPIRVMemAliasingINTEL.h"
 #include "SPIRVModule.h"
 #include "SPIRVType.h"
 #include "SPIRVUtil.h"
@@ -1228,6 +1229,46 @@ SPIRVValue *LLVMToSPIRV::transAtomicLoad(LoadInst *LD, SPIRVBasicBlock *BB) {
                                           BB, transType(LD->getType())));
 }
 
+// Aliasing list MD contains several scope MD nodes whithin it. Each scope MD
+// has a selfreference and an extra MD node for aliasing domain and also it
+// can contain an optional string operand. Domain MD contains a self-reference
+// with an optional string operand. Here we unfold the list, creating SPIR-V
+// aliasing instructions.
+// TODO: add support for an optional string operand.
+SPIRVEntry *addMemAliasingINTELInstructions(SPIRVModule *M,
+                                            MDNode *AliasingListMD) {
+  assert(AliasingListMD->getNumOperands() > 0 &&
+         "Aliasing list MD must have at least one operand");
+  std::vector<SPIRVId> ListId;
+  for (const MDOperand &MDListOp : AliasingListMD->operands()) {
+    if (MDNode *ScopeMD = dyn_cast<MDNode>(MDListOp)) {
+      assert(ScopeMD->getNumOperands() > 1 &&
+             "Aliasing scope MD must have at least two operands");
+      MDNode *DomainMD = cast<MDNode>(ScopeMD->getOperand(1));
+      auto *Domain =
+          M->getOrAddAliasDomainDeclINTELInst(std::vector<SPIRVId>(), DomainMD);
+      auto *Scope =
+          M->getOrAddAliasScopeDeclINTELInst({Domain->getId()}, ScopeMD);
+      ListId.push_back(Scope->getId());
+    }
+  }
+  return M->getOrAddAliasScopeListDeclINTELInst(ListId, AliasingListMD);
+}
+
+
+// Translate alias.scope/noalias metadata attached to store and load
+// instructions.
+void transAliasingMemAccess(SPIRVModule *BM, MDNode *AliasingListMD,
+                            std::vector<uint32_t> &MemoryAccess,
+                            SPIRVWord MemAccessMask) {
+  if (!BM->isAllowedToUseExtension(
+        ExtensionID::SPV_INTEL_memory_access_aliasing))
+    return;
+  MemoryAccess[0] |= MemAccessMask;
+  auto *MemAliasList = addMemAliasingINTELInstructions(BM, AliasingListMD);
+  MemoryAccess.push_back(MemAliasList->getId());
+}
+
 /// An instruction may use an instruction from another BB which has not been
 /// translated. SPIRVForward should be created as place holder for these
 /// instructions and replaced later by the real instructions.
@@ -1380,6 +1421,8 @@ SPIRVValue *LLVMToSPIRV::transValueWithoutDecoration(Value *V,
     if (ST->isAtomic())
       return transAtomicStore(ST, BB);
 
+    // Keep this vector to store MemoryAccess operands for both Alignment and
+    // Aliasing information.
     std::vector<SPIRVWord> MemoryAccess(1, 0);
     if (ST->isVolatile())
       MemoryAccess[0] |= MemoryAccessVolatileMask;
@@ -1389,6 +1432,13 @@ SPIRVValue *LLVMToSPIRV::transValueWithoutDecoration(Value *V,
     }
     if (ST->getMetadata(LLVMContext::MD_nontemporal))
       MemoryAccess[0] |= MemoryAccessNontemporalMask;
+    if (MDNode *AliasingListMD = ST->getMetadata(LLVMContext::MD_alias_scope))
+      transAliasingMemAccess(BM, AliasingListMD, MemoryAccess,
+                             internal::MemoryAccessAliasScopeINTELMask);
+    else if (MDNode *AliasingListMD =
+                 ST->getMetadata(LLVMContext::MD_noalias))
+      transAliasingMemAccess(BM, AliasingListMD, MemoryAccess,
+                             internal::MemoryAccessNoAliasINTELMask);
     if (MemoryAccess.front() == 0)
       MemoryAccess.clear();
 
@@ -1403,7 +1453,9 @@ SPIRVValue *LLVMToSPIRV::transValueWithoutDecoration(Value *V,
     if (LD->isAtomic())
       return transAtomicLoad(LD, BB);
 
-    std::vector<SPIRVWord> MemoryAccess(1, 0);
+    // Keep this vector to store MemoryAccess operands for both Alignment and
+    // Aliasing information.
+    std::vector<uint32_t> MemoryAccess(1, 0);
     if (LD->isVolatile())
       MemoryAccess[0] |= MemoryAccessVolatileMask;
     if (LD->getAlignment()) {
@@ -1412,6 +1464,13 @@ SPIRVValue *LLVMToSPIRV::transValueWithoutDecoration(Value *V,
     }
     if (LD->getMetadata(LLVMContext::MD_nontemporal))
       MemoryAccess[0] |= MemoryAccessNontemporalMask;
+    if (MDNode *AliasingListMD = LD->getMetadata(LLVMContext::MD_alias_scope))
+      transAliasingMemAccess(BM, AliasingListMD, MemoryAccess,
+                            internal::MemoryAccessAliasScopeINTELMask);
+    else if (MDNode *AliasingListMD =
+                 LD->getMetadata(LLVMContext::MD_noalias))
+      transAliasingMemAccess(BM, AliasingListMD, MemoryAccess,
+                             internal::MemoryAccessNoAliasINTELMask);
     if (MemoryAccess.front() == 0)
       MemoryAccess.clear();
     return mapValue(V, BM->addLoadInst(transValue(LD->getPointerOperand(), BB),
@@ -1789,6 +1848,7 @@ bool LLVMToSPIRV::transDecoration(Value *V, SPIRVValue *BV) {
         BV->setFPFastMathMode(M);
     }
   }
+  transMemAliasingINTELDecorations(V, BV);
 
   return true;
 }
@@ -1805,6 +1865,35 @@ bool LLVMToSPIRV::transAlign(Value *V, SPIRVValue *BV) {
   return true;
 }
 
+// Apply aliasing decorations to instructions annotated with aliasing metadata.
+// Do it for any instruction but loads and stores.
+void LLVMToSPIRV::transMemAliasingINTELDecorations(Value *V, SPIRVValue *BV) {
+  if (!BM->isAllowedToUseExtension(
+         ExtensionID::SPV_INTEL_memory_access_aliasing))
+    return;
+  // Loads and Stores are handled during memory access mask addition
+  if (isa<StoreInst>(V) || isa<LoadInst>(V))
+    return;
+
+  Instruction *Inst = dyn_cast<Instruction>(V);
+  if (!Inst)
+    return;
+
+  if (MDNode *AliasingListMD =
+          Inst->getMetadata(LLVMContext::MD_alias_scope)) {
+    auto *MemAliasList =
+        addMemAliasingINTELInstructions(BM, AliasingListMD);
+    BV->addDecorate(new SPIRVDecorateId(
+          internal::DecorationAliasScopeINTEL, BV, MemAliasList->getId()));
+  } else if (MDNode *AliasingListMD =
+                 Inst->getMetadata(LLVMContext::MD_noalias)) {
+    auto *MemAliasList =
+        addMemAliasingINTELInstructions(BM, AliasingListMD);
+    BV->addDecorate(new SPIRVDecorateId(
+          internal::DecorationNoAliasINTEL, BV, MemAliasList->getId()));
+  }
+}
+
 /// Do this after source language is set.
 bool LLVMToSPIRV::transBuiltinSet() {
   SPIRVId EISId;

llvm-spirv/lib/SPIRV/SPIRVWriter.h

@@ -112,6 +112,7 @@ class LLVMToSPIRV : public ModulePass {
   SPIRVValue *transAsmINTEL(InlineAsm *Asm);
   SPIRVValue *transAsmCallINTEL(CallInst *Call, SPIRVBasicBlock *BB);
   bool transDecoration(Value *V, SPIRVValue *BV);
+  void transMemAliasingINTELDecorations(Value *V, SPIRVValue *BV);
   SPIRVWord transFunctionControlMask(Function *);
   SPIRVFunction *transFunctionDecl(Function *F);
   void transVectorComputeMetadata(Function *F);

llvm-spirv/lib/SPIRV/libSPIRV/SPIRVDecorate.cpp

@@ -80,6 +80,7 @@ SPIRVDecorateGeneric::SPIRVDecorateGeneric(Op OC, SPIRVWord WC,
   validate();
   updateModuleVersion();
 }
+
 SPIRVDecorateGeneric::SPIRVDecorateGeneric(Op OC)
     : SPIRVAnnotationGeneric(OC), Dec(DecorationRelaxedPrecision),
       Owner(nullptr) {}
@@ -151,6 +152,22 @@ void SPIRVDecorate::decode(std::istream &I) {
   getOrCreateTarget()->addDecorate(this);
 }
 
+void SPIRVDecorateId::encode(spv_ostream &O) const {
+  SPIRVEncoder Encoder = getEncoder(O);
+  Encoder << Target << Dec << Literals;
+}
+
+void SPIRVDecorateId::setWordCount(SPIRVWord Count) {
+  WordCount = Count;
+  Literals.resize(WordCount - FixedWC);
+}
+
+void SPIRVDecorateId::decode(std::istream &I) {
+  SPIRVDecoder Decoder = getDecoder(I);
+  Decoder >> Target >> Dec >> Literals;
+  getOrCreateTarget()->addDecorate(this);
+}
+
 void SPIRVMemberDecorate::encode(spv_ostream &O) const {
   SPIRVEncoder Encoder = getEncoder(O);
   Encoder << Target << MemberNumber << Dec;