Implement SPV_INTEL_fpga_invocation_pipelining_attributes extension

MrSidims · AlexeySotkin · commit e303a5f8e9ad · 2021-05-27T17:24:48.000+03:00
Some FPGA devices and toolchains can support customizable levels or implementation of pipeline parallelism when mapping a SPIR-V module to hardware. Through pipeline parallelism, multiple invocations of a kernel or function can execute concurrently. This extension adds decorations to request that a kernel or function support invocations at a specified initiation interval, that multiple invocations are forbidden from executing concurrently, or that the kernel or function is limited to a maximum number of concurrent invocations. Spec: https://github.com/KhronosGroup/SPIRV-Registry/blob/38a7e56b9b7feab7fa8b3bea08770f7232b76ed2/extensions/INTEL/SPV_INTEL_fpga_invocation_pipelining_attributes.asciidoc Signed-off-by: Dmitry Sidorov <dmitry.sidorov@intel.com>
diff --git a/include/LLVMSPIRVExtensions.inc b/include/LLVMSPIRVExtensions.inc
@@ -37,3 +37,4 @@ EXT(SPV_INTEL_loop_fuse)
 EXT(SPV_INTEL_long_constant_composite)
 EXT(SPV_INTEL_optnone)
 EXT(SPV_INTEL_memory_access_aliasing)
+EXT(SPV_INTEL_fpga_invocation_pipelining_attributes)
diff --git a/lib/SPIRV/SPIRVInternal.h b/lib/SPIRV/SPIRVInternal.h
@@ -406,6 +406,9 @@ const static char NumSIMD[] = "num_simd_work_items";
 const static char StallEnable[] = "stall_enable";
 const static char FmaxMhz[] = "scheduler_target_fmax_mhz";
 const static char LoopFuse[] = "loop_fuse";
+const static char InitiationInterval[] = "initiation_interval";
+const static char MaxConcurrency[] = "max_concurrency";
+const static char DisableLoopPipelining[] = "disable_loop_pipelining";
 } // namespace kSPIR2MD
 
 enum Spir2SamplerKind {
diff --git a/lib/SPIRV/SPIRVReader.cpp b/lib/SPIRV/SPIRVReader.cpp
@@ -4272,6 +4272,32 @@ bool SPIRVToLLVM::transFPGAFunctionMetadata(SPIRVFunction *BF, Function *F) {
     MetadataVec.push_back(ConstantAsMetadata::get(getUInt32(M, Literals[1])));
     F->setMetadata(kSPIR2MD::LoopFuse, MDNode::get(*Context, MetadataVec));
   }
+  if (BF->hasDecorate(internal::DecorationInitiationIntervalINTEL)) {
+    std::vector<Metadata *> MetadataVec;
+    auto Literals =
+        BF->getDecorationLiterals(internal::DecorationInitiationIntervalINTEL);
+    MetadataVec.push_back(ConstantAsMetadata::get(getUInt32(M, Literals[0])));
+    F->setMetadata(kSPIR2MD::InitiationInterval,
+                   MDNode::get(*Context, MetadataVec));
+  }
+  if (BF->hasDecorate(internal::DecorationMaxConcurrencyINTEL)) {
+    std::vector<Metadata *> MetadataVec;
+    auto Literals =
+        BF->getDecorationLiterals(internal::DecorationMaxConcurrencyINTEL);
+    MetadataVec.push_back(ConstantAsMetadata::get(getUInt32(M, Literals[0])));
+    F->setMetadata(kSPIR2MD::MaxConcurrency,
+                   MDNode::get(*Context, MetadataVec));
+  }
+  if (BF->hasDecorate(internal::DecorationPipelineEnableINTEL)) {
+    auto Literals =
+        BF->getDecorationLiterals(internal::DecorationPipelineEnableINTEL);
+    if (!Literals[0]) {
+      std::vector<Metadata *> MetadataVec;
+      MetadataVec.push_back(ConstantAsMetadata::get(getInt32(M, 1)));
+      F->setMetadata(kSPIR2MD::DisableLoopPipelining,
+                     MDNode::get(*Context, MetadataVec));
+    }
+  }
   return true;
 }
 
diff --git a/lib/SPIRV/SPIRVWriter.cpp b/lib/SPIRV/SPIRVWriter.cpp
@@ -728,6 +728,29 @@ void LLVMToSPIRVBase::transFPGAFunctionMetadata(SPIRVFunction *BF,
           new SPIRVDecorateFuseLoopsInFunctionINTEL(BF, Depth, Independent));
     }
   }
+  if (MDNode *InitiationInterval =
+          F->getMetadata(kSPIR2MD::InitiationInterval)) {
+    if (BM->isAllowedToUseExtension(
+            ExtensionID::SPV_INTEL_fpga_invocation_pipelining_attributes)) {
+      if (size_t Cycles = getMDOperandAsInt(InitiationInterval, 0))
+        BF->addDecorate(new SPIRVDecorateInitiationIntervalINTEL(BF, Cycles));
+    }
+  }
+  if (MDNode *MaxConcurrency = F->getMetadata(kSPIR2MD::MaxConcurrency)) {
+    if (BM->isAllowedToUseExtension(
+            ExtensionID::SPV_INTEL_fpga_invocation_pipelining_attributes)) {
+      size_t Invocations = getMDOperandAsInt(MaxConcurrency, 0);
+      BF->addDecorate(new SPIRVDecorateMaxConcurrencyINTEL(BF, Invocations));
+    }
+  }
+  if (MDNode *DisableLoopPipelining =
+          F->getMetadata(kSPIR2MD::DisableLoopPipelining)) {
+    if (BM->isAllowedToUseExtension(
+            ExtensionID::SPV_INTEL_fpga_invocation_pipelining_attributes)) {
+      if (size_t Disable = getMDOperandAsInt(DisableLoopPipelining, 0))
+        BF->addDecorate(new SPIRVDecoratePipelineEnableINTEL(BF, !Disable));
+    }
+  }
 }
 
 SPIRVValue *LLVMToSPIRVBase::transConstant(Value *V) {
diff --git a/lib/SPIRV/libSPIRV/SPIRVDecorate.h b/lib/SPIRV/libSPIRV/SPIRVDecorate.h
@@ -154,7 +154,7 @@ class SPIRVDecorate : public SPIRVDecorateGeneric {
   SPIRVDecorate() : SPIRVDecorateGeneric(OC) {}
 
   llvm::Optional<ExtensionID> getRequiredExtension() const override {
-    switch (Dec) {
+    switch (static_cast<size_t>(Dec)) {
     case DecorationNoSignedWrap:
     case DecorationNoUnsignedWrap:
       return ExtensionID::SPV_KHR_no_integer_wrap_decoration;
@@ -192,6 +192,12 @@ class SPIRVDecorate : public SPIRVDecorateGeneric {
       return ExtensionID::SPV_INTEL_loop_fuse;
     case DecorationCallableFunctionINTEL:
       return ExtensionID::SPV_INTEL_fast_composite;
+    case internal::DecorationInitiationIntervalINTEL:
+      return ExtensionID::SPV_INTEL_fpga_invocation_pipelining_attributes;
+    case internal::DecorationMaxConcurrencyINTEL:
+      return ExtensionID::SPV_INTEL_fpga_invocation_pipelining_attributes;
+    case internal::DecorationPipelineEnableINTEL:
+      return ExtensionID::SPV_INTEL_fpga_invocation_pipelining_attributes;
     default:
       return {};
     }
@@ -682,6 +688,30 @@ class SPIRVDecorateNoAliasINTEL : public SPIRVDecorateId {
                         AliasList){};
 };
 
+class SPIRVDecorateInitiationIntervalINTEL : public SPIRVDecorate {
+public:
+  // Complete constructor for SPIRVDecorateInitiationIntervalINTEL
+  SPIRVDecorateInitiationIntervalINTEL(SPIRVEntry *TheTarget, SPIRVWord Cycles)
+      : SPIRVDecorate(spv::internal::DecorationInitiationIntervalINTEL,
+                      TheTarget, Cycles){};
+};
+
+class SPIRVDecorateMaxConcurrencyINTEL : public SPIRVDecorate {
+public:
+  // Complete constructor for SPIRVDecorateMaxConcurrencyINTEL
+  SPIRVDecorateMaxConcurrencyINTEL(SPIRVEntry *TheTarget, SPIRVWord Invocations)
+      : SPIRVDecorate(spv::internal::DecorationMaxConcurrencyINTEL, TheTarget,
+                      Invocations){};
+};
+
+class SPIRVDecoratePipelineEnableINTEL : public SPIRVDecorate {
+public:
+  // Complete constructor for SPIRVDecoratePipelineEnableINTEL
+  SPIRVDecoratePipelineEnableINTEL(SPIRVEntry *TheTarget, SPIRVWord Enable)
+      : SPIRVDecorate(spv::internal::DecorationPipelineEnableINTEL, TheTarget,
+                      Enable){};
+};
+
 } // namespace SPIRV
 
 #endif // SPIRV_LIBSPIRV_SPIRVDECORATE_H
diff --git a/lib/SPIRV/libSPIRV/SPIRVEnum.h b/lib/SPIRV/libSPIRV/SPIRVEnum.h
@@ -429,6 +429,12 @@ template <> inline void SPIRVMap<Decoration, SPIRVCapVec>::init() {
                {internal::CapabilityMemoryAccessAliasingINTEL});
   ADD_VEC_INIT(internal::DecorationNoAliasINTEL,
                {internal::CapabilityMemoryAccessAliasingINTEL});
+  ADD_VEC_INIT(internal::DecorationInitiationIntervalINTEL,
+               {internal::CapabilityFPGAInvocationPipeliningAttributesINTEL});
+  ADD_VEC_INIT(internal::DecorationMaxConcurrencyINTEL,
+               {internal::CapabilityFPGAInvocationPipeliningAttributesINTEL});
+  ADD_VEC_INIT(internal::DecorationPipelineEnableINTEL,
+               {internal::CapabilityFPGAInvocationPipeliningAttributesINTEL});
 }
 
 template <> inline void SPIRVMap<BuiltIn, SPIRVCapVec>::init() {
diff --git a/lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h b/lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h
@@ -169,6 +169,9 @@ template <> inline void SPIRVMap<Decoration, std::string>::init() {
   add(DecorationFuseLoopsInFunctionINTEL, "FuseLoopsInFunctionINTEL");
   add(internal::DecorationAliasScopeINTEL, "AliasScopeINTEL");
   add(internal::DecorationNoAliasINTEL, "NoAliasINTEL");
+  add(internal::DecorationInitiationIntervalINTEL, "InitiationIntervalINTEL");
+  add(internal::DecorationMaxConcurrencyINTEL, "MaxConcurrencyINTEL");
+  add(internal::DecorationPipelineEnableINTEL, "PipelineEnableINTEL");
   add(DecorationMax, "Max");
 }
 SPIRV_DEF_NAMEMAP(Decoration, SPIRVDecorationNameMap)
@@ -520,6 +523,8 @@ template <> inline void SPIRVMap<Capability, std::string>::init() {
   add(internal::CapabilityOptNoneINTEL, "OptNoneINTEL");
   add(internal::CapabilityMemoryAccessAliasingINTEL,
       "MemoryAccessAliasingINTEL");
+  add(internal::CapabilityFPGAInvocationPipeliningAttributesINTEL,
+      "FPGAInvocationPipeliningAttributesINTEL");
 }
 SPIRV_DEF_NAMEMAP(Capability, SPIRVCapabilityNameMap)
 
diff --git a/lib/SPIRV/libSPIRV/spirv_internal.hpp b/lib/SPIRV/libSPIRV/spirv_internal.hpp
@@ -44,11 +44,15 @@ enum InternalOp {
 
 enum InternalDecoration {
   IDecAliasScopeINTEL = 5914,
-  IDecNoAliasINTEL = 5915
+  IDecNoAliasINTEL = 5915,
+  IDecInitiationIntervalINTEL = 5917,
+  IDecMaxConcurrencyINTEL = 5918,
+  IDecPipelineEnableINTEL = 5919
 };
 
 enum InternalCapability {
   ICapMemoryAccessAliasingINTEL = 5910,
+  ICapFPGAInvocationPipeliningAttributesINTEL = 5916,
   ICapOptNoneINTEL = 6094
 };
 
@@ -72,11 +76,19 @@ constexpr Decoration DecorationAliasScopeINTEL =
     static_cast<Decoration>(IDecAliasScopeINTEL );
 constexpr Decoration DecorationNoAliasINTEL =
     static_cast<Decoration>(IDecNoAliasINTEL);
+constexpr Decoration DecorationInitiationIntervalINTEL =
+    static_cast<Decoration>(IDecInitiationIntervalINTEL);
+constexpr Decoration DecorationMaxConcurrencyINTEL =
+    static_cast<Decoration>(IDecMaxConcurrencyINTEL);
+constexpr Decoration DecorationPipelineEnableINTEL =
+    static_cast<Decoration>(IDecPipelineEnableINTEL);
 
 constexpr Capability CapabilityOptNoneINTEL =
     static_cast<Capability>(ICapOptNoneINTEL);
 constexpr Capability CapabilityMemoryAccessAliasingINTEL =
     static_cast<Capability>(ICapMemoryAccessAliasingINTEL);
+constexpr Capability CapabilityFPGAInvocationPipeliningAttributesINTEL =
+    static_cast<Capability>(ICapFPGAInvocationPipeliningAttributesINTEL);
 
 constexpr FunctionControlMask FunctionControlOptNoneINTELMask =
     static_cast<FunctionControlMask>(IFunctionControlOptNoneINTELMask);
diff --git a/test/transcoding/IntelFPGAFunctionAttributes.ll b/test/transcoding/IntelFPGAFunctionAttributes.ll
@@ -7,7 +7,10 @@
 ;;     intel::num_simd_work_items(8),
 ;;     intel::stall_enable,
 ;;     intel::scheduler_target_fmax_mhz(1000),
-;;     intel::loop_fuse_independent(3)]] void operator()() {}
+;;     intel::loop_fuse_independent(3),
+;;     intel::initiation_interval(10),
+;;     intel::max_concurrency(12),
+;;     intel::disable_loop_pipelining]] void operator()() {}
 ;; };
 ;;
 ;; template <typename name, typename Func>
@@ -22,7 +25,7 @@
 ;; }
 
 ; RUN: llvm-as %s -o %t.bc
-; RUN: llvm-spirv %t.bc --spirv-ext=+SPV_INTEL_kernel_attributes --spirv-ext=+SPV_INTEL_fpga_cluster_attributes,+SPV_INTEL_loop_fuse -o %t.spv
+; RUN: llvm-spirv %t.bc --spirv-ext=+SPV_INTEL_kernel_attributes,+SPV_INTEL_fpga_cluster_attributes,+SPV_INTEL_loop_fuse,+SPV_INTEL_fpga_invocation_pipelining_attributes -o %t.spv
 ; RUN: llvm-spirv %t.spv -to-text -o %t.spt
 ; RUN: FileCheck < %t.spt %s --check-prefix=CHECK-SPIRV
 
@@ -32,24 +35,41 @@
 ; RUN: llvm-spirv -spirv-text -r %t.spt -o %t.rev.bc
 ; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-LLVM
 
-; CHECK-SPIRV: 2 Capability KernelAttributesINTEL
-; CHECK-SPIRV: 2 Capability FPGAKernelAttributesINTEL
-; CHECK-SPIRV: 2 Capability FPGAClusterAttributesINTEL
-; CHECK-SPIRV: 2 Capability LoopFuseINTEL
-; CHECK-SPIRV: 6 ExecutionMode [[FUNCENTRY:[0-9]+]] 5893 1 1 1
-; CHECK-SPIRV: 4 ExecutionMode [[FUNCENTRY]] 5894 1
-; CHECK-SPIRV: 3 ExecutionMode [[FUNCENTRY]] 5895
-; CHECK-SPIRV: 4 ExecutionMode [[FUNCENTRY]] 5896 8
-; CHECK-SPIRV: 4 ExecutionMode [[FUNCENTRY]] 5903 1000
-; CHECK-SPIRV: 3 Decorate [[FUNCENTRY]] StallEnableINTEL
-; CHECK-SPIRV: 5 Decorate [[FUNCENTRY]] FuseLoopsInFunctionINTEL 3 1
-; CHECK-SPIRV: 5 Function {{.*}} [[FUNCENTRY]] {{.*}}
+; CHECK-SPIRV: Capability KernelAttributesINTEL
+; CHECK-SPIRV: Capability FPGAKernelAttributesINTEL
+; CHECK-SPIRV: Capability FPGAClusterAttributesINTEL
+; CHECK-SPIRV: Capability LoopFuseINTEL
+; CHECK-SPIRV: Capability FPGAInvocationPipeliningAttributesINTEL
+; CHECK-SPIRV: Extension "SPV_INTEL_fpga_cluster_attributes"
+; CHECK-SPIRV: Extension "SPV_INTEL_fpga_invocation_pipelining_attributes"
+; CHECK-SPIRV: Extension "SPV_INTEL_loop_fuse"
+; CHECK-SPIRV: ExecutionMode [[FUNCENTRY:[0-9]+]] 5893 1 1 1
+; CHECK-SPIRV: ExecutionMode [[FUNCENTRY]] 5894 1
+; CHECK-SPIRV: ExecutionMode [[FUNCENTRY]] 5895
+; CHECK-SPIRV: ExecutionMode [[FUNCENTRY]] 5896 8
+; CHECK-SPIRV: ExecutionMode [[FUNCENTRY]] 5903 1000
+; CHECK-SPIRV: Decorate [[FUNCENTRY]] StallEnableINTEL
+; CHECK-SPIRV: Decorate [[FUNCENTRY]] FuseLoopsInFunctionINTEL 3 1
+; CHECK-SPIRV: Decorate [[FUNCENTRY]] InitiationIntervalINTEL 10
+; CHECK-SPIRV: Decorate [[FUNCENTRY]] MaxConcurrencyINTEL 12
+; CHECK-SPIRV: Decorate [[FUNCENTRY]] PipelineEnableINTEL 0
+; CHECK-SPIRV: Function {{.*}} [[FUNCENTRY]] {{.*}}
 
-; CHECK-LLVM: define spir_kernel void {{.*}}kernel_name() {{.*}} !stall_enable ![[ONEMD:[0-9]+]] !loop_fuse ![[FUSE:[0-9]+]] !max_work_group_size ![[MAXWG:[0-9]+]] !no_global_work_offset ![[OFFSET:[0-9]+]] !max_global_work_dim ![[ONEMD:[0-9]+]] !num_simd_work_items ![[NUMSIMD:[0-9]+]] !scheduler_target_fmax_mhz ![[MAXMHZ:[0-9]+]]
+; CHECK-LLVM: define spir_kernel void {{.*}}kernel_name()
+; CHECK-LLVM-SAME: !stall_enable ![[ONEMD:[0-9]+]] !loop_fuse ![[FUSE:[0-9]+]]
+; CHECK-LLVM-SAME: !initiation_interval ![[II:[0-9]+]]
+; CHECK-LLVM-SAME: !max_concurrency ![[MAXCON:[0-9]+]]
+; CHECK-LLVM-SAME: !disable_loop_pipelining ![[ONEMD]]
+; CHECK-LLVM-SAME: !max_work_group_size ![[MAXWG:[0-9]+]]
+; CHECK-LLVM-SAME: !no_global_work_offset ![[OFFSET:[0-9]+]]
+; CHECK-LLVM-SAME: !max_global_work_dim ![[ONEMD]] !num_simd_work_items ![[NUMSIMD:[0-9]+]]
+; CHECK-LLVM-SAME: !scheduler_target_fmax_mhz ![[MAXMHZ:[0-9]+]]
 ; CHECK-LLVM-NOT: define spir_kernel void {{.*}}kernel_name2 {{.*}} !no_global_work_offset {{.*}}
 ; CHECK-LLVM: ![[OFFSET]] = !{}
 ; CHECK-LLVM: ![[ONEMD]] = !{i32 1}
 ; CHECK-LLVM: ![[FUSE]] = !{i32 3, i32 1}
+; CHECK-LLVM: ![[II]] = !{i32 10}
+; CHECK-LLVM: ![[MAXCON]] = !{i32 12}
 ; CHECK-LLVM: ![[MAXWG]] = !{i32 1, i32 1, i32 1}
 ; CHECK-LLVM: ![[NUMSIMD]] = !{i32 8}
 ; CHECK-LLVM: ![[MAXMHZ]] = !{i32 1000}
@@ -65,7 +85,7 @@ target triple = "spir64-unknown-linux-sycldevice"
 $_ZN3FooclEv = comdat any
 
 ; Function Attrs: nounwind
-define spir_kernel void @_ZTSZ3barvE11kernel_name() #0 !kernel_arg_addr_space !4 !kernel_arg_access_qual !4 !kernel_arg_type !4 !kernel_arg_base_type !4 !kernel_arg_type_qual !4 !num_simd_work_items !5 !max_work_group_size !6 !max_global_work_dim !7 !no_global_work_offset !4 !stall_enable !7 !scheduler_target_fmax_mhz !12 !loop_fuse !13 {
+define spir_kernel void @_ZTSZ3barvE11kernel_name() #0 !kernel_arg_addr_space !4 !kernel_arg_access_qual !4 !kernel_arg_type !4 !kernel_arg_base_type !4 !kernel_arg_type_qual !4 !num_simd_work_items !5 !max_work_group_size !6 !max_global_work_dim !7 !no_global_work_offset !4 !stall_enable !7 !scheduler_target_fmax_mhz !12 !loop_fuse !13 !initiation_interval !14 !max_concurrency !15 !disable_loop_pipelining !7 {
 entry:
   %Foo = alloca %class._ZTS3Foo.Foo, align 1
   %0 = bitcast %class._ZTS3Foo.Foo* %Foo to i8*
@@ -137,3 +157,5 @@ attributes #4 = { nounwind }
 !11 = !{!"Simple C++ TBAA"}
 !12 = !{i32 1000}
 !13 = !{i32 3, i32 1}
+!14 = !{i32 10}
+!15 = !{i32 12}
