Minimal support of floating-point operand bundles

clang/test/CodeGen/strictfp-elementwise-builtins.cpp

@@ -210,7 +210,7 @@ float4 strict_elementwise_rint(float4 a) {
 // CHECK-LABEL: define dso_local noundef <4 x float> @_Z28strict_elementwise_nearbyintDv4_f
 // CHECK-SAME: (<4 x float> noundef [[A:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[ELT_NEARBYINT:%.*]] = tail call <4 x float> @llvm.nearbyint.v4f32(<4 x float> [[A]]) #[[ATTR4]]
+// CHECK-NEXT:    [[ELT_NEARBYINT:%.*]] = tail call <4 x float> @llvm.nearbyint.v4f32(<4 x float> [[A]]) #[[ATTR5:[0-9]+]]
 // CHECK-NEXT:    ret <4 x float> [[ELT_NEARBYINT]]
 //
 float4 strict_elementwise_nearbyint(float4 a) {
@@ -300,7 +300,7 @@ float4 strict_elementwise_atan2(float4 a, float4 b) {
 // CHECK-LABEL: define dso_local noundef <4 x float> @_Z24strict_elementwise_truncDv4_f
 // CHECK-SAME: (<4 x float> noundef [[A:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[ELT_TRUNC:%.*]] = tail call <4 x float> @llvm.trunc.v4f32(<4 x float> [[A]]) #[[ATTR4]]
+// CHECK-NEXT:    [[ELT_TRUNC:%.*]] = tail call <4 x float> @llvm.trunc.v4f32(<4 x float> [[A]]) #[[ATTR5]]
 // CHECK-NEXT:    ret <4 x float> [[ELT_TRUNC]]
 //
 float4 strict_elementwise_trunc(float4 a) {

llvm/docs/LangRef.rst

@@ -3071,6 +3071,52 @@ A "convergencectrl" operand bundle is only valid on a ``convergent`` operation.
 When present, the operand bundle must contain exactly one value of token type.
 See the :doc:`ConvergentOperations` document for details.
 
+.. _ob_fp:
+
+Floating-point Operand Bundles
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+These operand bundles are used for calls that involve floating-point
+operations and interact with :ref:`floating-point environment <floatenv>` or
+depend on floating-point options, such as rounding mode, denormal modes, etc.
+There are two kinds of such operand bundles, which represent the value of
+floating-point control modes and the treatment of status bits respectively.
+
+An operand bundle tagged with "fp.control" contains information about the
+control modes used for the operation execution. Operands specified in this
+bundle represent particular options. Currently, only rounding mode is supported.
+
+Rounding mode is represented by a metadata string value, which specifies the
+the mode used for the operation evaluation. Possible values are:
+
+::
+
+    "rtz"  - toward zero
+    "rte"  - to nearest, ties to even
+    "rtp"  - toward positive infinity
+    "rtn"  - toward negative infinity
+    "rmm"  - to nearest, ties away from zero
+    "dyn"  - rounding mode is taken from control register
+
+Only one such value may be specified. If "fp.control" is absent, the default
+rounding rounding mode is taken from the control register (dynamic rounding).
+In the particular case of :ref:`default floating-point environment <floatenv>`,
+the operation uses rounding to nearest, ties to even.
+
+An operand bundle tagged with "fp.except" may be associated with operations
+that can read or write floating-point exception flags. It contains a single
+metadata string value, which can have one of the following values:
+
+::
+
+    "ignore"
+    "strict"
+    "maytrap"
+
+It has the same meaning as the corresponding argument in
+:ref:`constrained intrinsics <constrainedfp>`.
+
+
 .. _moduleasm:
 
 Module-Level Inline Assembly
@@ -3776,9 +3822,9 @@ round-to-nearest rounding mode, and subnormals are assumed to be preserved.
 Running LLVM code in an environment where these assumptions are not met
 typically leads to undefined behavior. The ``strictfp`` and ``denormal-fp-math``
 attributes as well as :ref:`Constrained Floating-Point Intrinsics
-<constrainedfp>` can be used to weaken LLVM's assumptions and ensure defined
-behavior in non-default floating-point environments; see their respective
-documentation for details.
+<constrainedfp>` or :ref:`floating-point operand bundles<ob_fp>` can be used to
+weaken LLVM's assumptions and ensure defined behavior in non-default
+floating-point environments; see their respective documentation for details.
 
 .. _floatnan:
 

llvm/docs/ReleaseNotes.md

@@ -64,6 +64,7 @@ Changes to the LLVM IR
 
 * Updated semantics of `llvm.type.checked.load.relative` to match that of
   `llvm.load.relative`.
+* Floating-point operand bundles have been added.
 
 Changes to LLVM infrastructure
 ------------------------------

llvm/include/llvm/IR/FPEnv.h

@@ -48,18 +48,36 @@ enum ExceptionBehavior : uint8_t {
 /// metadata.
 std::optional<RoundingMode> convertStrToRoundingMode(StringRef);
 
+/// Returns a valid RoundingMode enumerator given a string that is used as
+/// rounding mode specifier in operand bundles.
+std::optional<RoundingMode> convertBundleToRoundingMode(StringRef);
+
 /// For any RoundingMode enumerator, returns a string valid as input in
 /// constrained intrinsic rounding mode metadata.
 std::optional<StringRef> convertRoundingModeToStr(RoundingMode);
 
+/// For any RoundingMode enumerator, returns a string to be used in operand
+/// bundles.
+std::optional<StringRef> convertRoundingModeToBundle(RoundingMode);
+
 /// Returns a valid ExceptionBehavior enumerator when given a string
 /// valid as input in constrained intrinsic exception behavior metadata.
 std::optional<fp::ExceptionBehavior> convertStrToExceptionBehavior(StringRef);
 
+/// Returns a valid ExceptionBehavior enumerator given a string from the operand
+/// bundle argument.
+std::optional<fp::ExceptionBehavior>
+    convertBundleToExceptionBehavior(StringRef);
+
 /// For any ExceptionBehavior enumerator, returns a string valid as
 /// input in constrained intrinsic exception behavior metadata.
 std::optional<StringRef> convertExceptionBehaviorToStr(fp::ExceptionBehavior);
 
+/// Return string representing the given exception behavior for use in operand
+/// bundles
+std::optional<StringRef>
+    convertExceptionBehaviorToBundle(fp::ExceptionBehavior);
+
 /// Returns true if the exception handling behavior and rounding mode
 /// match what is used in the default floating point environment.
 inline bool isDefaultFPEnvironment(fp::ExceptionBehavior EB, RoundingMode RM) {

llvm/include/llvm/IR/FloatingPointOps.def

@@ -0,0 +1,24 @@
+//===- llvm/IR/FloatingPointOps.def - FP intrinsics -------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines set of intrinsics, which are classified as floating-point operations.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FUNCTION
+#define FUNCTION(N,D)
+#endif
+
+// Arguments of the entries are:
+// - intrinsic function name,
+// - DAG node corresponding to the intrinsic.
+
+FUNCTION(nearbyint,                          FNEARBYINT)
+FUNCTION(trunc,                              FTRUNC)
+
+#undef FUNCTION

llvm/include/llvm/IR/IRBuilder.h

@@ -1004,6 +1004,16 @@ class IRBuilderBase {
                             ArrayRef<Value *> Args, FMFSource FMFSource = {},
                             const Twine &Name = "");
 
+  /// Create a call to intrinsic \p ID with \p Args, mangled using \p Types and
+  /// with operand bundles.
+  /// If \p FMFSource is provided, copy fast-math-flags from that instruction to
+  /// the intrinsic.
+  CallInst *CreateIntrinsic(Intrinsic::ID ID, ArrayRef<Type *> Types,
+                            ArrayRef<Value *> Args,
+                            ArrayRef<OperandBundleDef> OpBundles,
+                            Instruction *FMFSource = nullptr,
+                            const Twine &Name = "");
+
   /// Create a call to non-overloaded intrinsic \p ID with \p Args. If
   /// \p FMFSource is provided, copy fast-math-flags from that instruction to
   /// the intrinsic.
@@ -2492,24 +2502,13 @@ class IRBuilderBase {
   CallInst *CreateCall(FunctionType *FTy, Value *Callee,
                        ArrayRef<Value *> Args = {}, const Twine &Name = "",
                        MDNode *FPMathTag = nullptr) {
-    CallInst *CI = CallInst::Create(FTy, Callee, Args, DefaultOperandBundles);
-    if (IsFPConstrained)
-      setConstrainedFPCallAttr(CI);
-    if (isa<FPMathOperator>(CI))
-      setFPAttrs(CI, FPMathTag, FMF);
-    return Insert(CI, Name);
+    return CreateCall(FTy, Callee, Args, DefaultOperandBundles, Name,
+                      FPMathTag);
   }
 
   CallInst *CreateCall(FunctionType *FTy, Value *Callee, ArrayRef<Value *> Args,
                        ArrayRef<OperandBundleDef> OpBundles,
-                       const Twine &Name = "", MDNode *FPMathTag = nullptr) {
-    CallInst *CI = CallInst::Create(FTy, Callee, Args, OpBundles);
-    if (IsFPConstrained)
-      setConstrainedFPCallAttr(CI);
-    if (isa<FPMathOperator>(CI))
-      setFPAttrs(CI, FPMathTag, FMF);
-    return Insert(CI, Name);
-  }
+                       const Twine &Name = "", MDNode *FPMathTag = nullptr);
 
   CallInst *CreateCall(FunctionCallee Callee, ArrayRef<Value *> Args = {},
                        const Twine &Name = "", MDNode *FPMathTag = nullptr) {
@@ -2731,6 +2730,24 @@ class IRBuilderBase {
   /// Create an assume intrinsic call that represents an dereferencable
   /// assumption on the provided pointer.
   CallInst *CreateDereferenceableAssumption(Value *PtrValue, Value *SizeValue);
+
+  /// Create an operand bundle in the provided bundle set to represent given FP
+  /// rounding mode.
+  ///
+  /// If the rounding mode is not defined, adds the default rounding mode,
+  /// stored in this builder object.
+  void
+  createFPRoundingBundle(SmallVectorImpl<OperandBundleDef> &Bundles,
+                         std::optional<RoundingMode> Rounding = std::nullopt);
+
+  /// Create an operand bundle in the provided bundle set to represent FP
+  /// exception behavior.
+  ///
+  /// If the exception behavior is not defined, adds the default behavior,
+  /// stored in this builder object.
+  void createFPExceptionBundle(
+      SmallVectorImpl<OperandBundleDef> &Bundles,
+      std::optional<fp::ExceptionBehavior> Except = std::nullopt);
 };
 
 /// This provides a uniform API for creating instructions and inserting

llvm/include/llvm/IR/InstrTypes.h

@@ -25,6 +25,7 @@
 #include "llvm/IR/CallingConv.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/FMF.h"
+#include "llvm/IR/FPEnv.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/LLVMContext.h"
@@ -1091,6 +1092,13 @@ template <typename InputTy> class OperandBundleDefT {
 using OperandBundleDef = OperandBundleDefT<Value *>;
 using ConstOperandBundleDef = OperandBundleDefT<const Value *>;
 
+void addFPRoundingBundle(LLVMContext &Ctx,
+                         SmallVectorImpl<OperandBundleDef> &Bundles,
+                         RoundingMode Rounding);
+void addFPExceptionBundle(LLVMContext &Ctx,
+                          SmallVectorImpl<OperandBundleDef> &Bundles,
+                          fp::ExceptionBehavior Except);
+
 //===----------------------------------------------------------------------===//
 //                               CallBase Class
 //===----------------------------------------------------------------------===//
@@ -1150,6 +1158,8 @@ class CallBase : public Instruction {
   /// number of extra operands.
   unsigned getNumSubclassExtraOperandsDynamic() const;
 
+  MemoryEffects getFloatingPointMemoryEffects() const;
+
 public:
   using Instruction::getContext;
 
@@ -2155,6 +2165,12 @@ class CallBase : public Instruction {
     return false;
   }
 
+  /// Return rounding mode specified for this call.
+  RoundingMode getRoundingMode() const;
+
+  /// Return exception behavior specified for this call.
+  fp::ExceptionBehavior getExceptionBehavior() const;
+
   /// Used to keep track of an operand bundle.  See the main comment on
   /// OperandBundleUser above.
   struct BundleOpInfo {

llvm/include/llvm/IR/IntrinsicInst.h

@@ -128,6 +128,14 @@ class IntrinsicInst : public CallInst {
   /// course of IR transformations
   static bool mayLowerToFunctionCall(Intrinsic::ID IID);
 
+  /// Check if \p ID represents a function that may access FP environment and
+  /// may have FP operand bundles.
+  ///
+  /// Access to FP environment means that in the strict FP environment the
+  /// function has read/write memory effect, which is used to maintain proper
+  /// instructions ordering.
+  static bool isFloatingPointOperation(Intrinsic::ID IID);
+
   /// Methods for support type inquiry through isa, cast, and dyn_cast:
   static bool classof(const CallInst *I) {
     if (const Function *CF = I->getCalledFunction())

llvm/include/llvm/IR/LLVMContext.h

@@ -96,6 +96,8 @@ class LLVMContext {
     OB_ptrauth = 7,                // "ptrauth"
     OB_kcfi = 8,                   // "kcfi"
     OB_convergencectrl = 9,        // "convergencectrl"
+    OB_fp_control = 10,            // "fp.control"
+    OB_fp_except = 11,             // "fp.except"
   };
 
   /// getMDKindID - Return a unique non-zero ID for the specified metadata kind.

llvm/include/llvm/Support/ModRef.h

@@ -234,6 +234,11 @@ template <typename LocationEnum> class MemoryEffectsBase {
     return getWithoutLoc(Location::InaccessibleMem).doesNotAccessMemory();
   }
 
+  /// Whether this function accesses inaccessible memory.
+  bool doesAccessInaccessibleMem() const {
+    return isModOrRefSet(getModRef(Location::InaccessibleMem));
+  }
+
   /// Whether this function only (at most) accesses errno memory.
   bool onlyAccessesErrnoMem() const {
     return getWithoutLoc(Location::ErrnoMem).doesNotAccessMemory();

llvm/lib/IR/FPEnv.cpp

@@ -34,6 +34,17 @@ std::optional<RoundingMode> convertStrToRoundingMode(StringRef RoundingArg) {
       .Default(std::nullopt);
 }
 
+std::optional<RoundingMode> convertBundleToRoundingMode(StringRef RoundingArg) {
+  return StringSwitch<std::optional<RoundingMode>>(RoundingArg)
+      .Case("dyn", RoundingMode::Dynamic)
+      .Case("rte", RoundingMode::NearestTiesToEven)
+      .Case("rmm", RoundingMode::NearestTiesToAway)
+      .Case("rtn", RoundingMode::TowardNegative)
+      .Case("rtp", RoundingMode::TowardPositive)
+      .Case("rtz", RoundingMode::TowardZero)
+      .Default(std::nullopt);
+}
+
 std::optional<StringRef> convertRoundingModeToStr(RoundingMode UseRounding) {
   std::optional<StringRef> RoundingStr;
   switch (UseRounding) {
@@ -61,6 +72,33 @@ std::optional<StringRef> convertRoundingModeToStr(RoundingMode UseRounding) {
   return RoundingStr;
 }
 
+std::optional<StringRef> convertRoundingModeToBundle(RoundingMode UseRounding) {
+  std::optional<StringRef> RoundingStr;
+  switch (UseRounding) {
+  case RoundingMode::Dynamic:
+    RoundingStr = "dyn";
+    break;
+  case RoundingMode::NearestTiesToEven:
+    RoundingStr = "rte";
+    break;
+  case RoundingMode::NearestTiesToAway:
+    RoundingStr = "rmm";
+    break;
+  case RoundingMode::TowardNegative:
+    RoundingStr = "rtn";
+    break;
+  case RoundingMode::TowardPositive:
+    RoundingStr = "rtp";
+    break;
+  case RoundingMode::TowardZero:
+    RoundingStr = "rtz";
+    break;
+  default:
+    break;
+  }
+  return RoundingStr;
+}
+
 std::optional<fp::ExceptionBehavior>
 convertStrToExceptionBehavior(StringRef ExceptionArg) {
   return StringSwitch<std::optional<fp::ExceptionBehavior>>(ExceptionArg)
@@ -70,6 +108,15 @@ convertStrToExceptionBehavior(StringRef ExceptionArg) {
       .Default(std::nullopt);
 }
 
+std::optional<fp::ExceptionBehavior>
+convertBundleToExceptionBehavior(StringRef ExceptionArg) {
+  return StringSwitch<std::optional<fp::ExceptionBehavior>>(ExceptionArg)
+      .Case("ignore", fp::ebIgnore)
+      .Case("maytrap", fp::ebMayTrap)
+      .Case("strict", fp::ebStrict)
+      .Default(std::nullopt);
+}
+
 std::optional<StringRef>
 convertExceptionBehaviorToStr(fp::ExceptionBehavior UseExcept) {
   std::optional<StringRef> ExceptStr;
@@ -87,6 +134,23 @@ convertExceptionBehaviorToStr(fp::ExceptionBehavior UseExcept) {
   return ExceptStr;
 }
 
+std::optional<StringRef>
+convertExceptionBehaviorToBundle(fp::ExceptionBehavior UseExcept) {
+  std::optional<StringRef> ExceptStr;
+  switch (UseExcept) {
+  case fp::ebStrict:
+    ExceptStr = "strict";
+    break;
+  case fp::ebIgnore:
+    ExceptStr = "ignore";
+    break;
+  case fp::ebMayTrap:
+    ExceptStr = "maytrap";
+    break;
+  }
+  return ExceptStr;
+}
+
 Intrinsic::ID getConstrainedIntrinsicID(const Instruction &Instr) {
   Intrinsic::ID IID = Intrinsic::not_intrinsic;
   switch (Instr.getOpcode()) {