Add deactivation symbol operand to ConstantPtrAuth.

clang/lib/CodeGen/CGPointerAuth.cpp

@@ -440,9 +440,9 @@ CodeGenModule::getConstantSignedPointer(llvm::Constant *Pointer, unsigned Key,
     IntegerDiscriminator = llvm::ConstantInt::get(Int64Ty, 0);
   }
 
-  return llvm::ConstantPtrAuth::get(Pointer,
-                                    llvm::ConstantInt::get(Int32Ty, Key),
-                                    IntegerDiscriminator, AddressDiscriminator);
+  return llvm::ConstantPtrAuth::get(
+      Pointer, llvm::ConstantInt::get(Int32Ty, Key), IntegerDiscriminator,
+      AddressDiscriminator, llvm::Constant::getNullValue(UnqualPtrTy));
 }
 
 /// Does a given PointerAuthScheme require us to sign a value

lld/ELF/Arch/AArch64.cpp

@@ -114,6 +114,7 @@ AArch64::AArch64(Ctx &ctx) : TargetInfo(ctx) {
   copyRel = R_AARCH64_COPY;
   relativeRel = R_AARCH64_RELATIVE;
   iRelativeRel = R_AARCH64_IRELATIVE;
+  iRelSymbolicRel = R_AARCH64_FUNCINIT64;
   gotRel = R_AARCH64_GLOB_DAT;
   pltRel = R_AARCH64_JUMP_SLOT;
   symbolicRel = R_AARCH64_ABS64;
@@ -137,6 +138,7 @@ RelExpr AArch64::getRelExpr(RelType type, const Symbol &s,
   case R_AARCH64_ABS16:
   case R_AARCH64_ABS32:
   case R_AARCH64_ABS64:
+  case R_AARCH64_FUNCINIT64:
   case R_AARCH64_ADD_ABS_LO12_NC:
   case R_AARCH64_LDST128_ABS_LO12_NC:
   case R_AARCH64_LDST16_ABS_LO12_NC:
@@ -267,7 +269,8 @@ bool AArch64::usesOnlyLowPageBits(RelType type) const {
 }
 
 RelType AArch64::getDynRel(RelType type) const {
-  if (type == R_AARCH64_ABS64 || type == R_AARCH64_AUTH_ABS64)
+  if (type == R_AARCH64_ABS64 || type == R_AARCH64_AUTH_ABS64 ||
+      type == R_AARCH64_FUNCINIT64)
     return type;
   return R_AARCH64_NONE;
 }

lld/ELF/Relocations.cpp

@@ -968,8 +968,8 @@ bool RelocationScanner::isStaticLinkTimeConstant(RelExpr e, RelType type,
   // only the low bits are used.
   if (e == R_GOT || e == R_PLT)
     return ctx.target->usesOnlyLowPageBits(type) || !ctx.arg.isPic;
-  // R_AARCH64_AUTH_ABS64 requires a dynamic relocation.
-  if (e == RE_AARCH64_AUTH)
+  // R_AARCH64_AUTH_ABS64 and iRelSymbolicRel require a dynamic relocation.
+  if (e == RE_AARCH64_AUTH || type == ctx.target->iRelSymbolicRel)
     return false;
 
   // The behavior of an undefined weak reference is implementation defined.
@@ -1142,6 +1142,23 @@ void RelocationScanner::process(RelExpr expr, RelType type, uint64_t offset,
         }
         return;
       }
+      if (LLVM_UNLIKELY(type == ctx.target->iRelSymbolicRel)) {
+        if (sym.isPreemptible) {
+          auto diag = Err(ctx);
+          diag << "relocation " << type
+               << " cannot be used against preemptible symbol '" << &sym << "'";
+          printLocation(diag, *sec, sym, offset);
+        } else if (isIfunc) {
+          auto diag = Err(ctx);
+          diag << "relocation " << type
+               << " cannot be used against ifunc symbol '" << &sym << "'";
+          printLocation(diag, *sec, sym, offset);
+        } else {
+          part.relaDyn->addReloc({ctx.target->iRelativeRel, sec, offset, false,
+                                  sym, addend, R_ABS});
+          return;
+        }
+      }
       part.relaDyn->addSymbolReloc(rel, *sec, offset, sym, addend, type);
 
       // MIPS ABI turns using of GOT and dynamic relocations inside out.

lld/ELF/Target.h

@@ -135,6 +135,7 @@ class TargetInfo {
   RelType relativeRel = 0;
   RelType iRelativeRel = 0;
   RelType symbolicRel = 0;
+  RelType iRelSymbolicRel = 0;
   RelType tlsDescRel = 0;
   RelType tlsGotRel = 0;
   RelType tlsModuleIndexRel = 0;

lld/test/ELF/aarch64-funcinit64-invalid.s

@@ -0,0 +1,18 @@
+# REQUIRES: aarch64
+
+# RUN: llvm-mc -filetype=obj -triple=aarch64 %s -o %t.o
+# RUN: not ld.lld %t.o -o %t 2>&1 | FileCheck --check-prefix=ERR %s
+
+.rodata
+# ERR: error: relocation R_AARCH64_FUNCINIT64 cannot be used against local symbol
+.8byte func@FUNCINIT
+
+.data
+# ERR: error: relocation R_AARCH64_FUNCINIT64 cannot be used against ifunc symbol 'ifunc'
+.8byte ifunc@FUNCINIT
+
+.text
+func:
+.type ifunc, @gnu_indirect_function
+ifunc:
+ret

lld/test/ELF/aarch64-funcinit64.s

@@ -0,0 +1,19 @@
+# REQUIRES: aarch64
+
+# RUN: llvm-mc -filetype=obj -triple=aarch64 %s -o %t.o
+# RUN: ld.lld %t.o -o %t
+# RUN: llvm-readelf -s -r %t | FileCheck %s
+# RUN: ld.lld %t.o -o %t -pie
+# RUN: llvm-readelf -s -r %t | FileCheck %s
+# RUN: not ld.lld %t.o -o %t -shared 2>&1 | FileCheck --check-prefix=ERR %s
+
+.data
+# CHECK: R_AARCH64_IRELATIVE [[FOO:[0-9a-f]*]]
+# ERR: relocation R_AARCH64_FUNCINIT64 cannot be used against preemptible symbol 'foo'
+.8byte foo@FUNCINIT
+
+.text
+# CHECK: {{0*}}[[FOO]] {{.*}} foo
+.globl foo
+foo:
+ret

llvm/docs/LangRef.rst

@@ -3199,6 +3199,24 @@ A "convergencectrl" operand bundle is only valid on a ``convergent`` operation.
 When present, the operand bundle must contain exactly one value of token type.
 See the :doc:`ConvergentOperations` document for details.
 
+.. _deactivationsymbol:
+
+Deactivation Symbol Operand Bundles
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+A ``"deactivation-symbol"`` operand bundle is valid on the following
+instructions (AArch64 only):
+
+- Call to a normal function with ``notail`` attribute and a first argument and
+  return value of type ``ptr``.
+- Call to ``llvm.ptrauth.sign`` or ``llvm.ptrauth.auth`` intrinsics.
+
+This operand bundle specifies that if the deactivation symbol is defined
+to a valid value for the target, the marked instruction will return the
+value of its first argument instead of calling the specified function
+or intrinsic. This is achieved with ``PATCHINST`` relocations on the
+target instructions (see the AArch64 psABI for details).
+
 .. _moduleasm:
 
 Module-Level Inline Assembly
@@ -5249,7 +5267,7 @@ need to refer to the actual function body.
 Pointer Authentication Constants
 --------------------------------
 
-``ptrauth (ptr CST, i32 KEY[, i64 DISC[, ptr ADDRDISC]?]?)``
+``ptrauth (ptr CST, i32 KEY[, i64 DISC[, ptr ADDRDISC[, ptr DS]?]?]?)``
 
 A '``ptrauth``' constant represents a pointer with a cryptographic
 authentication signature embedded into some bits, as described in the
@@ -5278,6 +5296,11 @@ Otherwise, the expression is equivalent to:
     %tmp2 = call i64 @llvm.ptrauth.sign(i64 ptrtoint (ptr CST to i64), i32 KEY, i64 %tmp1)
     %val = inttoptr i64 %tmp2 to ptr
 
+If the deactivation symbol operand ``DS`` has a non-null value,
+the semantics are as if a :ref:`deactivation-symbol operand bundle
+<deactivationsymbol>` were added to the ``llvm.ptrauth.sign`` intrinsic
+calls above, with ``DS`` as the only operand.
+
 .. _constantexprs:
 
 Constant Expressions

llvm/include/llvm/BinaryFormat/ELFRelocs/AArch64.def

@@ -62,6 +62,7 @@ ELF_RELOC(R_AARCH64_LD64_GOTPAGE_LO15,               0x139)
 ELF_RELOC(R_AARCH64_PLT32,                           0x13a)
 ELF_RELOC(R_AARCH64_GOTPCREL32,                      0x13b)
 ELF_RELOC(R_AARCH64_PATCHINST,                       0x13c)
+ELF_RELOC(R_AARCH64_FUNCINIT64,                      0x13d)
 // General dynamic TLS relocations
 ELF_RELOC(R_AARCH64_TLSGD_ADR_PREL21,                0x200)
 ELF_RELOC(R_AARCH64_TLSGD_ADR_PAGE21,                0x201)

llvm/include/llvm/Bitcode/LLVMBitCodes.h

@@ -437,6 +437,8 @@ enum ConstantsCodes {
   CST_CODE_CE_GEP_WITH_INRANGE = 31,  // [opty, flags, range, n x operands]
   CST_CODE_CE_GEP = 32,               // [opty, flags, n x operands]
   CST_CODE_PTRAUTH = 33,              // [ptr, key, disc, addrdisc]
+  CST_CODE_PTRAUTH2 = 34,             // [ptr, key, disc, addrdisc,
+                                      //  deactivation_symbol]
 };
 
 /// CastOpcodes - These are values used in the bitcode files to encode which

llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h

@@ -159,6 +159,8 @@ class LLVM_ABI CallLowering {
 
     /// True if this call results in convergent operations.
     bool IsConvergent = true;
+
+    GlobalValue *DeactivationSymbol = nullptr;
   };
 
   /// Argument handling is mostly uniform between the four places that

llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h

@@ -56,6 +56,7 @@ struct MachineIRBuilderState {
   MDNode *PCSections = nullptr;
   /// MMRA Metadata to be set on any instruction we create.
   MDNode *MMRA = nullptr;
+  Value *DS = nullptr;
 
   /// \name Fields describing the insertion point.
   /// @{
@@ -369,6 +370,7 @@ class LLVM_ABI MachineIRBuilder {
     State.II = MI.getIterator();
     setPCSections(MI.getPCSections());
     setMMRAMetadata(MI.getMMRAMetadata());
+    setDeactivationSymbol(MI.getDeactivationSymbol());
   }
   /// @}
 
@@ -405,6 +407,9 @@ class LLVM_ABI MachineIRBuilder {
   /// Set the PC sections metadata to \p MD for all the next build instructions.
   void setMMRAMetadata(MDNode *MMRA) { State.MMRA = MMRA; }
 
+  Value *getDeactivationSymbol() { return State.DS; }
+  void setDeactivationSymbol(Value *DS) { State.DS = DS; }
+
   /// Get the current instruction's MMRA metadata.
   MDNode *getMMRAMetadata() { return State.MMRA; }
 

llvm/include/llvm/CodeGen/ISDOpcodes.h

@@ -1569,6 +1569,10 @@ enum NodeType {
   // Outputs: Output Chain
   CLEAR_CACHE,
 
+  // Untyped node storing deactivation symbol reference
+  // (DeactivationSymbolSDNode).
+  DEACTIVATION_SYMBOL,
+
   /// BUILTIN_OP_END - This must be the last enum value in this list.
   /// The target-specific pre-isel opcode values start here.
   BUILTIN_OP_END

llvm/include/llvm/CodeGen/MachineFunction.h

@@ -1207,7 +1207,7 @@ class LLVM_ABI MachineFunction {
       ArrayRef<MachineMemOperand *> MMOs, MCSymbol *PreInstrSymbol = nullptr,
       MCSymbol *PostInstrSymbol = nullptr, MDNode *HeapAllocMarker = nullptr,
       MDNode *PCSections = nullptr, uint32_t CFIType = 0,
-      MDNode *MMRAs = nullptr);
+      MDNode *MMRAs = nullptr, Value *DS = nullptr);
 
   /// Allocate a string and populate it with the given external symbol name.
   const char *createExternalSymbolName(StringRef Name);

llvm/include/llvm/CodeGen/MachineInstr.h

@@ -160,29 +160,33 @@ class MachineInstr
   ///
   /// This has to be defined eagerly due to the implementation constraints of
   /// `PointerSumType` where it is used.
-  class ExtraInfo final : TrailingObjects<ExtraInfo, MachineMemOperand *,
-                                          MCSymbol *, MDNode *, uint32_t> {
+  class ExtraInfo final
+      : TrailingObjects<ExtraInfo, MachineMemOperand *, MCSymbol *, MDNode *,
+                        uint32_t, Value *> {
   public:
     static ExtraInfo *create(BumpPtrAllocator &Allocator,
                              ArrayRef<MachineMemOperand *> MMOs,
                              MCSymbol *PreInstrSymbol = nullptr,
                              MCSymbol *PostInstrSymbol = nullptr,
                              MDNode *HeapAllocMarker = nullptr,
                              MDNode *PCSections = nullptr, uint32_t CFIType = 0,
-                             MDNode *MMRAs = nullptr) {
+                             MDNode *MMRAs = nullptr, Value *DS = nullptr) {
       bool HasPreInstrSymbol = PreInstrSymbol != nullptr;
       bool HasPostInstrSymbol = PostInstrSymbol != nullptr;
       bool HasHeapAllocMarker = HeapAllocMarker != nullptr;
       bool HasMMRAs = MMRAs != nullptr;
       bool HasCFIType = CFIType != 0;
       bool HasPCSections = PCSections != nullptr;
+      bool HasDS = DS != nullptr;
       auto *Result = new (Allocator.Allocate(
-          totalSizeToAlloc<MachineMemOperand *, MCSymbol *, MDNode *, uint32_t>(
+          totalSizeToAlloc<MachineMemOperand *, MCSymbol *, MDNode *, uint32_t,
+                           Value *>(
               MMOs.size(), HasPreInstrSymbol + HasPostInstrSymbol,
-              HasHeapAllocMarker + HasPCSections + HasMMRAs, HasCFIType),
+              HasHeapAllocMarker + HasPCSections + HasMMRAs, HasCFIType, HasDS),
           alignof(ExtraInfo)))
           ExtraInfo(MMOs.size(), HasPreInstrSymbol, HasPostInstrSymbol,
-                    HasHeapAllocMarker, HasPCSections, HasCFIType, HasMMRAs);
+                    HasHeapAllocMarker, HasPCSections, HasCFIType, HasMMRAs,
+                    HasDS);
 
       // Copy the actual data into the trailing objects.
       std::copy(MMOs.begin(), MMOs.end(),
@@ -203,6 +207,8 @@ class MachineInstr
         Result->getTrailingObjects<uint32_t>()[0] = CFIType;
       if (HasMMRAs)
         Result->getTrailingObjects<MDNode *>()[MDNodeIdx++] = MMRAs;
+      if (HasDS)
+        Result->getTrailingObjects<Value *>()[0] = DS;
 
       return Result;
     }
@@ -241,6 +247,10 @@ class MachineInstr
                       : nullptr;
     }
 
+    Value *getDeactivationSymbol() const {
+      return HasDS ? getTrailingObjects<Value *>()[0] : 0;
+    }
+
   private:
     friend TrailingObjects;
 
@@ -256,6 +266,7 @@ class MachineInstr
     const bool HasPCSections;
     const bool HasCFIType;
     const bool HasMMRAs;
+    const bool HasDS;
 
     // Implement the `TrailingObjects` internal API.
     size_t numTrailingObjects(OverloadToken<MachineMemOperand *>) const {
@@ -270,16 +281,19 @@ class MachineInstr
     size_t numTrailingObjects(OverloadToken<uint32_t>) const {
       return HasCFIType;
     }
+    size_t numTrailingObjects(OverloadToken<Value *>) const {
+      return HasDS;
+    }
 
     // Just a boring constructor to allow us to initialize the sizes. Always use
     // the `create` routine above.
     ExtraInfo(int NumMMOs, bool HasPreInstrSymbol, bool HasPostInstrSymbol,
               bool HasHeapAllocMarker, bool HasPCSections, bool HasCFIType,
-              bool HasMMRAs)
+              bool HasMMRAs, bool HasDS)
         : NumMMOs(NumMMOs), HasPreInstrSymbol(HasPreInstrSymbol),
           HasPostInstrSymbol(HasPostInstrSymbol),
           HasHeapAllocMarker(HasHeapAllocMarker), HasPCSections(HasPCSections),
-          HasCFIType(HasCFIType), HasMMRAs(HasMMRAs) {}
+          HasCFIType(HasCFIType), HasMMRAs(HasMMRAs), HasDS(HasDS) {}
   };
 
   /// Enumeration of the kinds of inline extra info available. It is important
@@ -868,6 +882,14 @@ class MachineInstr
     return nullptr;
   }
 
+  Value *getDeactivationSymbol() const {
+    if (!Info)
+      return nullptr;
+    if (ExtraInfo *EI = Info.get<EIIK_OutOfLine>())
+      return EI->getDeactivationSymbol();
+    return nullptr;
+  }
+
   /// Helper to extract a CFI type hash if one has been added.
   uint32_t getCFIType() const {
     if (!Info)
@@ -1970,6 +1992,8 @@ class MachineInstr
   /// Set the CFI type for the instruction.
   LLVM_ABI void setCFIType(MachineFunction &MF, uint32_t Type);
 
+  LLVM_ABI void setDeactivationSymbol(MachineFunction &MF, Value *DS);
+
   /// Return the MIFlags which represent both MachineInstrs. This
   /// should be used when merging two MachineInstrs into one. This routine does
   /// not modify the MIFlags of this MachineInstr.
@@ -2089,7 +2113,7 @@ class MachineInstr
   void setExtraInfo(MachineFunction &MF, ArrayRef<MachineMemOperand *> MMOs,
                     MCSymbol *PreInstrSymbol, MCSymbol *PostInstrSymbol,
                     MDNode *HeapAllocMarker, MDNode *PCSections,
-                    uint32_t CFIType, MDNode *MMRAs);
+                    uint32_t CFIType, MDNode *MMRAs, Value *DS);
 };
 
 /// Special DenseMapInfo traits to compare MachineInstr* by *value* of the