Merge remote-tracking branch 'otcshare_llvm/sycl-web' into llvmspirv_pulldown

Author: KornevNikita

.mailmap

@@ -37,3 +37,4 @@ LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
 Martin Storsjö <martin@martin.st>
 Ramkumar Ramachandra <r@artagnon.com> <artagnon@gmail.com>
 Saleem Abdulrasool <compnerd@compnerd.org>
+Tommy Chiang <ty1208chiang@gmail.com> <oToToT@users.noreply.github.com>

bolt/include/bolt/Core/BinaryFunction.h

@@ -1614,11 +1614,10 @@ class BinaryFunction {
 
   /// Dump function information to debug output. If \p PrintInstructions
   /// is true - include instruction disassembly.
-  void dump(bool PrintInstructions = true) const;
+  void dump() const;
 
   /// Print function information to the \p OS stream.
-  void print(raw_ostream &OS, std::string Annotation = "",
-             bool PrintInstructions = true);
+  void print(raw_ostream &OS, std::string Annotation = "");
 
   /// Print all relocations between \p Offset and \p Offset + \p Size in
   /// this function.

bolt/include/bolt/Core/BinarySection.h

@@ -157,7 +157,7 @@ class BinarySection {
   BinarySection(BinaryContext &BC, SectionRef Section)
       : BC(BC), Name(getName(Section)), Section(Section),
         Contents(getContents(Section)), Address(Section.getAddress()),
-        Size(Section.getSize()), Alignment(Section.getAlignment()),
+        Size(Section.getSize()), Alignment(Section.getAlignment().value()),
         OutputName(Name), SectionNumber(++Count) {
     if (isELF()) {
       ELFType = ELFSectionRef(Section).getType();

bolt/include/bolt/Core/DebugData.h

@@ -1003,7 +1003,7 @@ class DebugAbbrevWriter {
   DWARFContext &Context;
 
   /// DWO ID used to identify unit contribution in DWP.
-  Optional<uint64_t> DWOId;
+  std::optional<uint64_t> DWOId;
 
   /// Add abbreviations from compile/type \p Unit to the writer.
   void addUnitAbbreviations(DWARFUnit &Unit);
@@ -1020,7 +1020,7 @@ class DebugAbbrevWriter {
   ///       If type units are used, the caller is responsible for verifying
   ///       that abbreviations are shared by CU and TUs.
   DebugAbbrevWriter(DWARFContext &Context,
-                    Optional<uint64_t> DWOId = std::nullopt)
+                    std::optional<uint64_t> DWOId = std::nullopt)
       : Context(Context), DWOId(DWOId) {}
 
   DebugAbbrevWriter(const DebugAbbrevWriter &) = delete;

bolt/include/bolt/Core/MCPlusBuilder.h

@@ -1420,6 +1420,11 @@ class MCPlusBuilder {
     llvm_unreachable("not implemented");
   }
 
+  virtual bool matchAdrpAddPair(const MCInst &Adrp, const MCInst &Add) const {
+    llvm_unreachable("not implemented");
+    return false;
+  }
+
   virtual int getShortJmpEncodingSize() const {
     llvm_unreachable("not implemented");
   }

bolt/include/bolt/Core/Relocation.h

@@ -58,8 +58,10 @@ struct Relocation {
   /// Skip relocations that we don't want to handle in BOLT
   static bool skipRelocationType(uint64_t Type);
 
-  /// Handle special cases when relocation should not be processed by BOLT
-  static bool skipRelocationProcess(uint64_t Type, uint64_t Contents);
+  /// Handle special cases when relocation should not be processed by BOLT or
+  /// change relocation \p Type to proper one before continuing if \p Contents
+  /// and \P Type mismatch occured.
+  static bool skipRelocationProcess(uint64_t &Type, uint64_t Contents);
 
   // Adjust value depending on relocation type (make it PC relative or not)
   static uint64_t adjustValue(uint64_t Type, uint64_t Value,

bolt/include/bolt/Passes/FixRelaxationPass.h

@@ -0,0 +1,40 @@
+//===- bolt/Passes/ADRRelaxationPass.h --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the FixRelaxations class, which locates instructions with
+// wrong targets and fixes them. Such problems usually occures when linker
+// relaxes (changes) instructions, but doesn't fix relocations types properly
+// for them.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef BOLT_PASSES_FIXRELAXATIONPASS_H
+#define BOLT_PASSES_FIXRELAXATIONPASS_H
+
+#include "bolt/Passes/BinaryPasses.h"
+
+namespace llvm {
+namespace bolt {
+
+class FixRelaxations : public BinaryFunctionPass {
+  void runOnFunction(BinaryFunction &Function);
+
+public:
+  explicit FixRelaxations(const cl::opt<bool> &PrintPass)
+      : BinaryFunctionPass(PrintPass) {}
+
+  const char *getName() const override { return "fix-relaxations"; }
+
+  /// Pass entry point
+  void runOnFunctions(BinaryContext &BC) override;
+};
+
+} // namespace bolt
+} // namespace llvm
+
+#endif

bolt/include/bolt/Rewrite/RewriteInstance.h

@@ -247,7 +247,7 @@ class RewriteInstance {
   /// The \p SymbolName, \p SymbolAddress, \p Addend and \p ExtractedValue
   /// parameters will be set on success. The \p Skip argument indicates
   /// that the relocation was analyzed, but it must not be processed.
-  bool analyzeRelocation(const object::RelocationRef &Rel, uint64_t RType,
+  bool analyzeRelocation(const object::RelocationRef &Rel, uint64_t &RType,
                          std::string &SymbolName, bool &IsSectionRelocation,
                          uint64_t &SymbolAddress, int64_t &Addend,
                          uint64_t &ExtractedValue, bool &Skip) const;

bolt/lib/Core/BinaryFunction.cpp

@@ -392,7 +392,7 @@ bool BinaryFunction::isForwardCall(const MCSymbol *CalleeSymbol) const {
   }
 }
 
-void BinaryFunction::dump(bool PrintInstructions) const {
+void BinaryFunction::dump() const {
   // getDynoStats calls FunctionLayout::updateLayoutIndices and
   // BasicBlock::analyzeBranch. The former cannot be const, but should be
   // removed, the latter should be made const, but seems to require refactoring.
@@ -402,11 +402,10 @@ void BinaryFunction::dump(bool PrintInstructions) const {
   // modified. Only BinaryBasicBlocks are actually modified (if it all) and we
   // have mutable pointers to those regardless whether this function is
   // const-qualified or not.
-  const_cast<BinaryFunction &>(*this).print(dbgs(), "", PrintInstructions);
+  const_cast<BinaryFunction &>(*this).print(dbgs(), "");
 }
 
-void BinaryFunction::print(raw_ostream &OS, std::string Annotation,
-                           bool PrintInstructions) {
+void BinaryFunction::print(raw_ostream &OS, std::string Annotation) {
   if (!opts::shouldPrint(*this))
     return;
 
@@ -485,7 +484,7 @@ void BinaryFunction::print(raw_ostream &OS, std::string Annotation,
 
   OS << "\n}\n";
 
-  if (opts::PrintDynoStatsOnly || !PrintInstructions || !BC.InstPrinter)
+  if (opts::PrintDynoStatsOnly || !BC.InstPrinter)
     return;
 
   // Offset of the instruction in function.

bolt/lib/Core/CMakeLists.txt

@@ -5,6 +5,7 @@ set(LLVM_LINK_COMPONENTS
   MCDisassembler
   Object
   Support
+  TargetParser
   )
 
 add_llvm_library(LLVMBOLTCore

bolt/lib/Core/Relocation.cpp

@@ -171,11 +171,11 @@ bool skipRelocationTypeAArch64(uint64_t Type) {
   return Type == ELF::R_AARCH64_NONE || Type == ELF::R_AARCH64_LD_PREL_LO19;
 }
 
-bool skipRelocationProcessX86(uint64_t Type, uint64_t Contents) {
+bool skipRelocationProcessX86(uint64_t &Type, uint64_t Contents) {
   return false;
 }
 
-bool skipRelocationProcessAArch64(uint64_t Type, uint64_t Contents) {
+bool skipRelocationProcessAArch64(uint64_t &Type, uint64_t Contents) {
   auto IsMov = [](uint64_t Contents) -> bool {
     // The bits 28-23 are 0b100101
     return (Contents & 0x1f800000) == 0x12800000;
@@ -191,12 +191,25 @@ bool skipRelocationProcessAArch64(uint64_t Type, uint64_t Contents) {
     return (Contents & 0x9f000000) == 0x10000000;
   };
 
+  auto IsAddImm = [](uint64_t Contents) -> bool {
+    // The bits 30-23 are 0b00100010
+    return (Contents & 0x7F800000) == 0x11000000;
+  };
+
   auto IsNop = [](uint64_t Contents) -> bool { return Contents == 0xd503201f; };
 
   // The linker might eliminate the instruction and replace it with NOP, ignore
   if (IsNop(Contents))
     return true;
 
+  // The linker might relax ADRP+LDR instruction sequence for loading symbol
+  // address from GOT table to ADRP+ADD sequence that would point to the
+  // binary-local symbol. Change relocation type in order to process it right.
+  if (Type == ELF::R_AARCH64_LD64_GOT_LO12_NC && IsAddImm(Contents)) {
+    Type = ELF::R_AARCH64_ADD_ABS_LO12_NC;
+    return false;
+  }
+
   // The linker might perform TLS relocations relaxations, such as
   // changed TLS access model (e.g. changed global dynamic model
   // to initial exec), thus changing the instructions. The static
@@ -548,7 +561,7 @@ bool Relocation::skipRelocationType(uint64_t Type) {
   return skipRelocationTypeX86(Type);
 }
 
-bool Relocation::skipRelocationProcess(uint64_t Type, uint64_t Contents) {
+bool Relocation::skipRelocationProcess(uint64_t &Type, uint64_t Contents) {
   if (Arch == Triple::aarch64)
     return skipRelocationProcessAArch64(Type, Contents);
   return skipRelocationProcessX86(Type, Contents);

bolt/lib/Passes/CMakeLists.txt

@@ -13,6 +13,7 @@ add_llvm_library(LLVMBOLTPasses
   DataflowInfoManager.cpp
   FrameAnalysis.cpp
   FrameOptimizer.cpp
+  FixRelaxationPass.cpp
   HFSort.cpp
   HFSortPlus.cpp
   Hugify.cpp
@@ -56,6 +57,7 @@ add_llvm_library(LLVMBOLTPasses
   AsmPrinter
   MC
   Support
+  TargetParser
   TransformUtils
   )
 

bolt/lib/Passes/FixRelaxationPass.cpp

@@ -0,0 +1,63 @@
+#include "bolt/Passes/FixRelaxationPass.h"
+#include "bolt/Core/ParallelUtilities.h"
+
+using namespace llvm;
+
+namespace llvm {
+namespace bolt {
+
+// This function finds ADRP+ADD instruction sequences that originally before
+// linker relaxations were ADRP+LDR. We've modified LDR/ADD relocation properly
+// during relocation reading, so its targeting right symbol. As for ADRP its
+// target is wrong before this pass since we won't be able to recognize and
+// properly change R_AARCH64_ADR_GOT_PAGE relocation to
+// R_AARCH64_ADR_PREL_PG_HI21 during relocation reading. Now we're searching for
+// ADRP+ADD sequences, checking that ADRP points to the GOT-table symbol and the
+// target of ADD is another symbol. When found change ADRP symbol reference to
+// the ADDs one.
+void FixRelaxations::runOnFunction(BinaryFunction &BF) {
+  BinaryContext &BC = BF.getBinaryContext();
+  for (BinaryBasicBlock &BB : BF) {
+    for (auto II = BB.begin(); II != BB.end(); ++II) {
+      MCInst &Adrp = *II;
+      if (BC.MIB->isPseudo(Adrp) || !BC.MIB->isADRP(Adrp))
+        continue;
+
+      const MCSymbol *AdrpSymbol = BC.MIB->getTargetSymbol(Adrp);
+      if (!AdrpSymbol || AdrpSymbol->getName() != "__BOLT_got_zero")
+        continue;
+
+      auto NextII = std::next(II);
+      if (NextII == BB.end())
+        continue;
+
+      const MCInst &Add = *NextII;
+      if (!BC.MIB->matchAdrpAddPair(Adrp, Add))
+        continue;
+
+      const MCSymbol *Symbol = BC.MIB->getTargetSymbol(Add);
+      if (!Symbol || AdrpSymbol == Symbol)
+        continue;
+
+      const int64_t Addend = BC.MIB->getTargetAddend(Add);
+      BC.MIB->setOperandToSymbolRef(Adrp, /*OpNum*/ 1, Symbol, Addend,
+                                    BC.Ctx.get(), ELF::R_AARCH64_NONE);
+    }
+  }
+}
+
+void FixRelaxations::runOnFunctions(BinaryContext &BC) {
+  if (!BC.isAArch64() || !BC.HasRelocations)
+    return;
+
+  ParallelUtilities::WorkFuncTy WorkFun = [&](BinaryFunction &BF) {
+    runOnFunction(BF);
+  };
+
+  ParallelUtilities::runOnEachFunction(
+      BC, ParallelUtilities::SchedulingPolicy::SP_INST_LINEAR, WorkFun, nullptr,
+      "FixRelaxations");
+}
+
+} // namespace bolt
+} // namespace llvm

bolt/lib/Passes/Hugify.cpp

@@ -47,4 +47,4 @@ void HugePage::runOnFunctions(BinaryContext &BC) {
                        BC.MIB->createSymbolTrampoline(StartSym, BC.Ctx.get()));
 }
 } // namespace bolt
-} // namespace llvm
+} // namespace llvm

bolt/lib/Passes/MCF.cpp

@@ -448,10 +448,10 @@ void estimateEdgeCounts(BinaryFunction &BF) {
     computeEdgeWeights<BinaryBasicBlock *>(BF, SuccEdgeWeights);
   }
   if (opts::EqualizeBBCounts) {
-    LLVM_DEBUG(BF.print(dbgs(), "before equalize BB counts", true));
+    LLVM_DEBUG(BF.print(dbgs(), "before equalize BB counts"));
     auto Info = DataflowInfoManager(BF, nullptr, nullptr);
     equalizeBBCounts(Info, BF);
-    LLVM_DEBUG(BF.print(dbgs(), "after equalize BB counts", true));
+    LLVM_DEBUG(BF.print(dbgs(), "after equalize BB counts"));
   }
   if (opts::IterativeGuess)
     guessEdgeByIterativeApproach(BF);

bolt/lib/Profile/DataAggregator.cpp

@@ -199,10 +199,10 @@ void DataAggregator::abort() {
   std::string Error;
 
   // Kill subprocesses in case they are not finished
-  sys::Wait(TaskEventsPPI.PI, 1, false, &Error);
-  sys::Wait(MMapEventsPPI.PI, 1, false, &Error);
-  sys::Wait(MainEventsPPI.PI, 1, false, &Error);
-  sys::Wait(MemEventsPPI.PI, 1, false, &Error);
+  sys::Wait(TaskEventsPPI.PI, 1, &Error);
+  sys::Wait(MMapEventsPPI.PI, 1, &Error);
+  sys::Wait(MainEventsPPI.PI, 1, &Error);
+  sys::Wait(MemEventsPPI.PI, 1, &Error);
 
   deleteTempFiles();
 
@@ -487,7 +487,7 @@ Error DataAggregator::preprocessProfile(BinaryContext &BC) {
     std::string Error;
     outs() << "PERF2BOLT: waiting for perf " << Name
            << " collection to finish...\n";
-    sys::ProcessInfo PI = sys::Wait(Process.PI, 0, true, &Error);
+    sys::ProcessInfo PI = sys::Wait(Process.PI, std::nullopt, &Error);
 
     if (!Error.empty()) {
       errs() << "PERF-ERROR: " << PerfPath << ": " << Error << "\n";
@@ -572,7 +572,7 @@ Error DataAggregator::preprocessProfile(BinaryContext &BC) {
 
   // Special handling for memory events
   std::string Error;
-  sys::ProcessInfo PI = sys::Wait(MemEventsPPI.PI, 0, true, &Error);
+  sys::ProcessInfo PI = sys::Wait(MemEventsPPI.PI, std::nullopt, &Error);
   if (PI.ReturnCode != 0) {
     ErrorOr<std::unique_ptr<MemoryBuffer>> MB =
         MemoryBuffer::getFileOrSTDIN(MemEventsPPI.StderrPath.data());

bolt/lib/Rewrite/BinaryPassManager.cpp

@@ -12,6 +12,7 @@
 #include "bolt/Passes/AllocCombiner.h"
 #include "bolt/Passes/AsmDump.h"
 #include "bolt/Passes/CMOVConversion.h"
+#include "bolt/Passes/FixRelaxationPass.h"
 #include "bolt/Passes/FrameOptimizer.h"
 #include "bolt/Passes/Hugify.h"
 #include "bolt/Passes/IdenticalCodeFolding.h"
@@ -179,6 +180,11 @@ static cl::opt<bool>
     PrintStoke("print-stoke", cl::desc("print functions after stoke analysis"),
                cl::cat(BoltOptCategory));
 
+static cl::opt<bool>
+    PrintFixRelaxations("print-fix-relaxations",
+                        cl::desc("print functions after fix relaxations pass"),
+                        cl::cat(BoltOptCategory));
+
 static cl::opt<bool> PrintVeneerElimination(
     "print-veneer-elimination",
     cl::desc("print functions after veneer elimination pass"),
@@ -298,7 +304,7 @@ void BinaryFunctionPassManager::runPasses() {
       if (!Pass->shouldPrint(Function))
         continue;
 
-      Function.print(outs(), Message, true);
+      Function.print(outs(), Message);
 
       if (opts::DumpDotAll)
         Function.dumpGraphForPass(PassIdName);
@@ -315,9 +321,12 @@ void BinaryFunctionPassManager::runAllPasses(BinaryContext &BC) {
   Manager.registerPass(std::make_unique<AsmDumpPass>(),
                        opts::AsmDump.getNumOccurrences());
 
-  if (BC.isAArch64())
+  if (BC.isAArch64()) {
+    Manager.registerPass(std::make_unique<FixRelaxations>(PrintFixRelaxations));
+
     Manager.registerPass(
         std::make_unique<VeneerElimination>(PrintVeneerElimination));
+  }
 
   // Here we manage dependencies/order manually, since passes are run in the
   // order they're registered.

bolt/lib/Rewrite/CMakeLists.txt

@@ -5,6 +5,7 @@ set(LLVM_LINK_COMPONENTS
   MC
   Object
   Support
+  TargetParser
   )
 
 set(TARGET_LINK_LIBRARIES