llvm-D28786-callclearance.patch

diff --git a/include/llvm/Target/TargetInstrInfo.h b/include/llvm/Target/TargetInstrInfo.h
index 83515bc..65b435a 100644
--- a/include/llvm/Target/TargetInstrInfo.h
+++ b/include/llvm/Target/TargetInstrInfo.h
@@ -1440,6 +1440,17 @@ public:
   virtual void breakPartialRegDependency(MachineInstr &MI, unsigned OpNum,
                                          const TargetRegisterInfo *TRI) const {}
 
+  /// May return true if the instruction in question is a dependency breaking
+  /// instruction. If so, the register number for which it is dependency
+  /// breaking should be returned in `OutReg`. It is prefereable to return
+  /// false if the result cannot be determined. This would at worst result
+  /// in the insertion of an unnecessary instruction, while the other
+  /// alternative could result in significant false-dependency penalties.
+  virtual bool isDependencyBreak(MachineInstr &MI,
+                                 unsigned *OutReg = nullptr) const {
+    return false;
+  }
+
   /// Create machine specific model for scheduling.
   virtual DFAPacketizer *
   CreateTargetScheduleState(const TargetSubtargetInfo &) const {
diff --git a/lib/CodeGen/ExecutionDepsFix.cpp b/lib/CodeGen/ExecutionDepsFix.cpp
index 6ac1db4..63065ea 100644
--- a/lib/CodeGen/ExecutionDepsFix.cpp
+++ b/lib/CodeGen/ExecutionDepsFix.cpp
@@ -214,13 +214,18 @@ private:
   bool isBlockDone(MachineBasicBlock *);
   void processBasicBlock(MachineBasicBlock *MBB, bool PrimaryPass, bool Done);
   void updateSuccessors(MachineBasicBlock *MBB, bool Primary, bool Done);
-  bool visitInstr(MachineInstr *);
+  bool visitInstr(MachineInstr *, bool PrimaryPass);
   void processDefs(MachineInstr *, bool BlockDone, bool Kill);
   void visitSoftInstr(MachineInstr*, unsigned mask);
   void visitHardInstr(MachineInstr*, unsigned domain);
-  void pickBestRegisterForUndef(MachineInstr *MI, unsigned OpIdx,
-                                unsigned Pref);
+  void pickBestRegisterForUndef(MachineInstr *MI, unsigned OpIdx, unsigned Pref,
+                                bool &TrueDependency);
   bool shouldBreakDependence(MachineInstr*, unsigned OpIdx, unsigned Pref);
+
+  // Undef Reads
+  void collapseUndefReads(unsigned from, unsigned to, unsigned Reg);
+  unsigned updateChooseableRegs(SparseSet<unsigned> &,
+                                const TargetRegisterClass *, bool);
   void processUndefReads(MachineBasicBlock*);
 };
 }
@@ -394,11 +399,19 @@ void ExeDepsFix::enterBasicBlock(MachineBasicBlock *MBB) {
 
   // This is the entry block.
   if (MBB->pred_empty()) {
+    // Treat all registers as being defined just before the first instruction.
+    // Howver, we want the logic later to prefer non live-ins over live-ins,
+    // so pretend the live-ins were defined slightly later.
+    // We used to only do this for live-ins, but that's a bit of a gamble.
+    // If our caller does arithmetic with these registers is is quite likely
+    // that it will have used registers beyond the ones that are live here.
+    // Given the immense penalty for getting this wrong, being conservative
+    // here seems worth it.
+    for (unsigned rx = 0; rx != NumRegs; ++rx) {
+      LiveRegs[rx].Def = -2;
+    }
     for (const auto &LI : MBB->liveins()) {
       for (int rx : regIndices(LI.PhysReg)) {
-        // Treat function live-ins as if they were defined just before the first
-        // instruction.  Usually, function arguments are set up immediately
-        // before the call.
         LiveRegs[rx].Def = -1;
       }
     }
@@ -470,24 +483,36 @@ void ExeDepsFix::leaveBasicBlock(MachineBasicBlock *MBB) {
   LiveRegs = nullptr;
 }
 
-bool ExeDepsFix::visitInstr(MachineInstr *MI) {
-  // Update instructions with explicit execution domains.
-  std::pair<uint16_t, uint16_t> DomP = TII->getExecutionDomain(*MI);
-  if (DomP.first) {
-    if (DomP.second)
-      visitSoftInstr(MI, DomP.second);
-    else
-      visitHardInstr(MI, DomP.first);
+bool ExeDepsFix::visitInstr(MachineInstr *MI, bool PrimaryPass) {
+  bool Kill = false;
+
+  if (PrimaryPass) {
+    // Update instructions with explicit execution domains.
+    std::pair<uint16_t, uint16_t> DomP = TII->getExecutionDomain(*MI);
+    if (DomP.first) {
+      if (DomP.second)
+        visitSoftInstr(MI, DomP.second);
+      else
+        visitHardInstr(MI, DomP.first);
+    }
+    Kill = !DomP.first;
   }
 
-  return !DomP.first;
+  // If this is a call, pretend all registers we are considering are def'd here.
+  // We have no idea which registers the callee may use.
+  if (MI->isCall()) {
+    for (unsigned i = 0, e = NumRegs; i != e; ++i)
+      LiveRegs[i].Def = CurInstr;
+  }
+
+  return Kill;
 }
 
 /// \brief Helps avoid false dependencies on undef registers by updating the
 /// machine instructions' undef operand to use a register that the instruction
 /// is truly dependent on, or use a register with clearance higher than Pref.
 void ExeDepsFix::pickBestRegisterForUndef(MachineInstr *MI, unsigned OpIdx,
-                                          unsigned Pref) {
+                                          unsigned Pref, bool &TrueDependency) {
   MachineOperand &MO = MI->getOperand(OpIdx);
   assert(MO.isUndef() && "Expected undef machine operand");
 
@@ -510,6 +535,7 @@ void ExeDepsFix::pickBestRegisterForUndef(MachineInstr *MI, unsigned OpIdx,
     // We found a true dependency - replace the undef register with the true
     // dependency.
     MO.setReg(CurrMO.getReg());
+    TrueDependency = true;
     return;
   }
 
@@ -571,9 +597,14 @@ void ExeDepsFix::processDefs(MachineInstr *MI, bool BlockDone, bool Kill) {
   if (BlockDone) {
     unsigned Pref = TII->getUndefRegClearance(*MI, OpNum, TRI);
     if (Pref) {
-      pickBestRegisterForUndef(MI, OpNum, Pref);
-      if (shouldBreakDependence(MI, OpNum, Pref))
+      bool TrueDependency = false;
+      pickBestRegisterForUndef(MI, OpNum, Pref, TrueDependency);
+      // Don't bother adding true dependencies to UndefReads. All we'd find out
+      // is that the register is live (since this very instruction depends on
+      // it), so we can't do anything.
+      if (!TrueDependency && shouldBreakDependence(MI, OpNum, Pref)) {
         UndefReads.push_back(std::make_pair(MI, OpNum));
+      }
     }
   }
   const MCInstrDesc &MCID = MI->getDesc();
@@ -606,9 +637,52 @@ void ExeDepsFix::processDefs(MachineInstr *MI, bool BlockDone, bool Kill) {
         kill(rx);
     }
   }
+  unsigned DepReg = 0;
+  if (TII->isDependencyBreak(*MI, &DepReg)) {
+    for (int rx : regIndices(DepReg)) {
+      // This instruction is a dependency break, so there are no clearance
+      // issues, reset the counter.
+      LiveRegs[rx].Def = -(1 << 20);
+    }
+  }
   ++CurInstr;
 }
 
+// Set the undef read register to `Reg` for all UndefReads in the range
+// [from,to).
+void ExeDepsFix::collapseUndefReads(unsigned from, unsigned to, unsigned Reg) {
+  if (from >= to)
+    return;
+  for (unsigned i = from; i < to; ++i) {
+    MachineInstr *MI = std::get<0>(UndefReads[i]);
+    unsigned OpIdx = std::get<1>(UndefReads[i]);
+    MachineOperand &MO = MI->getOperand(OpIdx);
+    MO.setReg(Reg);
+  }
+  TII->breakPartialRegDependency(*std::get<0>(UndefReads[from]),
+                                 std::get<1>(UndefReads[from]), TRI);
+}
+
+unsigned ExeDepsFix::updateChooseableRegs(SparseSet<unsigned> &ChoosableRegs,
+                                          const TargetRegisterClass *OpRC,
+                                          bool add) {
+  unsigned LowestValid = (unsigned)-1;
+
+  for (auto Reg : OpRC->getRegisters()) {
+    if (LiveRegSet.contains(Reg))
+      ChoosableRegs.erase(Reg);
+    else if (add) {
+      ChoosableRegs.insert(Reg);
+      if (LowestValid == (unsigned)-1)
+        LowestValid = Reg;
+    } else if (ChoosableRegs.count(Reg) == 1) {
+      if (LowestValid == (unsigned)-1)
+        LowestValid = Reg;
+    }
+  }
+  return LowestValid;
+}
+
 /// \break Break false dependencies on undefined register reads.
 ///
 /// Walk the block backward computing precise liveness. This is expensive, so we
@@ -619,31 +693,87 @@ void ExeDepsFix::processUndefReads(MachineBasicBlock *MBB) {
   if (UndefReads.empty())
     return;
 
+  // We want to be slightly clever here, to avoid the following common pattern:
+  // Suppose we have some instruction `vrandom %in, %out` and the following code
+  //    vrandom %xmm0<undef>, %xmm0<def>
+  //    vrandom %xmm1<undef>, %xmm1<def>
+  //    vrandom %xmm2<undef>, %xmm2<def>
+  //    vrandom %xmm3<undef>, %xmm3<def>
+  // The earlier logic likes to produce these, because it picks the first
+  // register
+  // to break ties in clearance. However, most register allocators pick the dest
+  // register the same way. Naively, we'd have to insert a dependency break,
+  // before every instruction above. However, what we really want is
+  //    vxorps %xmm3, %xmm3, %xmm3
+  //    vrandom %xmm3<undef>, %xmm0<def>
+  //    vrandom %xmm3<undef>, %xmm1<def>
+  //    vrandom %xmm3<undef>, %xmm2<def>
+  //    vrandom %xmm3<undef>, %xmm3<def>
+  // To do so, we walk backwards and cumulatively keep track of which registers
+  // we can use to break the dependency. Then, once the set has collapsed, we
+  // reset the undef read register for all following instructions.
+
   // Collect this block's live out register units.
   LiveRegSet.init(TRI);
   // We do not need to care about pristine registers as they are just preserved
   // but not actually used in the function.
   LiveRegSet.addLiveOutsNoPristines(*MBB);
 
-  MachineInstr *UndefMI = UndefReads.back().first;
-  unsigned OpIdx = UndefReads.back().second;
+  SparseSet<unsigned> ChoosableRegs;
+  ChoosableRegs.setUniverse(TRI->getNumRegs());
+
+  unsigned LastValid = (unsigned)-1;
+  const TargetRegisterClass *LastOpRC = nullptr;
+  size_t i, LastInit;
+  i = LastInit = UndefReads.size() - 1;
+  MachineInstr *UndefMI = std::get<0>(UndefReads[i]);
 
   for (MachineInstr &I : make_range(MBB->rbegin(), MBB->rend())) {
     // Update liveness, including the current instruction's defs.
     LiveRegSet.stepBackward(I);
 
+    // This ensures that we don't accidentally pick a register whose live region
+    // lies entirely between two undef reads (since that would defeat the
+    // purpose of breaking the dependency).
+    for (auto LiveReg : LiveRegSet)
+      ChoosableRegs.erase(LiveReg);
+
     if (UndefMI == &I) {
-      if (!LiveRegSet.contains(UndefMI->getOperand(OpIdx).getReg()))
-        TII->breakPartialRegDependency(*UndefMI, OpIdx, TRI);
+      unsigned OpIdx = std::get<1>(UndefReads[i]);
+      // Get the undef operand's register class
+      const TargetRegisterClass *OpRC =
+          TII->getRegClass(UndefMI->getDesc(), OpIdx, TRI, *MF);
+      if (OpRC != LastOpRC || ChoosableRegs.size() == 0) {
+        if (LastInit != i) {
+          if (LastValid != (unsigned)-1)
+            collapseUndefReads(i + 1, LastInit + 1, LastValid);
+          ChoosableRegs.clear();
+          LastInit = i;
+        }
+      }
+
+      unsigned LowestValid =
+          updateChooseableRegs(ChoosableRegs, OpRC, LastInit == i);
+
+      if (ChoosableRegs.size() == 0) {
+        if (LastInit != i) {
+          if (LastValid != (unsigned)-1)
+            collapseUndefReads(i + 1, LastInit + 1, LastValid);
+          LowestValid = updateChooseableRegs(ChoosableRegs, OpRC, true);
+          LastInit = i;
+        }
+      }
+      LastValid = LowestValid;
+      LastOpRC = OpRC;
 
-      UndefReads.pop_back();
-      if (UndefReads.empty())
-        return;
+      if (i == 0)
+        break;
 
-      UndefMI = UndefReads.back().first;
-      OpIdx = UndefReads.back().second;
+      UndefMI = std::get<0>(UndefReads[--i]);
     }
   }
+  if (LastValid != (unsigned)-1)
+    collapseUndefReads(0, LastInit + 1, LastValid);
 }
 
 // A hard instruction only works in one domain. All input registers will be
@@ -793,9 +923,7 @@ void ExeDepsFix::processBasicBlock(MachineBasicBlock *MBB, bool PrimaryPass,
   enterBasicBlock(MBB);
   for (MachineInstr &MI : *MBB) {
     if (!MI.isDebugValue()) {
-      bool Kill = false;
-      if (PrimaryPass)
-        Kill = visitInstr(&MI);
+      bool Kill = visitInstr(&MI, PrimaryPass);
       processDefs(&MI, isBlockDone(MBB), Kill);
     }
   }
diff --git a/lib/Target/X86/X86InstrInfo.cpp b/lib/Target/X86/X86InstrInfo.cpp
index 5793597..f31c97e 100644
--- a/lib/Target/X86/X86InstrInfo.cpp
+++ b/lib/Target/X86/X86InstrInfo.cpp
@@ -7496,6 +7496,23 @@ void X86InstrInfo::breakPartialRegDependency(
   }
 }
 
+bool X86InstrInfo::isDependencyBreak(MachineInstr &MI, unsigned *OutReg) const {
+  unsigned Opc = MI.getOpcode();
+  if (!(Opc == X86::VXORPSrr || Opc == X86::VXORPDrr || Opc == X86::XORPSrr ||
+        Opc == X86::XORPDrr))
+    return false;
+  unsigned Reg = 0;
+  for (unsigned i = 0; i < MI.getNumOperands(); ++i) {
+    const MachineOperand &MO = MI.getOperand(i);
+    if (!MO.isReg() || (Reg != 0 && MO.getReg() != Reg))
+      return false;
+    Reg = MO.getReg();
+  }
+  if (OutReg)
+    *OutReg = Reg;
+  return true;
+}
+
 MachineInstr *
 X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
                                     ArrayRef<unsigned> Ops,
diff --git a/lib/Target/X86/X86InstrInfo.h b/lib/Target/X86/X86InstrInfo.h
index 8d74617..fa86882 100644
--- a/lib/Target/X86/X86InstrInfo.h
+++ b/lib/Target/X86/X86InstrInfo.h
@@ -484,6 +484,7 @@ public:
                                 const TargetRegisterInfo *TRI) const override;
   void breakPartialRegDependency(MachineInstr &MI, unsigned OpNum,
                                  const TargetRegisterInfo *TRI) const override;
+  bool isDependencyBreak(MachineInstr &MI, unsigned *OutReg) const override;
 
   MachineInstr *foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
                                       unsigned OpNum,